Mercurial > jsparser

comparison jsparser.js @ 0:94358cebb977

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
origination
author Atul Varma <varmaa@toolness.com>
date Fri, 29 May 2009 15:40:53 -0700
parents
children 0f1ff7d5a524
comparison
equal deleted inserted replaced
-1:000000000000 0:94358cebb977
1 var Symbols = {
2 semicolon: ';',
3 leftBracket: '[',
4 rightBracket: ']',
5 leftBrace: '{',
6 rightBrace: '}',
7 leftParen: '(',
8 rightParen: ')',
9 plus: '+',
10 number: /^[0-9]+/,
11 name: /^[A-Za-z]+[0-9A-Za-z_]*/,
12 newline: '\n',
13 whitespace: /^[ \t]+/
14 };
15
16 function tokenize(options) {
17 var symbols = options.symbols;
18 var text = options.text;
19
20 var tokens = [];
21 var lineNo = 0;
22 var charNo = 0;
23
24 while (text) {
25 var wasSymbolFound = false;
26 for (name in symbols) {
27 var symbol = symbols[name];
28 if (typeof(symbol) == 'string') {
29 if (text.indexOf(symbol) == 0) {
30 if (name != 'newline') {
31 tokens.push({name: name,
32 charNo: charNo,
33 lineNo: lineNo});
34 charNo += symbol.length;
35 } else {
36 lineNo += 1;
37 charNo = 0;
38 }
39 text = text.slice(symbol.length);
40 wasSymbolFound = true;
41 break;
42 }
43 } else if (typeof(symbol) == 'function') {
44 // It's a Regular Expression.
45 var match = text.match(symbol);
46 if (match) {
47 match = match[0];
48 if (name != 'whitespace') {
49 tokens.push({name: name,
50 value: match,
51 charNo: charNo,
52 lineNo: lineNo});
53 }
54 charNo += match.length;
55 text = text.slice(match.length);
56 wasSymbolFound = true;
57 break;
58 }
59 } else
60 throw new Error('Unknown symbol type: ' + symbol);
61 }
62 if (!wasSymbolFound)
63 throw new Error("I have no idea what this is: " + text);
64 }
65
66 return tokens;
67 }
68
69 // test code
70
71 var code = 'blarg(); if (x+1) {}';
72 var tokens = tokenize({symbols: Symbols,
73 text: code});
74 tokens.forEach(
75 function(token) {
76 var repr = token.name;
77 if (token.value)
78 repr += ":" + token.value;
79 print(repr);
80 });