changeset 2:032f64260794

refactored tokenizer
author Atul Varma <varmaa@toolness.com>
date Fri, 29 May 2009 22:16:51 -0700
parents 0f1ff7d5a524
children f174153281a9
files jsparser.js
diffstat 1 files changed, 33 insertions(+), 29 deletions(-) [+]
line wrap: on
line diff
--- a/jsparser.js	Fri May 29 18:13:55 2009 -0700
+++ b/jsparser.js	Fri May 29 22:16:51 2009 -0700
@@ -20,48 +20,52 @@
   var text = options.text;
   var tokenConstructors = options.tokenConstructors;
 
+  var symbolArray = [];
+  var symbolFound;
   var tokens = [];
   var lineNo = 0;
   var charNo = 0;
 
+  for (name in symbols)
+    symbolArray.push({name: name, symbol: symbols[name]});
+
+  function onSymbolFound(name, value) {
+    symbolFound = true;
+    if (name != 'newline' &&
+        name != 'whitespace')
+      tokens.push(new tokenConstructors[name]({name: name,
+                                               value: value,
+                                               charNo: charNo,
+                                               lineNo: lineNo}));
+    if (name == 'newline') {
+      lineNo += 1;
+      charNo = 0;
+    } else
+      charNo += value.length;
+
+    text = text.slice(value.length);
+  }
+
   while (text) {
-    var token = null;
-    for (name in symbols) {
-      var symbol = symbols[name];
+    symbolFound = false;
+    for (var i = 0; i < symbolArray.length && !symbolFound; i++) {
+      var name = symbolArray[i].name;
+      var symbol = symbolArray[i].symbol;
+
       if (typeof(symbol) == 'string') {
-        if (text.indexOf(symbol) == 0) {
-          if (name != 'newline') {
-            token = new tokenConstructors[name]({name: name,
-                                                 charNo: charNo,
-                                                 lineNo: lineNo});
-            charNo += symbol.length;
-          } else {
-            lineNo += 1;
-            charNo = 0;
-          }
-          text = text.slice(symbol.length);
-        }
+        if (text.indexOf(symbol) == 0)
+          onSymbolFound(name, symbol);
       } else if (typeof(symbol) == 'function') {
         // It's a Regular Expression.
         var match = text.match(symbol);
-        if (match) {
-          match = match[0];
-          if (name != 'whitespace')
-            token = new tokenConstructors[name]({name: name,
-                                                 value: match,
-                                                 charNo: charNo,
-                                                 lineNo: lineNo});
-          charNo += match.length;
-          text = text.slice(match.length);
-        }
+        if (match)
+          onSymbolFound(name, match[0]);
       } else
         throw new Error('Unknown symbol type: ' + symbol);
-      if (token)
-        break;
     }
-    if (!token)
+
+    if (!symbolFound)
       throw new Error("I have no idea what this is: " + text);
-    tokens.push(token);
   }
 
   tokens.push(new tokenConstructors.end({name: "end",