Mercurial > jsparser
changeset 2:032f64260794
refactored tokenizer
author | Atul Varma <varmaa@toolness.com> |
---|---|
date | Fri, 29 May 2009 22:16:51 -0700 |
parents | 0f1ff7d5a524 |
children | f174153281a9 |
files | jsparser.js |
diffstat | 1 files changed, 33 insertions(+), 29 deletions(-) [+] |
line wrap: on
line diff
--- a/jsparser.js Fri May 29 18:13:55 2009 -0700 +++ b/jsparser.js Fri May 29 22:16:51 2009 -0700 @@ -20,48 +20,52 @@ var text = options.text; var tokenConstructors = options.tokenConstructors; + var symbolArray = []; + var symbolFound; var tokens = []; var lineNo = 0; var charNo = 0; + for (name in symbols) + symbolArray.push({name: name, symbol: symbols[name]}); + + function onSymbolFound(name, value) { + symbolFound = true; + if (name != 'newline' && + name != 'whitespace') + tokens.push(new tokenConstructors[name]({name: name, + value: value, + charNo: charNo, + lineNo: lineNo})); + if (name == 'newline') { + lineNo += 1; + charNo = 0; + } else + charNo += value.length; + + text = text.slice(value.length); + } + while (text) { - var token = null; - for (name in symbols) { - var symbol = symbols[name]; + symbolFound = false; + for (var i = 0; i < symbolArray.length && !symbolFound; i++) { + var name = symbolArray[i].name; + var symbol = symbolArray[i].symbol; + if (typeof(symbol) == 'string') { - if (text.indexOf(symbol) == 0) { - if (name != 'newline') { - token = new tokenConstructors[name]({name: name, - charNo: charNo, - lineNo: lineNo}); - charNo += symbol.length; - } else { - lineNo += 1; - charNo = 0; - } - text = text.slice(symbol.length); - } + if (text.indexOf(symbol) == 0) + onSymbolFound(name, symbol); } else if (typeof(symbol) == 'function') { // It's a Regular Expression. var match = text.match(symbol); - if (match) { - match = match[0]; - if (name != 'whitespace') - token = new tokenConstructors[name]({name: name, - value: match, - charNo: charNo, - lineNo: lineNo}); - charNo += match.length; - text = text.slice(match.length); - } + if (match) + onSymbolFound(name, match[0]); } else throw new Error('Unknown symbol type: ' + symbol); - if (token) - break; } - if (!token) + + if (!symbolFound) throw new Error("I have no idea what this is: " + text); - tokens.push(token); } tokens.push(new tokenConstructors.end({name: "end",