Mercurial > js-scanner
changeset 5:815520476fbb default tip
accept '/' as a literal
| author | Atul Varma <avarma@mozilla.com> |
|---|---|
| date | Thu, 22 Apr 2010 20:03:31 -0700 |
| parents | 30c1f55eff96 |
| children | |
| files | jsscan.py |
| diffstat | 1 files changed, 20 insertions(+), 21 deletions(-) [+] |
line wrap: on
line diff
--- a/jsscan.py Thu Apr 22 17:31:32 2010 -0700 +++ b/jsscan.py Thu Apr 22 20:03:31 2010 -0700 @@ -13,7 +13,7 @@ ) LITERALS = ('(){}[];.,:?' - '!=-+*&|<>') + '!=-+*&|<>/') def __init__(self, text): self.text = text @@ -29,30 +29,29 @@ while self.pos < len(self.text): found = None - if self.text[self.pos] in self.LITERALS: + for tokname, tokre in self.TOKENS.items(): + match = tokre.match(self.text, self.pos) + if match: + tokvalue = match.group(0) + found = (tokname, tokvalue, + (self.line, self.char)) + self.pos += len(tokvalue) + if tokre.flags & re.MULTILINE: + newlines = tokvalue.count('\n') + if newlines: + self.line += newlines + self.char = ((len(tokvalue) - 1) - + tokvalue.rfind('\n')) + else: + self.char += len(tokvalue) + else: + self.char += len(tokvalue) + continue + if found is None and self.text[self.pos] in self.LITERALS: found = ('literal', self.text[self.pos], (self.line, self.char)) self.pos += 1 self.char += 1 - else: - for tokname, tokre in self.TOKENS.items(): - match = tokre.match(self.text, self.pos) - if match: - tokvalue = match.group(0) - found = (tokname, tokvalue, - (self.line, self.char)) - self.pos += len(tokvalue) - if tokre.flags & re.MULTILINE: - newlines = tokvalue.count('\n') - if newlines: - self.line += newlines - self.char = ((len(tokvalue) - 1) - - tokvalue.rfind('\n')) - else: - self.char += len(tokvalue) - else: - self.char += len(tokvalue) - continue if found is not None: if found[0] not in ignore: yield found