Mercurial > js-scanner
diff jsscan.py @ 2:f82ff2c61c06
added ignore kwarg
author | Atul Varma <avarma@mozilla.com> |
---|---|
date | Thu, 22 Apr 2010 13:18:09 -0700 |
parents | daa1c6d996f3 |
children | ce894f57b30c |
line wrap: on
line diff
--- a/jsscan.py Thu Apr 22 13:12:12 2010 -0700 +++ b/jsscan.py Thu Apr 22 13:18:09 2010 -0700 @@ -21,7 +21,12 @@ self.line = 1 self.char = 0 - def tokenize(self): + def tokenize(self, ignore=None): + if ignore is None: + ignore = [] + elif isinstance(ignore, basestring): + ignore = [ignore] + while self.pos < len(self.text): found = None if self.text[self.pos] in self.LITERALS: @@ -49,7 +54,8 @@ self.char += len(tokvalue) continue if found is not None: - yield found + if found[0] not in ignore: + yield found else: raise TokenizationError('unrecognized token %s' % repr(self.text[self.pos]),