diff jsscan.py @ 2:f82ff2c61c06

added ignore kwarg
author Atul Varma <avarma@mozilla.com>
date Thu, 22 Apr 2010 13:18:09 -0700
parents daa1c6d996f3
children ce894f57b30c
line wrap: on
line diff
--- a/jsscan.py	Thu Apr 22 13:12:12 2010 -0700
+++ b/jsscan.py	Thu Apr 22 13:18:09 2010 -0700
@@ -21,7 +21,12 @@
         self.line = 1
         self.char = 0
 
-    def tokenize(self):
+    def tokenize(self, ignore=None):
+        if ignore is None:
+            ignore = []
+        elif isinstance(ignore, basestring):
+            ignore = [ignore]
+
         while self.pos < len(self.text):
             found = None
             if self.text[self.pos] in self.LITERALS:
@@ -49,7 +54,8 @@
                             self.char += len(tokvalue)
                         continue
             if found is not None:
-                yield found
+                if found[0] not in ignore:
+                    yield found
             else:
                 raise TokenizationError('unrecognized token %s' %
                                         repr(self.text[self.pos]),