Mercurial > js-scanner

comparison jsscan.py @ 2:f82ff2c61c06

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added ignore kwarg
author Atul Varma <avarma@mozilla.com>
date Thu, 22 Apr 2010 13:18:09 -0700
parents daa1c6d996f3
children ce894f57b30c
comparison
equal deleted inserted replaced
1:22781f88a857 2:f82ff2c61c06
19 self.text = text 19 self.text = text
20 self.pos = 0 20 self.pos = 0
21 self.line = 1 21 self.line = 1
22 self.char = 0 22 self.char = 0
23 23
24 def tokenize(self): 24 def tokenize(self, ignore=None):
25 if ignore is None:
26 ignore = []
27 elif isinstance(ignore, basestring):
28 ignore = [ignore]
29
25 while self.pos < len(self.text): 30 while self.pos < len(self.text):
26 found = None 31 found = None
27 if self.text[self.pos] in self.LITERALS: 32 if self.text[self.pos] in self.LITERALS:
28 found = ('literal', self.text[self.pos], 33 found = ('literal', self.text[self.pos],
29 (self.line, self.char)) 34 (self.line, self.char))
47 self.char += len(tokvalue) 52 self.char += len(tokvalue)
48 else: 53 else:
49 self.char += len(tokvalue) 54 self.char += len(tokvalue)
50 continue 55 continue
51 if found is not None: 56 if found is not None:
52 yield found 57 if found[0] not in ignore:
58 yield found
53 else: 59 else:
54 raise TokenizationError('unrecognized token %s' % 60 raise TokenizationError('unrecognized token %s' %
55 repr(self.text[self.pos]), 61 repr(self.text[self.pos]),
56 self.line, 62 self.line,
57 self.char) 63 self.char)