Mercurial > js-scanner
diff test_jsscan.py @ 2:f82ff2c61c06
added ignore kwarg
author | Atul Varma <avarma@mozilla.com> |
---|---|
date | Thu, 22 Apr 2010 13:18:09 -0700 |
parents | daa1c6d996f3 |
children | 30c1f55eff96 |
line wrap: on
line diff
--- a/test_jsscan.py Thu Apr 22 13:12:12 2010 -0700 +++ b/test_jsscan.py Thu Apr 22 13:18:09 2010 -0700 @@ -20,6 +20,11 @@ ('digits', '1', (1, 10)) ('literal', ';', (1, 11)) +Filtering: + + >>> tokenize(' k', ignore='whitespace') + ('name', 'k', (1, 2)) + Escaped double-quoted strings: >>> tokenize(r'"i say \\"tomato\\""') @@ -37,8 +42,8 @@ from jsscan import * -def tokenize(string): - for token in Tokenizer(string).tokenize(): +def tokenize(string, ignore=None): + for token in Tokenizer(string).tokenize(ignore=ignore): print token if __name__ == '__main__':