|
0
|
1 """
|
|
|
2 C-style comments:
|
|
|
3
|
|
|
4 >>> tokenize('/* hello */')
|
|
|
5 ('c_comment', '/* hello */', (1, 0))
|
|
|
6
|
|
|
7 C++-style comments:
|
|
|
8
|
|
|
9 >>> tokenize('// hello')
|
|
|
10 ('cpp_comment', '// hello', (1, 0))
|
|
|
11
|
|
|
12 Variable definitions:
|
|
|
13
|
|
|
14 >>> tokenize(' var k = 1;')
|
|
|
15 ('name', 'var', (1, 2))
|
|
|
16 ('name', 'k', (1, 6))
|
|
|
17 ('whitespace', ' ', (1, 7))
|
|
|
18 ('literal', '=', (1, 8))
|
|
|
19 ('whitespace', ' ', (1, 9))
|
|
|
20 ('digits', '1', (1, 10))
|
|
|
21 ('literal', ';', (1, 11))
|
|
|
22
|
|
|
23 Escaped double-quoted strings:
|
|
|
24
|
|
|
25 >>> tokenize(r'"i say \\"tomato\\""')
|
|
|
26 ('string', '"i say \\\\"tomato\\\\""', (1, 0))
|
|
|
27
|
|
|
28 Unterminated double-quoted strings:
|
|
|
29
|
|
|
30 >>> tokenize(r'"i say \\"tomato\\"')
|
|
|
31 Traceback (most recent call last):
|
|
|
32 ...
|
|
|
33 TokenizationError: unrecognized token '"' @ line 1, char 0
|
|
|
34 """
|
|
|
35
|
|
|
36 import doctest
|
|
|
37
|
|
|
38 from jsscan import *
|
|
|
39
|
|
|
40 def tokenize(string):
|
|
|
41 for token in Tokenizer(string).tokenize():
|
|
|
42 print token
|
|
|
43
|
|
|
44 if __name__ == '__main__':
|
|
|
45 doctest.testmod(verbose=True)
|
