mirror of https://github.com/python/cpython.git
Tests for tokenize.py (Ka-Ping Yee)
This commit is contained in:
Guido van Rossum
parent
fefc922cef
commit
0874f7fdaf
|
|
@ -0,0 +1,592 @@
|
|||
test_tokenize
|
||||
1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\012"
|
||||
2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \012'
|
||||
3,0-3,1: NEWLINE '\012'
|
||||
4,0-4,11: COMMENT '# Comments\012'
|
||||
5,0-5,3: STRING '"#"'
|
||||
5,3-5,4: NEWLINE '\012'
|
||||
6,0-6,3: COMMENT "#'\012"
|
||||
7,0-7,3: COMMENT '#"\012'
|
||||
8,0-8,3: COMMENT '#\\\012'
|
||||
9,7-9,9: COMMENT '#\012'
|
||||
10,4-10,10: COMMENT '# abc\012'
|
||||
11,0-12,4: STRING "'''#\012#'''"
|
||||
12,4-12,5: NEWLINE '\012'
|
||||
13,0-13,1: NEWLINE '\012'
|
||||
14,0-14,1: NAME 'x'
|
||||
14,2-14,3: OP '='
|
||||
14,4-14,5: NUMBER '1'
|
||||
14,7-14,8: COMMENT '#'
|
||||
14,8-14,9: NEWLINE '\012'
|
||||
15,0-15,1: NEWLINE '\012'
|
||||
16,0-16,25: COMMENT '# Balancing continuation\012'
|
||||
17,0-17,1: NEWLINE '\012'
|
||||
18,0-18,1: NAME 'a'
|
||||
18,2-18,3: OP '='
|
||||
18,4-18,5: OP '('
|
||||
18,5-18,6: NUMBER '3'
|
||||
18,6-18,7: OP ','
|
||||
18,8-18,9: NUMBER '4'
|
||||
18,9-18,10: OP ','
|
||||
18,10-18,11: NEWLINE '\012'
|
||||
19,2-19,3: NUMBER '5'
|
||||
19,3-19,4: OP ','
|
||||
19,5-19,6: NUMBER '6'
|
||||
19,6-19,7: OP ')'
|
||||
19,7-19,8: NEWLINE '\012'
|
||||
20,0-20,1: NAME 'y'
|
||||
20,2-20,3: OP '='
|
||||
20,4-20,5: OP '['
|
||||
20,5-20,6: NUMBER '3'
|
||||
20,6-20,7: OP ','
|
||||
20,8-20,9: NUMBER '4'
|
||||
20,9-20,10: OP ','
|
||||
20,10-20,11: NEWLINE '\012'
|
||||
21,2-21,3: NUMBER '5'
|
||||
21,3-21,4: OP ']'
|
||||
21,4-21,5: NEWLINE '\012'
|
||||
22,0-22,1: NAME 'z'
|
||||
22,2-22,3: OP '='
|
||||
22,4-22,5: OP '{'
|
||||
22,5-22,8: STRING "'a'"
|
||||
22,8-22,9: OP ':'
|
||||
22,9-22,10: NUMBER '5'
|
||||
22,10-22,11: OP ','
|
||||
22,11-22,12: NEWLINE '\012'
|
||||
23,2-23,5: STRING "'b'"
|
||||
23,5-23,6: OP ':'
|
||||
23,6-23,7: NUMBER '6'
|
||||
23,7-23,8: OP '}'
|
||||
23,8-23,9: NEWLINE '\012'
|
||||
24,0-24,1: NAME 'x'
|
||||
24,2-24,3: OP '='
|
||||
24,4-24,5: OP '('
|
||||
24,5-24,8: NAME 'len'
|
||||
24,8-24,9: OP '('
|
||||
24,9-24,10: OP '`'
|
||||
24,10-24,11: NAME 'y'
|
||||
24,11-24,12: OP '`'
|
||||
24,12-24,13: OP ')'
|
||||
24,14-24,15: OP '+'
|
||||
24,16-24,17: NUMBER '5'
|
||||
24,17-24,18: OP '*'
|
||||
24,18-24,19: NAME 'x'
|
||||
24,20-24,21: OP '-'
|
||||
24,22-24,23: NAME 'a'
|
||||
24,23-24,24: OP '['
|
||||
24,24-24,25: NEWLINE '\012'
|
||||
25,3-25,4: NUMBER '3'
|
||||
25,5-25,6: OP ']'
|
||||
25,6-25,7: NEWLINE '\012'
|
||||
26,3-26,4: OP '-'
|
||||
26,5-26,6: NAME 'x'
|
||||
26,7-26,8: OP '+'
|
||||
26,9-26,12: NAME 'len'
|
||||
26,12-26,13: OP '('
|
||||
26,13-26,14: OP '{'
|
||||
26,14-26,15: NEWLINE '\012'
|
||||
27,3-27,4: OP '}'
|
||||
27,4-27,5: NEWLINE '\012'
|
||||
28,4-28,5: OP ')'
|
||||
28,5-28,6: NEWLINE '\012'
|
||||
29,2-29,3: OP ')'
|
||||
29,3-29,4: NEWLINE '\012'
|
||||
30,0-30,1: NEWLINE '\012'
|
||||
31,0-31,37: COMMENT '# Backslash means line continuation:\012'
|
||||
32,0-32,1: NAME 'x'
|
||||
32,2-32,3: OP '='
|
||||
32,4-32,5: NUMBER '1'
|
||||
33,0-33,1: OP '+'
|
||||
33,2-33,3: NUMBER '1'
|
||||
33,3-33,4: NEWLINE '\012'
|
||||
34,0-34,1: NEWLINE '\012'
|
||||
35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\012'
|
||||
36,0-36,1: NAME 'x'
|
||||
36,2-36,3: OP '='
|
||||
36,4-36,5: NUMBER '0'
|
||||
36,5-36,6: NEWLINE '\012'
|
||||
37,0-37,1: NEWLINE '\012'
|
||||
38,0-38,20: COMMENT '# Ordinary integers\012'
|
||||
39,0-39,4: NUMBER '0xff'
|
||||
39,5-39,7: OP '<>'
|
||||
39,8-39,11: NUMBER '255'
|
||||
39,11-39,12: NEWLINE '\012'
|
||||
40,0-40,4: NUMBER '0377'
|
||||
40,5-40,7: OP '<>'
|
||||
40,8-40,11: NUMBER '255'
|
||||
40,11-40,12: NEWLINE '\012'
|
||||
41,0-41,10: NUMBER '2147483647'
|
||||
41,13-41,15: OP '!='
|
||||
41,16-41,28: NUMBER '017777777777'
|
||||
41,28-41,29: NEWLINE '\012'
|
||||
42,0-42,1: OP '-'
|
||||
42,1-42,11: NUMBER '2147483647'
|
||||
42,11-42,12: OP '-'
|
||||
42,12-42,13: NUMBER '1'
|
||||
42,14-42,16: OP '!='
|
||||
42,17-42,29: NUMBER '020000000000'
|
||||
42,29-42,30: NEWLINE '\012'
|
||||
43,0-43,12: NUMBER '037777777777'
|
||||
43,13-43,15: OP '!='
|
||||
43,16-43,17: OP '-'
|
||||
43,17-43,18: NUMBER '1'
|
||||
43,18-43,19: NEWLINE '\012'
|
||||
44,0-44,10: NUMBER '0xffffffff'
|
||||
44,11-44,13: OP '!='
|
||||
44,14-44,15: OP '-'
|
||||
44,15-44,16: NUMBER '1'
|
||||
44,16-44,17: NEWLINE '\012'
|
||||
45,0-45,1: NEWLINE '\012'
|
||||
46,0-46,16: COMMENT '# Long integers\012'
|
||||
47,0-47,1: NAME 'x'
|
||||
47,2-47,3: OP '='
|
||||
47,4-47,6: NUMBER '0L'
|
||||
47,6-47,7: NEWLINE '\012'
|
||||
48,0-48,1: NAME 'x'
|
||||
48,2-48,3: OP '='
|
||||
48,4-48,6: NUMBER '0l'
|
||||
48,6-48,7: NEWLINE '\012'
|
||||
49,0-49,1: NAME 'x'
|
||||
49,2-49,3: OP '='
|
||||
49,4-49,23: NUMBER '0xffffffffffffffffL'
|
||||
49,23-49,24: NEWLINE '\012'
|
||||
50,0-50,1: NAME 'x'
|
||||
50,2-50,3: OP '='
|
||||
50,4-50,23: NUMBER '0xffffffffffffffffl'
|
||||
50,23-50,24: NEWLINE '\012'
|
||||
51,0-51,1: NAME 'x'
|
||||
51,2-51,3: OP '='
|
||||
51,4-51,23: NUMBER '077777777777777777L'
|
||||
51,23-51,24: NEWLINE '\012'
|
||||
52,0-52,1: NAME 'x'
|
||||
52,2-52,3: OP '='
|
||||
52,4-52,23: NUMBER '077777777777777777l'
|
||||
52,23-52,24: NEWLINE '\012'
|
||||
53,0-53,1: NAME 'x'
|
||||
53,2-53,3: OP '='
|
||||
53,4-53,35: NUMBER '123456789012345678901234567890L'
|
||||
53,35-53,36: NEWLINE '\012'
|
||||
54,0-54,1: NAME 'x'
|
||||
54,2-54,3: OP '='
|
||||
54,4-54,35: NUMBER '123456789012345678901234567890l'
|
||||
54,35-54,36: NEWLINE '\012'
|
||||
55,0-55,1: NEWLINE '\012'
|
||||
56,0-56,25: COMMENT '# Floating-point numbers\012'
|
||||
57,0-57,1: NAME 'x'
|
||||
57,2-57,3: OP '='
|
||||
57,4-57,8: NUMBER '3.14'
|
||||
57,8-57,9: NEWLINE '\012'
|
||||
58,0-58,1: NAME 'x'
|
||||
58,2-58,3: OP '='
|
||||
58,4-58,8: NUMBER '314.'
|
||||
58,8-58,9: NEWLINE '\012'
|
||||
59,0-59,1: NAME 'x'
|
||||
59,2-59,3: OP '='
|
||||
59,4-59,9: NUMBER '0.314'
|
||||
59,9-59,10: NEWLINE '\012'
|
||||
60,0-60,18: COMMENT '# XXX x = 000.314\012'
|
||||
61,0-61,1: NAME 'x'
|
||||
61,2-61,3: OP '='
|
||||
61,4-61,8: NUMBER '.314'
|
||||
61,8-61,9: NEWLINE '\012'
|
||||
62,0-62,1: NAME 'x'
|
||||
62,2-62,3: OP '='
|
||||
62,4-62,8: NUMBER '3e14'
|
||||
62,8-62,9: NEWLINE '\012'
|
||||
63,0-63,1: NAME 'x'
|
||||
63,2-63,3: OP '='
|
||||
63,4-63,8: NUMBER '3E14'
|
||||
63,8-63,9: NEWLINE '\012'
|
||||
64,0-64,1: NAME 'x'
|
||||
64,2-64,3: OP '='
|
||||
64,4-64,9: NUMBER '3e-14'
|
||||
64,9-64,10: NEWLINE '\012'
|
||||
65,0-65,1: NAME 'x'
|
||||
65,2-65,3: OP '='
|
||||
65,4-65,9: NUMBER '3e+14'
|
||||
65,9-65,10: NEWLINE '\012'
|
||||
66,0-66,1: NAME 'x'
|
||||
66,2-66,3: OP '='
|
||||
66,4-66,9: NUMBER '3.e14'
|
||||
66,9-66,10: NEWLINE '\012'
|
||||
67,0-67,1: NAME 'x'
|
||||
67,2-67,3: OP '='
|
||||
67,4-67,9: NUMBER '.3e14'
|
||||
67,9-67,10: NEWLINE '\012'
|
||||
68,0-68,1: NAME 'x'
|
||||
68,2-68,3: OP '='
|
||||
68,4-68,9: NUMBER '3.1e4'
|
||||
68,9-68,10: NEWLINE '\012'
|
||||
69,0-69,1: NEWLINE '\012'
|
||||
70,0-70,18: COMMENT '# String literals\012'
|
||||
71,0-71,1: NAME 'x'
|
||||
71,2-71,3: OP '='
|
||||
71,4-71,6: STRING "''"
|
||||
71,6-71,7: OP ';'
|
||||
71,8-71,9: NAME 'y'
|
||||
71,10-71,11: OP '='
|
||||
71,12-71,14: STRING '""'
|
||||
71,14-71,15: OP ';'
|
||||
71,15-71,16: NEWLINE '\012'
|
||||
72,0-72,1: NAME 'x'
|
||||
72,2-72,3: OP '='
|
||||
72,4-72,8: STRING "'\\''"
|
||||
72,8-72,9: OP ';'
|
||||
72,10-72,11: NAME 'y'
|
||||
72,12-72,13: OP '='
|
||||
72,14-72,17: STRING '"\'"'
|
||||
72,17-72,18: OP ';'
|
||||
72,18-72,19: NEWLINE '\012'
|
||||
73,0-73,1: NAME 'x'
|
||||
73,2-73,3: OP '='
|
||||
73,4-73,7: STRING '\'"\''
|
||||
73,7-73,8: OP ';'
|
||||
73,9-73,10: NAME 'y'
|
||||
73,11-73,12: OP '='
|
||||
73,13-73,17: STRING '"\\""'
|
||||
73,17-73,18: OP ';'
|
||||
73,18-73,19: NEWLINE '\012'
|
||||
74,0-74,1: NAME 'x'
|
||||
74,2-74,3: OP '='
|
||||
74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"'
|
||||
74,32-74,33: NEWLINE '\012'
|
||||
75,0-75,1: NAME 'y'
|
||||
75,2-75,3: OP '='
|
||||
75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\''
|
||||
75,31-75,32: NEWLINE '\012'
|
||||
76,0-76,1: NAME 'x'
|
||||
76,2-76,3: OP '='
|
||||
76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"'
|
||||
76,32-76,33: NEWLINE '\012'
|
||||
77,0-77,1: NAME 'y'
|
||||
77,2-77,3: OP '='
|
||||
77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\''
|
||||
77,31-77,32: NEWLINE '\012'
|
||||
78,0-78,1: NAME 'x'
|
||||
78,2-78,3: OP '='
|
||||
78,4-83,3: STRING '"""\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012"""'
|
||||
83,3-83,4: NEWLINE '\012'
|
||||
84,0-84,1: NAME 'y'
|
||||
84,2-84,3: OP '='
|
||||
84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
|
||||
84,63-84,64: NEWLINE '\012'
|
||||
85,0-85,1: NAME 'y'
|
||||
85,2-85,3: OP '='
|
||||
85,4-90,3: STRING '\'\'\'\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012\'\'\''
|
||||
90,3-90,4: OP ';'
|
||||
90,4-90,5: NEWLINE '\012'
|
||||
91,0-91,1: NAME 'y'
|
||||
91,2-91,3: OP '='
|
||||
91,4-96,1: STRING '"\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \'lazy\' dog.\\n\\\012"'
|
||||
96,1-96,2: OP ';'
|
||||
96,2-96,3: NEWLINE '\012'
|
||||
97,0-97,1: NAME 'y'
|
||||
97,2-97,3: OP '='
|
||||
97,4-102,1: STRING '\'\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \\\'lazy\\\' dog.\\n\\\012\''
|
||||
102,1-102,2: OP ';'
|
||||
102,2-102,3: NEWLINE '\012'
|
||||
103,0-103,1: NAME 'x'
|
||||
103,2-103,3: OP '='
|
||||
103,4-103,9: STRING "r'\\\\'"
|
||||
103,10-103,11: OP '+'
|
||||
103,12-103,17: STRING "R'\\\\'"
|
||||
103,17-103,18: NEWLINE '\012'
|
||||
104,0-104,1: NAME 'x'
|
||||
104,2-104,3: OP '='
|
||||
104,4-104,9: STRING "r'\\''"
|
||||
104,10-104,11: OP '+'
|
||||
104,12-104,14: STRING "''"
|
||||
104,14-104,15: NEWLINE '\012'
|
||||
105,0-105,1: NAME 'y'
|
||||
105,2-105,3: OP '='
|
||||
105,4-107,6: STRING "r'''\012foo bar \\\\\012baz'''"
|
||||
107,7-107,8: OP '+'
|
||||
107,9-108,6: STRING "R'''\012foo'''"
|
||||
108,6-108,7: NEWLINE '\012'
|
||||
109,0-109,1: NAME 'y'
|
||||
109,2-109,3: OP '='
|
||||
109,4-111,3: STRING 'r"""foo\012bar \\\\ baz\012"""'
|
||||
111,4-111,5: OP '+'
|
||||
111,6-112,3: STRING "R'''spam\012'''"
|
||||
112,3-112,4: NEWLINE '\012'
|
||||
113,0-113,1: NEWLINE '\012'
|
||||
114,0-114,14: COMMENT '# Indentation\012'
|
||||
115,0-115,2: NAME 'if'
|
||||
115,3-115,4: NUMBER '1'
|
||||
115,4-115,5: OP ':'
|
||||
115,5-115,6: NEWLINE '\012'
|
||||
116,0-116,4: INDENT ' '
|
||||
116,4-116,5: NAME 'x'
|
||||
116,6-116,7: OP '='
|
||||
116,8-116,9: NUMBER '2'
|
||||
116,9-116,10: NEWLINE '\012'
|
||||
117,0-117,0: DEDENT ''
|
||||
117,0-117,2: NAME 'if'
|
||||
117,3-117,4: NUMBER '1'
|
||||
117,4-117,5: OP ':'
|
||||
117,5-117,6: NEWLINE '\012'
|
||||
118,0-118,8: INDENT ' '
|
||||
118,8-118,9: NAME 'x'
|
||||
118,10-118,11: OP '='
|
||||
118,12-118,13: NUMBER '2'
|
||||
118,13-118,14: NEWLINE '\012'
|
||||
119,0-119,0: DEDENT ''
|
||||
119,0-119,2: NAME 'if'
|
||||
119,3-119,4: NUMBER '1'
|
||||
119,4-119,5: OP ':'
|
||||
119,5-119,6: NEWLINE '\012'
|
||||
120,0-120,4: INDENT ' '
|
||||
120,4-120,9: NAME 'while'
|
||||
120,10-120,11: NUMBER '0'
|
||||
120,11-120,12: OP ':'
|
||||
120,12-120,13: NEWLINE '\012'
|
||||
121,0-121,5: INDENT ' '
|
||||
121,5-121,7: NAME 'if'
|
||||
121,8-121,9: NUMBER '0'
|
||||
121,9-121,10: OP ':'
|
||||
121,10-121,11: NEWLINE '\012'
|
||||
122,0-122,11: INDENT ' '
|
||||
122,11-122,12: NAME 'x'
|
||||
122,13-122,14: OP '='
|
||||
122,15-122,16: NUMBER '2'
|
||||
122,16-122,17: NEWLINE '\012'
|
||||
123,5-123,5: DEDENT ''
|
||||
123,5-123,6: NAME 'x'
|
||||
123,7-123,8: OP '='
|
||||
123,9-123,10: NUMBER '2'
|
||||
123,10-123,11: NEWLINE '\012'
|
||||
124,0-124,0: DEDENT ''
|
||||
124,0-124,0: DEDENT ''
|
||||
124,0-124,2: NAME 'if'
|
||||
124,3-124,4: NUMBER '0'
|
||||
124,4-124,5: OP ':'
|
||||
124,5-124,6: NEWLINE '\012'
|
||||
125,0-125,2: INDENT ' '
|
||||
125,2-125,4: NAME 'if'
|
||||
125,5-125,6: NUMBER '2'
|
||||
125,6-125,7: OP ':'
|
||||
125,7-125,8: NEWLINE '\012'
|
||||
126,0-126,3: INDENT ' '
|
||||
126,3-126,8: NAME 'while'
|
||||
126,9-126,10: NUMBER '0'
|
||||
126,10-126,11: OP ':'
|
||||
126,11-126,12: NEWLINE '\012'
|
||||
127,0-127,8: INDENT ' '
|
||||
127,8-127,10: NAME 'if'
|
||||
127,11-127,12: NUMBER '1'
|
||||
127,12-127,13: OP ':'
|
||||
127,13-127,14: NEWLINE '\012'
|
||||
128,0-128,10: INDENT ' '
|
||||
128,10-128,11: NAME 'x'
|
||||
128,12-128,13: OP '='
|
||||
128,14-128,15: NUMBER '2'
|
||||
128,15-128,16: NEWLINE '\012'
|
||||
129,0-129,1: NEWLINE '\012'
|
||||
130,0-130,12: COMMENT '# Operators\012'
|
||||
131,0-131,1: NEWLINE '\012'
|
||||
132,0-132,0: DEDENT ''
|
||||
132,0-132,0: DEDENT ''
|
||||
132,0-132,0: DEDENT ''
|
||||
132,0-132,0: DEDENT ''
|
||||
132,0-132,3: NAME 'def'
|
||||
132,4-132,7: NAME 'd22'
|
||||
132,7-132,8: OP '('
|
||||
132,8-132,9: NAME 'a'
|
||||
132,9-132,10: OP ','
|
||||
132,11-132,12: NAME 'b'
|
||||
132,12-132,13: OP ','
|
||||
132,14-132,15: NAME 'c'
|
||||
132,15-132,16: OP '='
|
||||
132,16-132,17: NUMBER '1'
|
||||
132,17-132,18: OP ','
|
||||
132,19-132,20: NAME 'd'
|
||||
132,20-132,21: OP '='
|
||||
132,21-132,22: NUMBER '2'
|
||||
132,22-132,23: OP ')'
|
||||
132,23-132,24: OP ':'
|
||||
132,25-132,29: NAME 'pass'
|
||||
132,29-132,30: NEWLINE '\012'
|
||||
133,0-133,3: NAME 'def'
|
||||
133,4-133,8: NAME 'd01v'
|
||||
133,8-133,9: OP '('
|
||||
133,9-133,10: NAME 'a'
|
||||
133,10-133,11: OP '='
|
||||
133,11-133,12: NUMBER '1'
|
||||
133,12-133,13: OP ','
|
||||
133,14-133,15: OP '*'
|
||||
133,15-133,19: NAME 'rest'
|
||||
133,19-133,20: OP ','
|
||||
133,21-133,23: OP '**'
|
||||
133,23-133,27: NAME 'rest'
|
||||
133,27-133,28: OP ')'
|
||||
133,28-133,29: OP ':'
|
||||
133,30-133,34: NAME 'pass'
|
||||
133,34-133,35: NEWLINE '\012'
|
||||
134,0-134,1: NEWLINE '\012'
|
||||
135,0-135,1: OP '('
|
||||
135,1-135,2: NAME 'x'
|
||||
135,2-135,3: OP ','
|
||||
135,4-135,5: NAME 'y'
|
||||
135,5-135,6: OP ')'
|
||||
135,7-135,9: OP '<>'
|
||||
135,10-135,11: OP '('
|
||||
135,11-135,12: OP '{'
|
||||
135,12-135,15: STRING "'a'"
|
||||
135,15-135,16: OP ':'
|
||||
135,16-135,17: NUMBER '1'
|
||||
135,17-135,18: OP '}'
|
||||
135,18-135,19: OP ','
|
||||
135,20-135,21: OP '{'
|
||||
135,21-135,24: STRING "'b'"
|
||||
135,24-135,25: OP ':'
|
||||
135,25-135,26: NUMBER '2'
|
||||
135,26-135,27: OP '}'
|
||||
135,27-135,28: OP ')'
|
||||
135,28-135,29: NEWLINE '\012'
|
||||
136,0-136,1: NEWLINE '\012'
|
||||
137,0-137,13: COMMENT '# comparison\012'
|
||||
138,0-138,2: NAME 'if'
|
||||
138,3-138,4: NUMBER '1'
|
||||
138,5-138,6: OP '<'
|
||||
138,7-138,8: NUMBER '1'
|
||||
138,9-138,10: OP '>'
|
||||
138,11-138,12: NUMBER '1'
|
||||
138,13-138,15: OP '=='
|
||||
138,16-138,17: NUMBER '1'
|
||||
138,18-138,20: OP '>='
|
||||
138,21-138,22: NUMBER '1'
|
||||
138,23-138,25: OP '<='
|
||||
138,26-138,27: NUMBER '1'
|
||||
138,28-138,30: OP '<>'
|
||||
138,31-138,32: NUMBER '1'
|
||||
138,33-138,35: OP '!='
|
||||
138,36-138,37: NUMBER '1'
|
||||
138,38-138,40: NAME 'in'
|
||||
138,41-138,42: NUMBER '1'
|
||||
138,43-138,46: NAME 'not'
|
||||
138,47-138,49: NAME 'in'
|
||||
138,50-138,51: NUMBER '1'
|
||||
138,52-138,54: NAME 'is'
|
||||
138,55-138,56: NUMBER '1'
|
||||
138,57-138,59: NAME 'is'
|
||||
138,60-138,63: NAME 'not'
|
||||
138,64-138,65: NUMBER '1'
|
||||
138,65-138,66: OP ':'
|
||||
138,67-138,71: NAME 'pass'
|
||||
138,71-138,72: NEWLINE '\012'
|
||||
139,0-139,1: NEWLINE '\012'
|
||||
140,0-140,9: COMMENT '# binary\012'
|
||||
141,0-141,1: NAME 'x'
|
||||
141,2-141,3: OP '='
|
||||
141,4-141,5: NUMBER '1'
|
||||
141,6-141,7: OP '&'
|
||||
141,8-141,9: NUMBER '1'
|
||||
141,9-141,10: NEWLINE '\012'
|
||||
142,0-142,1: NAME 'x'
|
||||
142,2-142,3: OP '='
|
||||
142,4-142,5: NUMBER '1'
|
||||
142,6-142,7: OP '^'
|
||||
142,8-142,9: NUMBER '1'
|
||||
142,9-142,10: NEWLINE '\012'
|
||||
143,0-143,1: NAME 'x'
|
||||
143,2-143,3: OP '='
|
||||
143,4-143,5: NUMBER '1'
|
||||
143,6-143,7: OP '|'
|
||||
143,8-143,9: NUMBER '1'
|
||||
143,9-143,10: NEWLINE '\012'
|
||||
144,0-144,1: NEWLINE '\012'
|
||||
145,0-145,8: COMMENT '# shift\012'
|
||||
146,0-146,1: NAME 'x'
|
||||
146,2-146,3: OP '='
|
||||
146,4-146,5: NUMBER '1'
|
||||
146,6-146,8: OP '<<'
|
||||
146,9-146,10: NUMBER '1'
|
||||
146,11-146,13: OP '>>'
|
||||
146,14-146,15: NUMBER '1'
|
||||
146,15-146,16: NEWLINE '\012'
|
||||
147,0-147,1: NEWLINE '\012'
|
||||
148,0-148,11: COMMENT '# additive\012'
|
||||
149,0-149,1: NAME 'x'
|
||||
149,2-149,3: OP '='
|
||||
149,4-149,5: NUMBER '1'
|
||||
149,6-149,7: OP '-'
|
||||
149,8-149,9: NUMBER '1'
|
||||
149,10-149,11: OP '+'
|
||||
149,12-149,13: NUMBER '1'
|
||||
149,14-149,15: OP '-'
|
||||
149,16-149,17: NUMBER '1'
|
||||
149,18-149,19: OP '+'
|
||||
149,20-149,21: NUMBER '1'
|
||||
149,21-149,22: NEWLINE '\012'
|
||||
150,0-150,1: NEWLINE '\012'
|
||||
151,0-151,17: COMMENT '# multiplicative\012'
|
||||
152,0-152,1: NAME 'x'
|
||||
152,2-152,3: OP '='
|
||||
152,4-152,5: NUMBER '1'
|
||||
152,6-152,7: OP '/'
|
||||
152,8-152,9: NUMBER '1'
|
||||
152,10-152,11: OP '*'
|
||||
152,12-152,13: NUMBER '1'
|
||||
152,14-152,15: OP '%'
|
||||
152,16-152,17: NUMBER '1'
|
||||
152,17-152,18: NEWLINE '\012'
|
||||
153,0-153,1: NEWLINE '\012'
|
||||
154,0-154,8: COMMENT '# unary\012'
|
||||
155,0-155,1: NAME 'x'
|
||||
155,2-155,3: OP '='
|
||||
155,4-155,5: OP '~'
|
||||
155,5-155,6: NUMBER '1'
|
||||
155,7-155,8: OP '^'
|
||||
155,9-155,10: NUMBER '1'
|
||||
155,11-155,12: OP '&'
|
||||
155,13-155,14: NUMBER '1'
|
||||
155,15-155,16: OP '|'
|
||||
155,17-155,18: NUMBER '1'
|
||||
155,19-155,20: OP '&'
|
||||
155,21-155,22: NUMBER '1'
|
||||
155,23-155,24: OP '^'
|
||||
155,25-155,26: OP '-'
|
||||
155,26-155,27: NUMBER '1'
|
||||
155,27-155,28: NEWLINE '\012'
|
||||
156,0-156,1: NAME 'x'
|
||||
156,2-156,3: OP '='
|
||||
156,4-156,5: OP '-'
|
||||
156,5-156,6: NUMBER '1'
|
||||
156,6-156,7: OP '*'
|
||||
156,7-156,8: NUMBER '1'
|
||||
156,8-156,9: OP '/'
|
||||
156,9-156,10: NUMBER '1'
|
||||
156,11-156,12: OP '+'
|
||||
156,13-156,14: NUMBER '1'
|
||||
156,14-156,15: OP '*'
|
||||
156,15-156,16: NUMBER '1'
|
||||
156,17-156,18: OP '-'
|
||||
156,19-156,20: OP '-'
|
||||
156,20-156,21: OP '-'
|
||||
156,21-156,22: OP '-'
|
||||
156,22-156,23: NUMBER '1'
|
||||
156,23-156,24: OP '*'
|
||||
156,24-156,25: NUMBER '1'
|
||||
156,25-156,26: NEWLINE '\012'
|
||||
157,0-157,1: NEWLINE '\012'
|
||||
158,0-158,11: COMMENT '# selector\012'
|
||||
159,0-159,6: NAME 'import'
|
||||
159,7-159,10: NAME 'sys'
|
||||
159,10-159,11: OP ','
|
||||
159,12-159,16: NAME 'time'
|
||||
159,16-159,17: NEWLINE '\012'
|
||||
160,0-160,1: NAME 'x'
|
||||
160,2-160,3: OP '='
|
||||
160,4-160,7: NAME 'sys'
|
||||
160,7-160,8: OP '.'
|
||||
160,8-160,15: NAME 'modules'
|
||||
160,15-160,16: OP '['
|
||||
160,16-160,22: STRING "'time'"
|
||||
160,22-160,23: OP ']'
|
||||
160,23-160,24: OP '.'
|
||||
160,24-160,28: NAME 'time'
|
||||
160,28-160,29: OP '('
|
||||
160,29-160,30: OP ')'
|
||||
160,30-160,31: NEWLINE '\012'
|
||||
161,0-161,1: NEWLINE '\012'
|
||||
162,0-162,0: ENDMARKER ''
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from test_support import verbose
|
||||
import tokenize, os, sys
|
||||
|
||||
def findfile(file):
|
||||
if os.path.isabs(file): return file
|
||||
path = sys.path
|
||||
try:
|
||||
path = [os.path.dirname(__file__)] + path
|
||||
except NameError:
|
||||
pass
|
||||
for dn in path:
|
||||
fn = os.path.join(dn, file)
|
||||
if os.path.exists(fn): return fn
|
||||
return file
|
||||
|
||||
if verbose:
|
||||
print 'starting...'
|
||||
file = open(findfile('tokenize_tests.py'))
|
||||
tokenize.tokenize(file.readline)
|
||||
if verbose:
|
||||
print 'finished'
|
||||
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
# Tests for the 'tokenize' module.
|
||||
# Large bits stolen from test_grammar.py.
|
||||
|
||||
# Comments
|
||||
"#"
|
||||
#'
|
||||
#"
|
||||
#\
|
||||
#
|
||||
# abc
|
||||
'''#
|
||||
#'''
|
||||
|
||||
x = 1 #
|
||||
|
||||
# Balancing continuation
|
||||
|
||||
a = (3, 4,
|
||||
5, 6)
|
||||
y = [3, 4,
|
||||
5]
|
||||
z = {'a':5,
|
||||
'b':6}
|
||||
x = (len(`y`) + 5*x - a[
|
||||
3 ]
|
||||
- x + len({
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Backslash means line continuation:
|
||||
x = 1 \
|
||||
+ 1
|
||||
|
||||
# Backslash does not means continuation in comments :\
|
||||
x = 0
|
||||
|
||||
# Ordinary integers
|
||||
0xff <> 255
|
||||
0377 <> 255
|
||||
2147483647 != 017777777777
|
||||
-2147483647-1 != 020000000000
|
||||
037777777777 != -1
|
||||
0xffffffff != -1
|
||||
|
||||
# Long integers
|
||||
x = 0L
|
||||
x = 0l
|
||||
x = 0xffffffffffffffffL
|
||||
x = 0xffffffffffffffffl
|
||||
x = 077777777777777777L
|
||||
x = 077777777777777777l
|
||||
x = 123456789012345678901234567890L
|
||||
x = 123456789012345678901234567890l
|
||||
|
||||
# Floating-point numbers
|
||||
x = 3.14
|
||||
x = 314.
|
||||
x = 0.314
|
||||
# XXX x = 000.314
|
||||
x = .314
|
||||
x = 3e14
|
||||
x = 3E14
|
||||
x = 3e-14
|
||||
x = 3e+14
|
||||
x = 3.e14
|
||||
x = .3e14
|
||||
x = 3.1e4
|
||||
|
||||
# String literals
|
||||
x = ''; y = "";
|
||||
x = '\''; y = "'";
|
||||
x = '"'; y = "\"";
|
||||
x = "doesn't \"shrink\" does it"
|
||||
y = 'doesn\'t "shrink" does it'
|
||||
x = "does \"shrink\" doesn't it"
|
||||
y = 'does "shrink" doesn\'t it'
|
||||
x = """
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
"""
|
||||
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
|
||||
y = '''
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
''';
|
||||
y = "\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the 'lazy' dog.\n\
|
||||
";
|
||||
y = '\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the \'lazy\' dog.\n\
|
||||
';
|
||||
x = r'\\' + R'\\'
|
||||
x = r'\'' + ''
|
||||
y = r'''
|
||||
foo bar \\
|
||||
baz''' + R'''
|
||||
foo'''
|
||||
y = r"""foo
|
||||
bar \\ baz
|
||||
""" + R'''spam
|
||||
'''
|
||||
|
||||
# Indentation
|
||||
if 1:
|
||||
x = 2
|
||||
if 1:
|
||||
x = 2
|
||||
if 1:
|
||||
while 0:
|
||||
if 0:
|
||||
x = 2
|
||||
x = 2
|
||||
if 0:
|
||||
if 2:
|
||||
while 0:
|
||||
if 1:
|
||||
x = 2
|
||||
|
||||
# Operators
|
||||
|
||||
def d22(a, b, c=1, d=2): pass
|
||||
def d01v(a=1, *rest, **rest): pass
|
||||
|
||||
(x, y) <> ({'a':1}, {'b':2})
|
||||
|
||||
# comparison
|
||||
if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
|
||||
|
||||
# binary
|
||||
x = 1 & 1
|
||||
x = 1 ^ 1
|
||||
x = 1 | 1
|
||||
|
||||
# shift
|
||||
x = 1 << 1 >> 1
|
||||
|
||||
# additive
|
||||
x = 1 - 1 + 1 - 1 + 1
|
||||
|
||||
# multiplicative
|
||||
x = 1 / 1 * 1 % 1
|
||||
|
||||
# unary
|
||||
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
|
||||
x = -1*1/1 + 1*1 - ---1*1
|
||||
|
||||
# selector
|
||||
import sys, time
|
||||
x = sys.modules['time'].time()
|
||||
|
||||
Loading…
Reference in New Issue