Skip to content

Commit a4e0f20

Browse files
pablogsalpablogsal
committed
gh-153569: move tokenizer state to source offsets
1 parent adf62b2 commit a4e0f20

41 files changed

Lines changed: 3684 additions & 2449 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Lib/test/test_codeop.py‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22
Test cases for codeop.py
33
Nick Mathewson
44
"""
5+
import builtins
56
import unittest
67
import warnings
78
from test.support import subTests, warnings_helper
89
from textwrap import dedent
910
import functools
1011

1112
from codeop import compile_command, CommandCompiler, Compile
12-
from codeop import PyCF_DONT_IMPLY_DEDENT, PyCF_ONLY_AST
13+
from codeop import (
14+
PyCF_ALLOW_INCOMPLETE_INPUT,
15+
PyCF_DONT_IMPLY_DEDENT,
16+
PyCF_ONLY_AST,
17+
)
1318
import ast
1419

1520

@@ -248,6 +253,72 @@ def test_incomplete(self, compiler):
248253
ai('a = f"""')
249254
ai('a = \\')
250255

256+
def test_tokenizer_incomplete_input_classification(self):
257+
cases = [
258+
(
259+
"x = 'abc",
260+
"single",
261+
builtins._IncompleteInputError,
262+
("incomplete input", 1, 5, 1, -1),
263+
),
264+
(
265+
'f"""abc',
266+
"single",
267+
builtins._IncompleteInputError,
268+
("incomplete input", 1, 1, 1, -1),
269+
),
270+
(
271+
"x = \\\n",
272+
"single",
273+
builtins._IncompleteInputError,
274+
("incomplete input", 1, 6, 1, -1),
275+
),
276+
(
277+
"x = 'abc",
278+
"exec",
279+
SyntaxError,
280+
(
281+
"unterminated string literal (detected at line 1)",
282+
1, 5, 1, 5,
283+
),
284+
),
285+
(
286+
'f"abc',
287+
"single",
288+
SyntaxError,
289+
(
290+
"unterminated f-string literal (detected at line 1)",
291+
1, 1, 1, 1,
292+
),
293+
),
294+
(
295+
"x = \\",
296+
"single",
297+
SyntaxError,
298+
(
299+
"unexpected character after line continuation character",
300+
1, 5, 1, 0,
301+
),
302+
),
303+
]
304+
305+
for source, mode, exception_type, expected in cases:
306+
with self.subTest(source=source, mode=mode):
307+
with self.assertRaises(exception_type) as caught:
308+
compile(
309+
source,
310+
"<test>",
311+
mode,
312+
PyCF_ALLOW_INCOMPLETE_INPUT,
313+
)
314+
self.assertIs(type(caught.exception), exception_type)
315+
error = caught.exception
316+
self.assertEqual(
317+
(error.msg, error.lineno, error.offset,
318+
error.end_lineno, error.end_offset),
319+
expected,
320+
)
321+
251322
@subTests('compiler', COMPILERS)
252323
def test_invalid(self, compiler):
253324
ai = functools.partial(self.assertInvalid, compiler=compiler)

‎Lib/test/test_eof.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@ def test_line_continuation_EOF(self):
126126
@unittest.skipIf(not sys.executable, "sys.executable required")
127127
@force_not_colorized
128128
def test_line_continuation_EOF_from_file_bpo2180(self):
129-
"""Ensure tok_nextc() does not add too many ending newlines."""
130129
with os_helper.temp_dir() as temp_dir:
131130
file_name = script_helper.make_script(temp_dir, 'foo', '\\')
132131
rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name)

‎Lib/test/test_fstring.py‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -710,14 +710,16 @@ def test_double_braces(self):
710710
])
711711

712712
def test_double_brace_ast_location_covers_both_source_braces(self):
713-
value = ast.parse('f"a{{"').body[0].value.values[0]
714-
self.assertIsInstance(value, ast.Constant)
715-
self.assertEqual(value.value, "a{")
716-
self.assertEqual(
717-
(value.lineno, value.col_offset, value.end_lineno,
718-
value.end_col_offset),
719-
(1, 2, 1, 5),
720-
)
713+
for source, expected in [('f"a{{"', "a{"), ('f"a}}"', "a}")]:
714+
with self.subTest(source=source):
715+
value = ast.parse(source).body[0].value.values[0]
716+
self.assertIsInstance(value, ast.Constant)
717+
self.assertEqual(value.value, expected)
718+
self.assertEqual(
719+
(value.lineno, value.col_offset, value.end_lineno,
720+
value.end_col_offset),
721+
(1, 2, 1, 5),
722+
)
721723

722724
def test_compile_time_concat(self):
723725
x = 'def'
@@ -1679,6 +1681,12 @@ def __repr__(self):
16791681

16801682
self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
16811683
self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
1684+
self.assertEqual(f'''{" \" # nooo \" " # real comment
1685+
=}''', '" \\" # nooo \\" " \n=\' " # nooo " \'')
1686+
self.assertEqual(f'{"""a" # inside"""=}',
1687+
'"""a" # inside"""=\'a" # inside\'')
1688+
self.assertEqual(f"{'''a' # inside'''=}",
1689+
"'''a' # inside'''=\"a' # inside\"")
16821690

16831691
self.assertEqual(f'{ # some comment goes here
16841692
"""hello"""=}', ' \n """hello"""=\'hello\'')

‎Lib/test/test_syntax.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3088,6 +3088,24 @@ def test_expression_with_assignment(self):
30883088
def test_curly_brace_after_primary_raises_immediately(self):
30893089
self._check_error("f{}", "invalid syntax", mode="single")
30903090

3091+
def test_tokenizer_eof_error_offsets_after_non_ascii(self):
3092+
self._check_error(
3093+
"é + (",
3094+
re.escape("'(' was never closed"),
3095+
lineno=1,
3096+
offset=5,
3097+
end_lineno=1,
3098+
end_offset=0,
3099+
)
3100+
self._check_error(
3101+
"é + \\\n",
3102+
"unexpected EOF while parsing",
3103+
lineno=1,
3104+
offset=6,
3105+
end_lineno=1,
3106+
end_offset=-1,
3107+
)
3108+
30913109
def test_assign_call(self):
30923110
self._check_error("f() = 1", "assign")
30933111

‎Lib/test/test_tokenize.py‎

Lines changed: 123 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import token
99
import tokenize
1010
import unittest
11+
import warnings
12+
import weakref
1113
from io import BytesIO, StringIO
1214
from textwrap import dedent
1315
from unittest import TestCase, mock
@@ -2244,6 +2246,42 @@ def _get_tokens(source, *, extra_tokens=False):
22442246
extra_tokens=extra_tokens,
22452247
))
22462248

2249+
def test_readline_reentry_is_rejected_without_leaking_cycle(self):
2250+
def make_cycle():
2251+
iterator = None
2252+
2253+
def readline():
2254+
next(iterator)
2255+
2256+
iterator = _tokenize.TokenizerIter(readline, extra_tokens=False)
2257+
with self.assertRaisesRegex(
2258+
RuntimeError, "^tokenizer is already executing$"
2259+
):
2260+
next(iterator)
2261+
return weakref.ref(readline)
2262+
2263+
readline_ref = make_cycle()
2264+
support.gc_collect()
2265+
self.assertIsNone(readline_ref())
2266+
2267+
def test_warning_reentry_is_rejected(self):
2268+
iterator = None
2269+
2270+
def showwarning(*args, **kwargs):
2271+
next(iterator)
2272+
2273+
with warnings.catch_warnings():
2274+
warnings.simplefilter("always")
2275+
with mock.patch.object(warnings, "showwarning", showwarning):
2276+
iterator = _tokenize.TokenizerIter(
2277+
StringIO("1if\n").readline,
2278+
extra_tokens=False,
2279+
)
2280+
with self.assertRaisesRegex(
2281+
RuntimeError, "^tokenizer is already executing$"
2282+
):
2283+
next(iterator)
2284+
22472285
def check_tokenize(self, s, expected):
22482286
# Format the tokens in s in a table format.
22492287
# The ENDMARKER and final NEWLINE are omitted.
@@ -2427,6 +2465,47 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
24272465
(token.ENDMARKER, "", (1, 0), (1, 0), ""),
24282466
)
24292467

2468+
def test_fstring_offsets_remain_valid_after_source_reallocation(self):
2469+
padding = " " * 9000
2470+
expression_line = ")=:>{2}}\n"
2471+
lines = iter([
2472+
'f"""\n',
2473+
"{(\n",
2474+
padding + "1\n",
2475+
expression_line,
2476+
'"""\n',
2477+
"",
2478+
])
2479+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2480+
lines.__next__,
2481+
extra_tokens=True,
2482+
))
2483+
self.assertEqual(tokens, [
2484+
tokenize.TokenInfo(token.FSTRING_START, 'f"""', (1, 0), (1, 4), 'f"""\n'),
2485+
tokenize.TokenInfo(token.FSTRING_MIDDLE, "\n", (1, 4), (2, 0), 'f"""\n{(\n'),
2486+
tokenize.TokenInfo(token.OP, "{", (2, 0), (2, 1), "{(\n"),
2487+
tokenize.TokenInfo(token.OP, "(", (2, 1), (2, 2), "{(\n"),
2488+
tokenize.TokenInfo(token.NL, "\n", (2, 2), (2, 3), "{(\n"),
2489+
tokenize.TokenInfo(token.NUMBER, "1", (3, 9000), (3, 9001), padding + "1\n"),
2490+
tokenize.TokenInfo(token.NL, "\n", (3, 9001), (3, 9002), padding + "1\n"),
2491+
tokenize.TokenInfo(token.OP, ")", (4, 0), (4, 1), expression_line),
2492+
tokenize.TokenInfo(token.OP, "=", (4, 1), (4, 2), expression_line),
2493+
tokenize.TokenInfo(token.OP, ":", (4, 2), (4, 3), expression_line),
2494+
tokenize.TokenInfo(token.FSTRING_MIDDLE, ">", (4, 3), (4, 4), expression_line),
2495+
tokenize.TokenInfo(token.OP, "{", (4, 4), (4, 5), expression_line),
2496+
tokenize.TokenInfo(token.NUMBER, "2", (4, 5), (4, 6), expression_line),
2497+
tokenize.TokenInfo(token.OP, "}", (4, 6), (4, 7), expression_line),
2498+
tokenize.TokenInfo(token.FSTRING_MIDDLE, "", (4, 7), (4, 7), expression_line),
2499+
tokenize.TokenInfo(token.OP, "}", (4, 7), (4, 8), expression_line),
2500+
tokenize.TokenInfo(
2501+
token.FSTRING_MIDDLE, "\n", (4, 8), (5, 0),
2502+
expression_line + '"""\n',
2503+
),
2504+
tokenize.TokenInfo(token.FSTRING_END, '"""', (5, 0), (5, 3), '"""\n'),
2505+
tokenize.TokenInfo(token.NEWLINE, "\n", (5, 3), (5, 4), '"""\n'),
2506+
tokenize.TokenInfo(token.ENDMARKER, "", (6, 0), (6, 0), ""),
2507+
])
2508+
24302509
def test_extra_tokens_relaxes_lexer_errors(self):
24312510
cases = [
24322511
(
@@ -2550,16 +2629,50 @@ def test_degraded_fstring_format_spec(self):
25502629
)
25512630

25522631
def test_escaped_fstring_brace_has_a_position_gap(self):
2553-
tokens = self._get_tokens('f"a{{"', extra_tokens=True)
2554-
self.assertEqual(
2555-
[(tok.type, tok.string, tok.start, tok.end)
2556-
for tok in tokens
2557-
if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
2558-
[
2559-
(token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)),
2560-
(token.FSTRING_END, '"', (1, 5), (1, 6)),
2561-
],
2562-
)
2632+
for source, middle in [('f"a{{"', "a{"), ('f"a}}"', "a}")]:
2633+
with self.subTest(source=source):
2634+
tokens = self._get_tokens(source, extra_tokens=True)
2635+
self.assertEqual(
2636+
[(tok.type, tok.string, tok.start, tok.end)
2637+
for tok in tokens
2638+
if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
2639+
[
2640+
(token.FSTRING_MIDDLE, middle, (1, 2), (1, 4)),
2641+
(token.FSTRING_END, '"', (1, 5), (1, 6)),
2642+
],
2643+
)
2644+
2645+
def test_unclosed_parenthesis_error_position_after_non_ascii(self):
2646+
for extra_tokens in (False, True):
2647+
with self.subTest(extra_tokens=extra_tokens):
2648+
with self.assertRaises(tokenize.TokenError) as caught:
2649+
self._get_tokens("é = (\n", extra_tokens=extra_tokens)
2650+
self.assertEqual(
2651+
caught.exception.args,
2652+
("unexpected EOF in multi-line statement", (1, 0)),
2653+
)
2654+
2655+
def test_line_continuation_error_uses_logical_line_position(self):
2656+
cases = [
2657+
("é \\'f\n", (1, 6)),
2658+
("x1\\\n==\\_", (2, 9)),
2659+
]
2660+
for extra_tokens in (False, True):
2661+
for source, position in cases:
2662+
with self.subTest(
2663+
source=source,
2664+
extra_tokens=extra_tokens,
2665+
):
2666+
with self.assertRaises(tokenize.TokenError) as caught:
2667+
self._get_tokens(source, extra_tokens=extra_tokens)
2668+
self.assertEqual(
2669+
caught.exception.args,
2670+
(
2671+
"unexpected character after line continuation "
2672+
"character",
2673+
position,
2674+
),
2675+
)
25632676

25642677
def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
25652678
with self.assertRaises(tokenize.TokenError) as caught:

‎Makefile.pre.in‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,6 @@ PEGEN_OBJS= \
394394
Parser/peg_api.o
395395

396396
TOKENIZER_OBJS= \
397-
Parser/lexer/buffer.o \
398397
Parser/lexer/lexer.o \
399398
Parser/lexer/number.o \
400399
Parser/lexer/state.o \
@@ -403,6 +402,8 @@ TOKENIZER_OBJS= \
403402
Parser/tokenizer/decoder.o \
404403
Parser/tokenizer/reader.o \
405404
Parser/tokenizer/source.o \
405+
Parser/tokenizer/api.o \
406+
Parser/tokenizer/errors.o \
406407
Parser/tokenizer/helpers.o
407408

408409
PEGEN_HEADERS= \
@@ -411,7 +412,6 @@ PEGEN_HEADERS= \
411412
$(srcdir)/Parser/string_parser.h
412413

413414
TOKENIZER_HEADERS= \
414-
Parser/lexer/buffer.h \
415415
Parser/lexer/lexer.h \
416416
Parser/lexer/lexer_internal.h \
417417
Parser/lexer/state.h \
@@ -420,6 +420,7 @@ TOKENIZER_HEADERS= \
420420
Parser/tokenizer/reader_internal.h \
421421
Parser/tokenizer/source.h \
422422
Parser/tokenizer/tokenizer.h \
423+
Parser/tokenizer/errors.h \
423424
Parser/tokenizer/helpers.h
424425

425426
POBJS= \

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,12 +181,14 @@
181181
<ClCompile Include="..\Parser\action_helpers.c" />
182182
<ClCompile Include="..\Parser\string_parser.c" />
183183
<ClCompile Include="..\Parser\token.c" />
184-
<ClCompile Include="..\Parser\lexer\buffer.c" />
185184
<ClCompile Include="..\Parser\lexer\state.c" />
186185
<ClCompile Include="..\Parser\lexer\lexer.c" />
187186
<ClCompile Include="..\Parser\lexer\number.c" />
188187
<ClCompile Include="..\Parser\lexer\string.c" />
188+
<ClCompile Include="..\Parser\tokenizer\api.c" />
189+
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
189190
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
191+
<ClCompile Include="..\Parser\tokenizer\errors.c" />
190192
<ClCompile Include="..\Parser\tokenizer\reader.c" />
191193
<ClCompile Include="..\Parser\tokenizer\source.c" />
192194
<ClCompile Include="..\Parser\tokenizer\helpers.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -469,15 +469,21 @@
469469
<ClCompile Include="..\Parser\lexer\string.c">
470470
<Filter>Source Files</Filter>
471471
</ClCompile>
472-
<ClCompile Include="..\Parser\lexer\buffer.c">
472+
<ClCompile Include="..\Parser\lexer\state.c">
473473
<Filter>Source Files</Filter>
474474
</ClCompile>
475-
<ClCompile Include="..\Parser\lexer\state.c">
475+
<ClCompile Include="..\Parser\tokenizer\api.c">
476+
<Filter>Source Files</Filter>
477+
</ClCompile>
478+
<ClCompile Include="..\Parser\tokenizer\cursor.c">
476479
<Filter>Source Files</Filter>
477480
</ClCompile>
478481
<ClCompile Include="..\Parser\tokenizer\decoder.c">
479482
<Filter>Source Files</Filter>
480483
</ClCompile>
484+
<ClCompile Include="..\Parser\tokenizer\errors.c">
485+
<Filter>Source Files</Filter>
486+
</ClCompile>
481487
<ClCompile Include="..\Parser\tokenizer\reader.c">
482488
<Filter>Source Files</Filter>
483489
</ClCompile>

0 commit comments

Comments
 (0)