Skip to content

Commit f5dbcba

Browse files
pablogsalpablogsal
authored
gh-153569: Unify tokenizer input readers (#156472)
1 parent 8e0159b commit f5dbcba

32 files changed

Lines changed: 1684 additions & 1214 deletions

‎Lib/test/test_cmd_line_script.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import os
1010
import os.path
1111
import py_compile
12+
import select
1213
import subprocess
1314
import io
1415

@@ -168,6 +169,24 @@ def test_stdin_loader(self):
168169
expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
169170
self.assertIn(expected, out)
170171

172+
@unittest.skipIf(sys.platform == "win32", "select() cannot wait for pipes")
173+
def test_stdin_syntax_error_does_not_read_ahead(self):
174+
process = spawn_python()
175+
try:
176+
process.stdin.write(b")\n")
177+
process.stdin.flush()
178+
output = b""
179+
while b"SyntaxError" not in output:
180+
ready, _, _ = select.select(
181+
[process.stdout], [], [], support.SHORT_TIMEOUT
182+
)
183+
self.assertTrue(ready, output)
184+
data = os.read(process.stdout.fileno(), 4096)
185+
self.assertTrue(data, output)
186+
output += data
187+
finally:
188+
kill_python(process)
189+
171190
@contextlib.contextmanager
172191
def interactive_python(self, separate_stderr=False):
173192
if separate_stderr:

‎Lib/test/test_fstring.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1603,6 +1603,8 @@ def test_debug_conversion(self):
16031603
self.assertEqual(f'''{
16041604
3
16051605
=}''', '\n3\n=3')
1606+
x = 1
1607+
self.assertEqual(eval('f"""{(\nx\n)=}"""'), '(\nx\n)=1')
16061608

16071609
# Since = is handled specially, make sure all existing uses of
16081610
# it still work.

‎Lib/test/test_repl.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,36 @@ def test_multiline_string_parsing(self):
152152
output = kill_python(p)
153153
self.assertEqual(p.returncode, 0)
154154

155+
@unittest.skipIf(sys.platform == "win32", "select() cannot wait for pipes")
156+
def test_secondary_prompt_is_not_read_ahead(self):
157+
process = spawn_repl()
158+
output = ""
159+
160+
def read_until(marker, start=0):
161+
nonlocal output
162+
while marker not in output[start:]:
163+
ready, _, _ = select.select(
164+
[process.stdout], [], [], SHORT_TIMEOUT
165+
)
166+
self.assertTrue(ready, output)
167+
data = os.read(process.stdout.fileno(), 4096)
168+
self.assertTrue(data, output)
169+
output += data.decode()
170+
171+
try:
172+
read_until(">>> ")
173+
process.stdin.write("(\n")
174+
process.stdin.flush()
175+
read_until("... ")
176+
after_secondary_prompt = len(output)
177+
178+
process.stdin.write("1)\n")
179+
process.stdin.flush()
180+
read_until(">>> ", after_secondary_prompt)
181+
self.assertEqual(output[after_secondary_prompt:], "1\n>>> ")
182+
finally:
183+
kill_python(process)
184+
155185
@cpython_only
156186
def test_lexer_buffer_realloc_with_null_start(self):
157187
# gh-144759: NULL pointer arithmetic in the lexer when start and

‎Lib/test/test_source_encoding.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,46 @@ def test_truncated_utf8_at_eof(self):
8282
with self.subTest(seq=seq):
8383
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
8484

85+
def test_invalid_utf8_offset_after_non_ascii(self):
86+
with self.assertRaises(SyntaxError) as caught:
87+
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
88+
error = caught.exception
89+
self.assertEqual(
90+
(error.lineno, error.offset, error.end_lineno, error.end_offset),
91+
(1, 6, 1, 6),
92+
)
93+
94+
def test_long_bom_conflict_message_is_not_truncated(self):
95+
encoding = "x" * 400
96+
source = b"\xef\xbb\xbf# coding:" + encoding.encode() + b"\n"
97+
with self.assertRaises(SyntaxError) as caught:
98+
compile(source, "<test>", "exec")
99+
self.assertEqual(
100+
caught.exception.msg,
101+
f"encoding problem: {encoding} with BOM",
102+
)
103+
104+
def _assert_python_file_ok(self, source):
105+
with tempfile.TemporaryDirectory() as directory:
106+
filename = script_helper.make_script(directory, "source", source)
107+
script_helper.assert_python_ok(filename)
108+
109+
@support.requires_subprocess()
110+
def test_stateful_file_decoder_spans_lines(self):
111+
encoded_name = "変数".encode("iso2022_jp")
112+
payload = encoded_name[3:-3]
113+
source = (
114+
b"# coding: iso2022_jp\n"
115+
b"# \x1b$B" + payload + b"\n"
116+
+ payload + b"\x1b(B = 1\n"
117+
)
118+
self._assert_python_file_ok(source)
119+
120+
@support.requires_subprocess()
121+
def test_stateful_file_decoder_finalizes_before_implicit_newline(self):
122+
source = b"# coding: hz\n# ~{1dA?"
123+
self._assert_python_file_ok(source)
124+
85125
@support.requires_subprocess()
86126
def test_20731(self):
87127
sub = subprocess.Popen([sys.executable,

‎Lib/test/test_tokenize.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import contextlib
2+
import _tokenize
23
import itertools
34
import os
45
import re
@@ -2273,6 +2274,159 @@ def readline(encoding):
22732274
))
22742275
self.assertEqual(tokens, expected)
22752276

2277+
def test_stateful_decoder_spans_readline_calls(self):
2278+
encoded_name = "変数".encode("iso2022_jp")
2279+
payload = encoded_name[3:-3]
2280+
lines = iter([
2281+
b"# \x1b$B" + payload + b"\n",
2282+
payload + b"\x1b(B\n",
2283+
b"",
2284+
])
2285+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2286+
lines.__next__,
2287+
extra_tokens=True,
2288+
encoding="iso2022_jp",
2289+
))
2290+
self.assertEqual(tokens, [
2291+
tokenize.TokenInfo(
2292+
token.COMMENT, "# 変数", (1, 0), (1, 4), "# 変数\n"
2293+
),
2294+
tokenize.TokenInfo(token.NL, "\n", (1, 4), (1, 5), "# 変数\n"),
2295+
tokenize.TokenInfo(token.NAME, "変数", (2, 0), (2, 2), "変数\n"),
2296+
tokenize.TokenInfo(token.NEWLINE, "\n", (2, 2), (2, 3), "変数\n"),
2297+
tokenize.TokenInfo(token.ENDMARKER, "", (3, 0), (3, 0), ""),
2298+
])
2299+
2300+
def test_utf16_bom_in_each_readline_chunk(self):
2301+
lines = iter([
2302+
"x\n".encode("utf-16"),
2303+
"y\n".encode("utf-16"),
2304+
b"",
2305+
])
2306+
tokens = _tokenize.TokenizerIter(
2307+
lines.__next__, encoding="utf-16", extra_tokens=True
2308+
)
2309+
self.assertEqual(list(tokens), [
2310+
(token.NAME, "x", (1, 0), (1, 1), "x\n"),
2311+
(token.NEWLINE, "\n", (1, 1), (1, 2), "x\n"),
2312+
(token.NAME, "y", (2, 0), (2, 1), "y\n"),
2313+
(token.NEWLINE, "\n", (2, 1), (2, 2), "y\n"),
2314+
(token.ENDMARKER, "", (3, 0), (3, 0), ""),
2315+
])
2316+
2317+
def test_utf8_decoder_spans_readline_calls(self):
2318+
lines = iter([b"x\xc3", b"\xa9\n", b""])
2319+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2320+
lines.__next__,
2321+
extra_tokens=True,
2322+
encoding="utf-8",
2323+
))
2324+
self.assertEqual(tokens, [
2325+
tokenize.TokenInfo(token.NAME, "xé", (1, 0), (1, 2), "xé\n"),
2326+
tokenize.TokenInfo(token.NEWLINE, "\n", (1, 2), (1, 3), "xé\n"),
2327+
tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""),
2328+
])
2329+
2330+
def test_utf8_decoder_replaces_incomplete_input_at_eof(self):
2331+
expected = [
2332+
tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"),
2333+
tokenize.TokenInfo(token.NEWLINE, "", (1, 2), (1, 3), "x�"),
2334+
tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""),
2335+
]
2336+
for chunks in ([b"x\xe9", b""], [b"x\xe9"]):
2337+
with self.subTest(chunks=chunks):
2338+
lines = iter(chunks)
2339+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2340+
lines.__next__,
2341+
extra_tokens=True,
2342+
encoding="utf-8",
2343+
))
2344+
self.assertEqual(tokens, expected)
2345+
2346+
def test_multiline_readline_chunk(self):
2347+
expected = [
2348+
tokenize.TokenInfo(token.NAME, "x", (1, 0), (1, 1), "x=1\n"),
2349+
tokenize.TokenInfo(token.OP, "=", (1, 1), (1, 2), "x=1\n"),
2350+
tokenize.TokenInfo(token.NUMBER, "1", (1, 2), (1, 3), "x=1\n"),
2351+
tokenize.TokenInfo(token.NEWLINE, "\n", (1, 3), (1, 4), "x=1\n"),
2352+
tokenize.TokenInfo(token.NAME, "y", (2, 0), (2, 1), "y=2\n"),
2353+
tokenize.TokenInfo(token.OP, "=", (2, 1), (2, 2), "y=2\n"),
2354+
tokenize.TokenInfo(token.NUMBER, "2", (2, 2), (2, 3), "y=2\n"),
2355+
tokenize.TokenInfo(token.NEWLINE, "\n", (2, 3), (2, 4), "y=2\n"),
2356+
tokenize.TokenInfo(token.ENDMARKER, "", (3, 0), (3, 0), ""),
2357+
]
2358+
lines = iter([b"x=1\ny=2\n", b""])
2359+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2360+
lines.__next__,
2361+
extra_tokens=True,
2362+
encoding="utf-8",
2363+
))
2364+
self.assertEqual(tokens, expected)
2365+
2366+
def test_multiline_readline_chunk_with_unterminated_tail(self):
2367+
readline = mock.Mock(side_effect=["x\nz", ""])
2368+
iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
2369+
expected = [
2370+
(token.NAME, "x", (1, 0), (1, 1), "x\n"),
2371+
(token.NEWLINE, "\n", (1, 1), (1, 2), "x\n"),
2372+
(token.NAME, "z", (2, 0), (2, 1), "z"),
2373+
(token.NEWLINE, "", (2, 1), (2, 2), "z"),
2374+
]
2375+
self.assertEqual(readline.call_count, 0)
2376+
for token_info in expected:
2377+
self.assertEqual(next(iterator), token_info)
2378+
self.assertEqual(readline.call_count, 1)
2379+
self.assertEqual(
2380+
next(iterator),
2381+
(token.ENDMARKER, "", (3, 0), (3, 0), ""),
2382+
)
2383+
self.assertEqual(readline.call_count, 2)
2384+
2385+
def test_readline_callback_is_not_read_ahead(self):
2386+
readline = mock.Mock(side_effect=["x\n", "y\n", ""])
2387+
iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
2388+
expected = [
2389+
((token.NAME, "x", (1, 0), (1, 1), "x\n"), 1),
2390+
((token.NEWLINE, "\n", (1, 1), (1, 2), "x\n"), 1),
2391+
((token.NAME, "y", (2, 0), (2, 1), "y\n"), 2),
2392+
]
2393+
self.assertEqual(readline.call_count, 0)
2394+
for token_info, calls in expected:
2395+
self.assertEqual(next(iterator), token_info)
2396+
self.assertEqual(readline.call_count, calls)
2397+
2398+
def test_encoded_readline_replaces_invalid_bytes(self):
2399+
lines = iter([b"\xff\n", b""])
2400+
tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2401+
lines.__next__,
2402+
extra_tokens=True,
2403+
encoding="utf-8",
2404+
))
2405+
self.assertEqual(tokens, [
2406+
tokenize.TokenInfo(token.NAME, "�", (1, 0), (1, 1), "�\n"),
2407+
tokenize.TokenInfo(token.NEWLINE, "\n", (1, 1), (1, 2), "�\n"),
2408+
tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""),
2409+
])
2410+
2411+
def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
2412+
iterator = _tokenize.TokenizerIter(
2413+
lambda: b"",
2414+
extra_tokens=True,
2415+
encoding="missing-tokenizer-codec",
2416+
)
2417+
with self.assertRaises(LookupError):
2418+
next(iterator)
2419+
2420+
iterator = _tokenize.TokenizerIter(
2421+
iter(()).__next__,
2422+
extra_tokens=True,
2423+
encoding="missing-tokenizer-codec",
2424+
)
2425+
self.assertEqual(
2426+
next(iterator),
2427+
(token.ENDMARKER, "", (1, 0), (1, 0), ""),
2428+
)
2429+
22762430
def test_extra_tokens_relaxes_lexer_errors(self):
22772431
cases = [
22782432
(

‎Makefile.pre.in‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -400,11 +400,9 @@ TOKENIZER_OBJS= \
400400
Parser/lexer/state.o \
401401
Parser/lexer/string.o \
402402
Parser/tokenizer/cursor.o \
403-
Parser/tokenizer/file_tokenizer.o \
404-
Parser/tokenizer/readline_tokenizer.o \
403+
Parser/tokenizer/decoder.o \
404+
Parser/tokenizer/reader.o \
405405
Parser/tokenizer/source.o \
406-
Parser/tokenizer/string_tokenizer.o \
407-
Parser/tokenizer/utf8_tokenizer.o \
408406
Parser/tokenizer/helpers.o
409407

410408
PEGEN_HEADERS= \
@@ -418,6 +416,8 @@ TOKENIZER_HEADERS= \
418416
Parser/lexer/lexer_internal.h \
419417
Parser/lexer/state.h \
420418
Parser/tokenizer/cursor.h \
419+
Parser/tokenizer/reader.h \
420+
Parser/tokenizer/reader_internal.h \
421421
Parser/tokenizer/source.h \
422422
Parser/tokenizer/tokenizer.h \
423423
Parser/tokenizer/helpers.h

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,9 @@
186186
<ClCompile Include="..\Parser\lexer\lexer.c" />
187187
<ClCompile Include="..\Parser\lexer\number.c" />
188188
<ClCompile Include="..\Parser\lexer\string.c" />
189-
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
190-
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
191-
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />
192-
<ClCompile Include="..\Parser\tokenizer\readline_tokenizer.c" />
189+
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
190+
<ClCompile Include="..\Parser\tokenizer\reader.c" />
191+
<ClCompile Include="..\Parser\tokenizer\source.c" />
193192
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
194193
<ClCompile Include="..\PC\invalid_parameter_handler.c" />
195194
<ClCompile Include="..\PC\msvcrtmodule.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -475,16 +475,13 @@
475475
<ClCompile Include="..\Parser\lexer\state.c">
476476
<Filter>Source Files</Filter>
477477
</ClCompile>
478-
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c">
478+
<ClCompile Include="..\Parser\tokenizer\decoder.c">
479479
<Filter>Source Files</Filter>
480480
</ClCompile>
481-
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c">
481+
<ClCompile Include="..\Parser\tokenizer\reader.c">
482482
<Filter>Source Files</Filter>
483483
</ClCompile>
484-
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c">
485-
<Filter>Source Files</Filter>
486-
</ClCompile>
487-
<ClCompile Include="..\Parser\tokenizer\readline_tokenizer.c">
484+
<ClCompile Include="..\Parser\tokenizer\source.c">
488485
<Filter>Source Files</Filter>
489486
</ClCompile>
490487
<ClCompile Include="..\Parser\tokenizer\helpers.c">

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,8 @@
424424
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
425425
<ClInclude Include="..\Parser\lexer\buffer.h" />
426426
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
427+
<ClInclude Include="..\Parser\tokenizer\reader.h" />
428+
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
427429
<ClInclude Include="..\Parser\tokenizer\source.h" />
428430
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
429431
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -593,10 +595,8 @@
593595
<ClCompile Include="..\Parser\lexer\buffer.c" />
594596
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
595597
<ClCompile Include="..\Parser\tokenizer\source.c" />
596-
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
597-
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
598-
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />
599-
<ClCompile Include="..\Parser\tokenizer\readline_tokenizer.c" />
598+
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
599+
<ClCompile Include="..\Parser\tokenizer\reader.c" />
600600
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
601601
<ClCompile Include="..\Parser\token.c" />
602602
<ClCompile Include="..\Parser\pegen.c" />

0 commit comments

Comments
 (0)