Skip to content

Commit 595b345

Browse files
pablogsalpablogsal
committed
Merge branch 'gh-153569-tokenizer-offset-state' into gh-153569-tokenizer-validation-tools
2 parents bd28225 + b73a1a6 commit 595b345

7 files changed

Lines changed: 121 additions & 19 deletions

File tree

‎Lib/test/test_syntax.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3106,6 +3106,17 @@ def test_tokenizer_eof_error_offsets_after_non_ascii(self):
31063106
end_offset=-1,
31073107
)
31083108

3109+
def test_error_line_excludes_old_newline(self):
3110+
cases = [
3111+
('f"{a:{\n0\n\'\'=(\n="0("', (2, 1, "0", 3, 2)),
3112+
("f'{x!=[\n]a\n}(]==!", (2, 3, "]a", 2, 2)),
3113+
]
3114+
for source, expected in cases:
3115+
with self.subTest(source=source):
3116+
with self.assertRaises(SyntaxError) as caught:
3117+
compile(source, "<testcase>", "exec")
3118+
self.assertEqual(caught.exception.args[1][1:], expected)
3119+
31093120
def test_assign_call(self):
31103121
self._check_error("f() = 1", "assign")
31113122

‎Lib/test/test_tokenize.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2754,6 +2754,67 @@ def test_tolerant_fstring_mismatch_keeps_delimiter_state(self):
27542754
("f-string: unmatched ']'", tolerant_position),
27552755
)
27562756

2757+
def test_nested_fstring_preserves_delimiter_errors(self):
2758+
cases = [
2759+
(
2760+
'f\'{f"{:"])',
2761+
("f-string: unmatched ']'", (1, 9)),
2762+
("f-string: unmatched ')'", (1, 10)),
2763+
),
2764+
(
2765+
'f\'{f"{:"0a})0=',
2766+
("invalid decimal literal", (1, 9)),
2767+
(
2768+
"unterminated f-string literal (detected at line 1)",
2769+
(1, 1),
2770+
),
2771+
),
2772+
(
2773+
'f\'{f"{:"=\n)):',
2774+
("f-string: unmatched ')'", (2, 1)),
2775+
("f-string: unmatched ')'", (2, 2)),
2776+
),
2777+
(
2778+
'f\'{f"{x:[")=00\n!',
2779+
("f-string: unmatched ')'", (1, 11)),
2780+
("unexpected EOF in multi-line statement", (2, 17)),
2781+
),
2782+
]
2783+
for source, parser_error, tolerant_error in cases:
2784+
for extra_tokens, expected in [
2785+
(False, parser_error),
2786+
(True, tolerant_error),
2787+
]:
2788+
with self.subTest(
2789+
source=source, extra_tokens=extra_tokens
2790+
):
2791+
with self.assertRaises(tokenize.TokenError) as caught:
2792+
self._get_tokens(source, extra_tokens=extra_tokens)
2793+
self.assertEqual(caught.exception.args, expected)
2794+
2795+
with self.subTest(source=source, public=True):
2796+
with self.assertRaises(tokenize.TokenError) as caught:
2797+
list(tokenize.tokenize(
2798+
BytesIO(source.encode()).readline
2799+
))
2800+
self.assertEqual(caught.exception.args, tolerant_error)
2801+
2802+
def test_fstring_expression_depth_error_location(self):
2803+
source = "f'{x:{:{:!{{:}{:"
2804+
expected = (
2805+
"f-string: expressions nested too deeply",
2806+
(1, 10),
2807+
)
2808+
for extra_tokens in (False, True):
2809+
with self.subTest(extra_tokens=extra_tokens):
2810+
with self.assertRaises(tokenize.TokenError) as caught:
2811+
self._get_tokens(source, extra_tokens=extra_tokens)
2812+
self.assertEqual(caught.exception.args, expected)
2813+
2814+
with self.assertRaises(tokenize.TokenError) as caught:
2815+
list(tokenize.tokenize(BytesIO(source.encode()).readline))
2816+
self.assertEqual(caught.exception.args, expected)
2817+
27572818
def test_int(self):
27582819

27592820
self.check_tokenize('0xff <= 255', """\

‎Parser/lexer/string.c‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,17 @@ frame_pop_string(struct _PyTokenizer *tok)
7777
frame_pop(tok);
7878
}
7979
assert(tok->frame_index > 0);
80+
if (_PyTok_FrameDepth(tok) > 1) {
81+
_PyTok_Frame *body = _PyTok_CurrentFrame(tok);
82+
assert(tok->level >= body->paren_depth_at_entry);
83+
int delta = tok->level - body->paren_depth_at_entry;
84+
if (delta != 0) {
85+
/* Keep parent frame boundaries relative to open inner delimiters. */
86+
for (int i = 1; i < tok->frame_index; i++) {
87+
tok->frames[i].paren_depth_at_entry += delta;
88+
}
89+
}
90+
}
8091
frame_pop(tok);
8192
}
8293

@@ -94,6 +105,16 @@ frame_expr_depth(struct _PyTokenizer *tok)
94105
return depth;
95106
}
96107

108+
static void
109+
record_expr_depth_error(struct _PyTokenizer *tok)
110+
{
111+
_PyTok_Loc loc = _PyTok_LexerLocation(tok, tok->cursor.pos - 1);
112+
_PyTok_FormattedErrorAt(
113+
tok, _PYTOK_ERR_SYNTAX, loc, loc, loc.lineno,
114+
"%c-string: expressions nested too deeply",
115+
_PyTok_FramePrefix(tok));
116+
}
117+
97118
static int
98119
materialize_frame_metadata(struct _PyTokenizer *tok, _PyTok_Token *token)
99120
{
@@ -484,9 +505,7 @@ _PyTok_LexFStringMiddle(struct _PyTokenizer *tok, _PyTok_Frame* frame,
484505
if (start_char == '{') {
485506
if (_PyTok_LexerPeek(tok, 0) != '{') {
486507
if (frame_expr_depth(tok) >= MAX_EXPR_NESTING) {
487-
_PyTok_SyntaxError(
488-
tok, "%c-string: expressions nested too deeply",
489-
_PyTok_FramePrefix(tok));
508+
record_expr_depth_error(tok);
490509
return _PyTok_EmitToken(
491510
tok, token, ERRORTOKEN, p_start, p_end);
492511
}
@@ -603,9 +622,7 @@ _PyTok_LexFStringMiddle(struct _PyTokenizer *tok, _PyTok_Frame* frame,
603622
int peek = _PyTok_LexerPeek(tok, 0);
604623
if (peek != '{' || active_format_spec) {
605624
if (frame_expr_depth(tok) >= MAX_EXPR_NESTING) {
606-
_PyTok_SyntaxError(
607-
tok, "%c-string: expressions nested too deeply",
608-
_PyTok_FramePrefix(tok));
625+
record_expr_depth_error(tok);
609626
return _PyTok_EmitToken(
610627
tok, token, ERRORTOKEN, p_start, p_end);
611628
}

‎Parser/pegen_errors.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,9 +267,10 @@ get_error_line_from_tokenizer_source(Parser *p, Py_ssize_t lineno)
267267
if (nul != NULL) {
268268
len = nul - line;
269269
}
270-
if (_PyTok_SourceIsFile(p->tok) &&
271-
_PyTok_LineIsImplicit(p->tok, (int)relative_lineno) &&
272-
len > 0 && line[len - 1] == '\n') {
270+
int strip_newline = _PyTok_SourceIsFile(p->tok)
271+
? _PyTok_LineIsImplicit(p->tok, (int)relative_lineno)
272+
: relative_lineno < _PyTok_Lineno(p->tok);
273+
if (strip_newline && len > 0 && line[len - 1] == '\n') {
273274
len--;
274275
}
275276
return PyUnicode_DecodeUTF8(line, len, "replace");

‎Parser/tokenizer/decoder.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -279,10 +279,10 @@ _PyTok_DetectEncoding(struct _PyTokenizer *tok, const _PyTok_Chunk *first,
279279

280280
int
281281
_PyTok_DecodeOnce(struct _PyTokenizer *tok, _PyTok_Chunk *chunk,
282-
const char *encoding)
282+
const char *encoding, const char *errors)
283283
{
284284
PyObject *unicode = PyUnicode_Decode(
285-
chunk->data, chunk->len, encoding, NULL);
285+
chunk->data, chunk->len, encoding, errors);
286286
if (unicode == NULL) {
287287
_PyTok_RecordPending(tok, _PYTOK_ERR_DECODE);
288288
return -1;
@@ -399,7 +399,8 @@ _PyTok_PrepareString(struct _PyTokenizer *tok, const char *input,
399399
.ownership = _PYTOK_CHUNK_BORROWED,
400400
};
401401
if (tok->reader->encoding != NULL && strcmp(tok->reader->encoding, "utf-8") != 0) {
402-
if (_PyTok_DecodeOnce(tok, &decoded, tok->reader->encoding) < 0) {
402+
if (_PyTok_DecodeOnce(
403+
tok, &decoded, tok->reader->encoding, NULL) < 0) {
403404
return -1;
404405
}
405406
}

‎Parser/tokenizer/reader.c‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -376,15 +376,24 @@ next_readline(struct _PyTokenizer *tok, _PyTok_Chunk *chunk)
376376
Py_DECREF(raw);
377377
return _PYTOK_READ_ERROR;
378378
}
379-
if (_PyTok_StartDecoder(tok, "replace") < 0) {
380-
Py_DECREF(raw);
381-
return _PYTOK_READ_ERROR;
382-
}
383379
input.owner = raw;
384380
input.data = PyBytes_AS_STRING(raw);
385381
input.len = PyBytes_GET_SIZE(raw);
386382
input.ownership = _PYTOK_CHUNK_PYOBJECT;
387-
if (input.len > 0 && _PyTok_DecodeChunk(tok, &input, 0) < 0) {
383+
int decoded;
384+
if (reader->decoder == NULL &&
385+
strcmp(reader->encoding, "utf-8") == 0 &&
386+
chunk_is_line(&input)) {
387+
decoded = _PyTok_DecodeOnce(
388+
tok, &input, "utf-8", "replace");
389+
}
390+
else {
391+
decoded = _PyTok_StartDecoder(tok, "replace");
392+
if (decoded == 0 && input.len > 0) {
393+
decoded = _PyTok_DecodeChunk(tok, &input, 0);
394+
}
395+
}
396+
if (decoded < 0) {
388397
_PyTok_ChunkClear(&input);
389398
return _PYTOK_READ_ERROR;
390399
}
@@ -468,7 +477,8 @@ next_interactive(struct _PyTokenizer *tok, _PyTok_Chunk *chunk)
468477
.ownership = _PYTOK_CHUNK_PYMEM,
469478
};
470479
if (reader->encoding != NULL &&
471-
_PyTok_DecodeOnce(tok, &decoded, reader->encoding) < 0) {
480+
_PyTok_DecodeOnce(
481+
tok, &decoded, reader->encoding, NULL) < 0) {
472482
_PyTok_ChunkClear(&decoded);
473483
return _PYTOK_READ_ERROR;
474484
}

‎Parser/tokenizer/reader_internal.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ struct _PyTok_Reader {
6868
struct _PyTokenizer;
6969

7070
char *_PyTok_CopyBytes(const char *, Py_ssize_t);
71-
int _PyTok_DecodeOnce(struct _PyTokenizer *, _PyTok_Chunk *, const char *);
71+
int _PyTok_DecodeOnce(
72+
struct _PyTokenizer *, _PyTok_Chunk *, const char *, const char *);
7273
char *_PyTok_NormalizeNewlines(
7374
const char *, Py_ssize_t, int, int, Py_ssize_t *, int *);
7475
void _PyTok_ChunkClear(_PyTok_Chunk *);

0 commit comments

Comments
 (0)