gh-153569: preserve malformed f-string delimiter state

Author: pablogsal

Lib/test/test_syntax.py

@@ -3106,6 +3106,17 @@ def test_tokenizer_eof_error_offsets_after_non_ascii(self):
             end_offset=-1,
         )
 
+    def test_error_line_excludes_old_newline(self):
+        cases = [
+            ('f"{a:{\n0\n\'\'=(\n="0("', (2, 1, "0", 3, 2)),
+            ("f'{x!=[\n]a\n}(]==!", (2, 3, "]a", 2, 2)),
+        ]
+        for source, expected in cases:
+            with self.subTest(source=source):
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, "<testcase>", "exec")
+                self.assertEqual(caught.exception.args[1][1:], expected)
+
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")
 

Lib/test/test_tokenize.py

@@ -2754,6 +2754,67 @@ def test_tolerant_fstring_mismatch_keeps_delimiter_state(self):
                     ("f-string: unmatched ']'", tolerant_position),
                 )
 
+    def test_nested_fstring_preserves_delimiter_errors(self):
+        cases = [
+            (
+                'f\'{f"{:"])',
+                ("f-string: unmatched ']'", (1, 9)),
+                ("f-string: unmatched ')'", (1, 10)),
+            ),
+            (
+                'f\'{f"{:"0a})0=',
+                ("invalid decimal literal", (1, 9)),
+                (
+                    "unterminated f-string literal (detected at line 1)",
+                    (1, 1),
+                ),
+            ),
+            (
+                'f\'{f"{:"=\n)):',
+                ("f-string: unmatched ')'", (2, 1)),
+                ("f-string: unmatched ')'", (2, 2)),
+            ),
+            (
+                'f\'{f"{x:[")=00\n!',
+                ("f-string: unmatched ')'", (1, 11)),
+                ("unexpected EOF in multi-line statement", (2, 17)),
+            ),
+        ]
+        for source, parser_error, tolerant_error in cases:
+            for extra_tokens, expected in [
+                (False, parser_error),
+                (True, tolerant_error),
+            ]:
+                with self.subTest(
+                    source=source, extra_tokens=extra_tokens
+                ):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(source, extra_tokens=extra_tokens)
+                    self.assertEqual(caught.exception.args, expected)
+
+            with self.subTest(source=source, public=True):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    list(tokenize.tokenize(
+                        BytesIO(source.encode()).readline
+                    ))
+                self.assertEqual(caught.exception.args, tolerant_error)
+
+    def test_fstring_expression_depth_error_location(self):
+        source = "f'{x:{:{:!{{:}{:"
+        expected = (
+            "f-string: expressions nested too deeply",
+            (1, 10),
+        )
+        for extra_tokens in (False, True):
+            with self.subTest(extra_tokens=extra_tokens):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    self._get_tokens(source, extra_tokens=extra_tokens)
+                self.assertEqual(caught.exception.args, expected)
+
+        with self.assertRaises(tokenize.TokenError) as caught:
+            list(tokenize.tokenize(BytesIO(source.encode()).readline))
+        self.assertEqual(caught.exception.args, expected)
+
     def test_int(self):
 
         self.check_tokenize('0xff <= 255', """\

Parser/lexer/string.c

@@ -77,6 +77,17 @@ frame_pop_string(struct _PyTokenizer *tok)
         frame_pop(tok);
     }
     assert(tok->frame_index > 0);
+    if (_PyTok_FrameDepth(tok) > 1) {
+        _PyTok_Frame *body = _PyTok_CurrentFrame(tok);
+        assert(tok->level >= body->paren_depth_at_entry);
+        int delta = tok->level - body->paren_depth_at_entry;
+        if (delta != 0) {
+            /* Keep parent frame boundaries relative to open inner delimiters. */
+            for (int i = 1; i < tok->frame_index; i++) {
+                tok->frames[i].paren_depth_at_entry += delta;
+            }
+        }
+    }
     frame_pop(tok);
 }
 
@@ -94,6 +105,16 @@ frame_expr_depth(struct _PyTokenizer *tok)
     return depth;
 }
 
+static void
+record_expr_depth_error(struct _PyTokenizer *tok)
+{
+    _PyTok_Loc loc = _PyTok_LexerLocation(tok, tok->cursor.pos - 1);
+    _PyTok_FormattedErrorAt(
+        tok, _PYTOK_ERR_SYNTAX, loc, loc, loc.lineno,
+        "%c-string: expressions nested too deeply",
+        _PyTok_FramePrefix(tok));
+}
+
 static int
 materialize_frame_metadata(struct _PyTokenizer *tok, _PyTok_Token *token)
 {
@@ -484,9 +505,7 @@ _PyTok_LexFStringMiddle(struct _PyTokenizer *tok, _PyTok_Frame* frame,
     if (start_char == '{') {
         if (_PyTok_LexerPeek(tok, 0) != '{') {
             if (frame_expr_depth(tok) >= MAX_EXPR_NESTING) {
-                _PyTok_SyntaxError(
-                    tok, "%c-string: expressions nested too deeply",
-                    _PyTok_FramePrefix(tok));
+                record_expr_depth_error(tok);
                 return _PyTok_EmitToken(
                     tok, token, ERRORTOKEN, p_start, p_end);
             }
@@ -603,9 +622,7 @@ _PyTok_LexFStringMiddle(struct _PyTokenizer *tok, _PyTok_Frame* frame,
             int peek = _PyTok_LexerPeek(tok, 0);
             if (peek != '{' || active_format_spec) {
                 if (frame_expr_depth(tok) >= MAX_EXPR_NESTING) {
-                    _PyTok_SyntaxError(
-                        tok, "%c-string: expressions nested too deeply",
-                        _PyTok_FramePrefix(tok));
+                    record_expr_depth_error(tok);
                     return _PyTok_EmitToken(
                         tok, token, ERRORTOKEN, p_start, p_end);
                 }

Parser/pegen_errors.c

@@ -267,9 +267,10 @@ get_error_line_from_tokenizer_source(Parser *p, Py_ssize_t lineno)
     if (nul != NULL) {
         len = nul - line;
     }
-    if (_PyTok_SourceIsFile(p->tok) &&
-            _PyTok_LineIsImplicit(p->tok, (int)relative_lineno) &&
-            len > 0 && line[len - 1] == '\n') {
+    int strip_newline = _PyTok_SourceIsFile(p->tok)
+        ? _PyTok_LineIsImplicit(p->tok, (int)relative_lineno)
+        : relative_lineno < _PyTok_Lineno(p->tok);
+    if (strip_newline && len > 0 && line[len - 1] == '\n') {
         len--;
     }
     return PyUnicode_DecodeUTF8(line, len, "replace");