gh-153569: preserve tolerant f-string delimiter state

Author: pablogsal

Lib/test/test_tokenize.py

@@ -2726,6 +2726,34 @@ def test_tolerant_fstring_closer_at_expression_entry_depth(self):
                     ("f-string: unmatched ']'", position),
                 )
 
+    def test_tolerant_fstring_mismatch_keeps_delimiter_state(self):
+        sources = [
+            ("f'{x:{)}!]", (1, 10)),
+            ("f'{x:{)}!\n]", (2, 1)),
+        ]
+        for source, tolerant_position in sources:
+            cases = [
+                (False, ("f-string: unmatched ')'", (1, 7))),
+                (True, ("f-string: unmatched ']'", tolerant_position)),
+            ]
+            for extra_tokens, expected in cases:
+                with self.subTest(
+                    source=source, extra_tokens=extra_tokens
+                ):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(source, extra_tokens=extra_tokens)
+                    self.assertEqual(caught.exception.args, expected)
+
+            with self.subTest(source=source, public=True):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    list(tokenize.tokenize(
+                        BytesIO(source.encode()).readline
+                    ))
+                self.assertEqual(
+                    caught.exception.args,
+                    ("f-string: unmatched ']'", tolerant_position),
+                )
+
     def test_int(self):
 
         self.check_tokenize('0xff <= 255', """\

Parser/lexer/string.c

@@ -381,7 +381,9 @@ _PyTok_LexOperator(struct _PyTokenizer *tok, _PyTok_Frame *frame,
     case ']':
     case '}':
         if (INSIDE_FSTRING_EXPR(frame) && c != '}' &&
-                tok->level == frame->paren_depth_at_entry) {
+                tok->level == (tok->extra_tokens
+                    ? frame_body_paren_depth(tok)
+                    : frame->paren_depth_at_entry)) {
             _PyTok_SyntaxError(
                 tok, "%c-string: unmatched '%c'",
                 _PyTok_FramePrefix(tok), c);