gh-153569: move tokenizer state to source offsets

Author: pablogsal

Lib/test/test_codeop.py

@@ -2,14 +2,19 @@
    Test cases for codeop.py
    Nick Mathewson
 """
+import builtins
 import unittest
 import warnings
 from test.support import subTests, warnings_helper
 from textwrap import dedent
 import functools
 
 from codeop import compile_command, CommandCompiler, Compile
-from codeop import PyCF_DONT_IMPLY_DEDENT, PyCF_ONLY_AST
+from codeop import (
+    PyCF_ALLOW_INCOMPLETE_INPUT,
+    PyCF_DONT_IMPLY_DEDENT,
+    PyCF_ONLY_AST,
+)
 import ast
 
 
@@ -248,6 +253,72 @@ def test_incomplete(self, compiler):
         ai('a = f"""')
         ai('a = \\')
 
+    def test_tokenizer_incomplete_input_classification(self):
+        cases = [
+            (
+                "x = 'abc",
+                "single",
+                builtins._IncompleteInputError,
+                ("incomplete input", 1, 5, 1, -1),
+            ),
+            (
+                'f"""abc',
+                "single",
+                builtins._IncompleteInputError,
+                ("incomplete input", 1, 1, 1, -1),
+            ),
+            (
+                "x = \\\n",
+                "single",
+                builtins._IncompleteInputError,
+                ("incomplete input", 1, 6, 1, -1),
+            ),
+            (
+                "x = 'abc",
+                "exec",
+                SyntaxError,
+                (
+                    "unterminated string literal (detected at line 1)",
+                    1, 5, 1, 5,
+                ),
+            ),
+            (
+                'f"abc',
+                "single",
+                SyntaxError,
+                (
+                    "unterminated f-string literal (detected at line 1)",
+                    1, 1, 1, 1,
+                ),
+            ),
+            (
+                "x = \\",
+                "single",
+                SyntaxError,
+                (
+                    "unexpected character after line continuation character",
+                    1, 5, 1, 0,
+                ),
+            ),
+        ]
+
+        for source, mode, exception_type, expected in cases:
+            with self.subTest(source=source, mode=mode):
+                with self.assertRaises(exception_type) as caught:
+                    compile(
+                        source,
+                        "<test>",
+                        mode,
+                        PyCF_ALLOW_INCOMPLETE_INPUT,
+                    )
+                self.assertIs(type(caught.exception), exception_type)
+                error = caught.exception
+                self.assertEqual(
+                    (error.msg, error.lineno, error.offset,
+                     error.end_lineno, error.end_offset),
+                    expected,
+                )
+
     @subTests('compiler', COMPILERS)
     def test_invalid(self, compiler):
         ai = functools.partial(self.assertInvalid, compiler=compiler)

Lib/test/test_eof.py

@@ -126,7 +126,6 @@ def test_line_continuation_EOF(self):
     @unittest.skipIf(not sys.executable, "sys.executable required")
     @force_not_colorized
     def test_line_continuation_EOF_from_file_bpo2180(self):
-        """Ensure tok_nextc() does not add too many ending newlines."""
         with os_helper.temp_dir() as temp_dir:
             file_name = script_helper.make_script(temp_dir, 'foo', '\\')
             rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name)

Lib/test/test_free_threading/test_tokenize.py

@@ -45,7 +45,7 @@ def next_token(it):
             tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line='  pass'),
             tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line='  pass'),
             tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line='  pass'),
-            tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line='  pass'),
+            tokenize.TokenInfo(type=0, string='', start=(3, 0), end=(3, 0), line=''),
         ]
 
         tokens.sort()

Lib/test/test_fstring.py

@@ -710,14 +710,16 @@ def test_double_braces(self):
                              ])
 
     def test_double_brace_ast_location_covers_both_source_braces(self):
-        value = ast.parse('f"a{{"').body[0].value.values[0]
-        self.assertIsInstance(value, ast.Constant)
-        self.assertEqual(value.value, "a{")
-        self.assertEqual(
-            (value.lineno, value.col_offset, value.end_lineno,
-             value.end_col_offset),
-            (1, 2, 1, 5),
-        )
+        for source, expected in [('f"a{{"', "a{"), ('f"a}}"', "a}")]:
+            with self.subTest(source=source):
+                value = ast.parse(source).body[0].value.values[0]
+                self.assertIsInstance(value, ast.Constant)
+                self.assertEqual(value.value, expected)
+                self.assertEqual(
+                    (value.lineno, value.col_offset, value.end_lineno,
+                     value.end_col_offset),
+                    (1, 2, 1, 5),
+                )
 
     def test_compile_time_concat(self):
         x = 'def'
@@ -1301,6 +1303,13 @@ def test_nested_fstrings(self):
         self.assertEqual(f'{f"{0}"*3}', '000')
         self.assertEqual(f'{f"{y}"*3}', '555')
 
+    def test_deeply_nested_fstrings_with_leading_text(self):
+        source = "0"
+        for depth in range(20):
+            quote = '"' if depth % 2 == 0 else "'"
+            source = f"f{quote}x{{{source}}}{quote}"
+        self.assertEqual(eval(source), "x" * 20 + "0")
+
     def test_invalid_string_prefixes(self):
         single_quote_cases = ["fu''",
                              "uf''",
@@ -1679,6 +1688,12 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'''{" \" # nooo \" " # real comment
+=}''', '" \\" # nooo \\" " \n=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')

Lib/test/test_syntax.py

@@ -3088,6 +3088,42 @@ def test_expression_with_assignment(self):
     def test_curly_brace_after_primary_raises_immediately(self):
         self._check_error("f{}", "invalid syntax", mode="single")
 
+    def test_tokenizer_eof_error_offsets_after_non_ascii(self):
+        self._check_error(
+            "é + (",
+            re.escape("'(' was never closed"),
+            lineno=1,
+            offset=5,
+            end_lineno=1,
+            end_offset=0,
+        )
+        self._check_error(
+            "é + \\\n",
+            "unexpected EOF while parsing",
+            lineno=1,
+            offset=6,
+            end_lineno=1,
+            end_offset=-1,
+        )
+
+    def test_fstring_error_text_excludes_previous_newline(self):
+        cases = [
+            ('f"{a:{\n0\n\'\'=(\n="0("', (2, 1, "0", 3, 2)),
+            ("f'{x!=[\n]a\n}(]==!", (2, 3, "]a", 2, 2)),
+        ]
+        for source, expected in cases:
+            with self.subTest(source=source):
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, "<testcase>", "exec")
+                actual = (
+                    caught.exception.lineno,
+                    caught.exception.offset,
+                    caught.exception.text,
+                    caught.exception.end_lineno,
+                    caught.exception.end_offset,
+                )
+                self.assertEqual(actual, expected)
+
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")