gh-153569: cover reader source relocation

Author: pablogsal

Lib/test/test_tokenize.py

@@ -2363,6 +2363,26 @@ def test_multiline_readline_chunk(self):
         ))
         self.assertEqual(tokens, expected)
 
+    def test_multiline_readline_matches_physical_lines(self):
+        source = (
+            'value = f"""first {\n'
+            '    40 + 2\n'
+            '} last"""\n'
+            'result = value\n'
+        )
+        physical_lines = iter(source.splitlines(keepends=True) + [""])
+        multiline_chunks = iter([source, ""])
+
+        expected = list(tokenize._generate_tokens_from_c_tokenizer(
+            physical_lines.__next__,
+            extra_tokens=True,
+        ))
+        actual = list(tokenize._generate_tokens_from_c_tokenizer(
+            multiline_chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(actual, expected)
+
     def test_multiline_readline_chunk_with_unterminated_tail(self):
         readline = mock.Mock(side_effect=["x\nz", ""])
         iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
@@ -2427,6 +2447,34 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_source_reallocation(self):
+        padding = " " * 9000
+        expression_line = ")=:>{2}}\n"
+        physical_lines = [
+            'f"""\n',
+            "{(\n",
+            padding + "1\n",
+            expression_line,
+            '"""\n',
+        ]
+        source = "".join(physical_lines)
+        chunks = iter([
+            "".join(physical_lines[:2]),
+            "".join(physical_lines[2:4]),
+            physical_lines[4],
+            "",
+        ])
+
+        expected = self._get_tokens(source, extra_tokens=True)
+        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+            chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(tokens, expected)
+        number = next(tok for tok in tokens if tok.type == token.NUMBER)
+        self.assertEqual(number.string, "1")
+        self.assertEqual((number.start, number.end), ((3, 9000), (3, 9001)))
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (