diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py
index 2fe1fef241e90a..e986e0f6b74f40 100644
--- a/Lib/test/test_capi/test_tokenizer.py
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -9,9 +9,6 @@ class TokenizerTests(unittest.TestCase):
     def test_source(self):
         _testinternalcapi.test_tokenizer_source()
 
-    def test_cursor(self):
-        _testinternalcapi.test_tokenizer_cursor()
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index c1ef1a73f05c20..debd9a41063e49 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1679,6 +1679,12 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 7e02191db86be5..0b5bdcdeda3c19 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_buffer_reallocation(self):
+        padding = " " * 9000
+        expression_line = ")=:>{2}}\n"
+        physical_lines = [
+            'f"""\n',
+            "{(\n",
+            padding + "1\n",
+            expression_line,
+            '"""\n',
+        ]
+        source = "".join(physical_lines)
+        chunks = iter([
+            "".join(physical_lines[:2]),
+            "".join(physical_lines[2:4]),
+            physical_lines[4],
+            "",
+        ])
+
+        expected = self._get_tokens(source, extra_tokens=True)
+        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+            chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(tokens, expected)
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de1..b60b50e446e365 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -287,5 +287,13 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
index adcfe4c5259eb2..982b2d66216c98 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -399,7 +399,6 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
-		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -415,7 +414,6 @@ TOKENIZER_HEADERS= \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
-		Parser/tokenizer/cursor.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3462,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
index 0b292410d3eb4e..46ac3c40724cff 100644
--- a/Modules/_testinternalcapi/tokenizer.c
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -1,6 +1,6 @@
 #include "parts.h"
 
-#include "../../Parser/tokenizer/cursor.h"
+#include "../../Parser/tokenizer/source.h"
 
 static int
 check(int condition, const char *message)
@@ -23,16 +23,6 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
-static int
-same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
-{
-    return left->source == right->source &&
-        left->pos == right->pos &&
-        left->line_start == right->line_start &&
-        left->line_end == right->line_end &&
-        left->lineno == right->lineno;
-}
-
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
@@ -40,80 +30,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
-    _PyTok_Loc loc;
-    _PyTok_Line line;
-    if (check(_PyTok_SourceLocation(
-                  &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-              "cannot locate empty source") < 0 ||
-            check(loc.lineno == 1 && loc.byte_col == 0,
-                  "wrong empty source location") < 0 ||
-            check(_PyTok_SourceLine(&source, 1, &line) == 0,
-                  "cannot find empty source line") < 0 ||
-            check(line.start == 0 && line.end == 0,
-                  "wrong empty source line") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
-                "accepted empty source line") < 0 ||
+    if (check_system_error(
+            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+            "accepted empty source line") < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
                 "accepted multiple source lines") < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
-              "wrong first source offset") < 0 ||
+                "accepted missing implicit newline") < 0 ||
+            check(_PyTok_SourceAppendLine(
+                      &source, "alpha\n", 6, 0) == 0,
+                  "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
                       &source, "\xce\xb2\n", 3, 1) == 6,
                   "wrong second source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "nul\0x\n", 6, 0) == 9,
-                  "wrong third source offset") < 0) {
-        goto error;
-    }
-
-    int marker_line = 257;
-    int final_line = 300;
-    _PyTok_Off marker_start = -1;
-    for (int lineno = 4; lineno <= final_line; lineno++) {
-        const char *text = lineno == marker_line ? "marker\n" : "x\n";
-        Py_ssize_t len = (Py_ssize_t)strlen(text);
-        _PyTok_Off start = _PyTok_SourceAppendLine(
-            &source, text, len, lineno == final_line);
-        if (start < 0) {
-            goto error;
-        }
-        if (lineno == marker_line) {
-            marker_start = start;
-        }
-    }
-
-    if (check(source.nlines == final_line, "wrong source line count") < 0 ||
-            check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
-                  "cannot find late source line") < 0 ||
-            check(line.start == marker_start &&
-                      line.end == marker_start + 7,
-                  "wrong late source line") < 0 ||
-            check(!line.implicit_newline && !line.contains_nul,
-                  "wrong late source flags") < 0 ||
-            check(_PyTok_SourceLine(&source, 2, &line) == 0,
-                  "cannot find second source line") < 0 ||
-            check(line.start == 6 && line.end == 9 &&
-                      line.implicit_newline && !line.contains_nul,
-                  "wrong second source line") < 0 ||
             check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
                       _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong early implicit newline flags") < 0 ||
-            check(_PyTok_SourceLine(&source, 3, &line) == 0,
-                  "cannot find third source line") < 0 ||
-            check(line.contains_nul, "missing null byte flag") < 0 ||
-            check(_PyTok_SourceLine(&source, final_line, &line) == 0,
-                  "cannot find final source line") < 0 ||
-            check(line.implicit_newline &&
-                      _PyTok_SourceLineIsImplicit(&source, final_line),
-                  "missing late implicit newline flag") < 0) {
+                  "wrong implicit newline flags") < 0) {
         goto error;
     }
 
@@ -123,178 +57,21 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     if (check(view != NULL && view_len == 2 &&
                   memcmp(view, "\xce\xb2", 2) == 0,
               "wrong source span view") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start,
-                      _PYTOK_AFFINITY_LEFT, &loc) == 0,
-                  "cannot locate left line boundary") < 0 ||
-            check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
-                  "wrong left boundary location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate right line boundary") < 0 ||
-            check(loc.lineno == marker_line && loc.byte_col == 0,
-                  "wrong right boundary location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, marker_start + 1,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate late source byte") < 0 ||
-            check(loc.lineno == marker_line && loc.byte_col == 1,
-                  "wrong late source location") < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_SourceLocation(
-                  &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
-              "cannot locate left EOF") < 0 ||
-            check(loc.lineno == final_line && loc.byte_col == 2,
-                  "wrong left EOF location") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, source.len,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate right EOF") < 0 ||
-            check(loc.lineno == final_line + 1 && loc.byte_col == 0,
-                  "wrong right EOF location") < 0 ||
-            check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
-                  "cannot find virtual EOF line") < 0 ||
-            check(line.start == source.len && line.end == source.len,
-                  "wrong virtual EOF line") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
-                      !_PyTok_SourceLineIsImplicit(
-                          &source, final_line + 1),
-                  "virtual or invalid line is implicit") < 0) {
-        goto error;
-    }
-
-    view = _PyTok_SourceSpanView(
-        &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
-    if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
             check_system_error(
-                _PyTok_SourceLocation(
-                    &source, source.len + 1,
-                    _PYTOK_AFFINITY_RIGHT, &loc) < 0,
-                "accepted invalid source offset") < 0 ||
-            check_system_error(
-                _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
-                "accepted invalid source line") < 0) {
+                _PyTok_SourceSpanView(
+                    &source, _PyTok_SpanFromBounds(0, source.len + 1),
+                    &view_len) == NULL,
+                "accepted invalid source span") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    _PyTok_SourceInit(&source);
     if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
             check_system_error(
                 _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
-                "appended after unterminated source line") < 0 ||
-            check(_PyTok_SourceLocation(
-                      &source, source.len,
-                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
-                  "cannot locate unterminated EOF") < 0 ||
-            check(loc.lineno == 1 && loc.byte_col == 4,
-                  "wrong unterminated EOF location") < 0) {
-        goto error;
-    }
-
-    _PyTok_SourceClear(&source);
-    Py_RETURN_NONE;
-
-error:
-    _PyTok_SourceClear(&source);
-    return NULL;
-}
-
-static PyObject *
-test_tokenizer_cursor(PyObject *Py_UNUSED(module),
-                      PyObject *Py_UNUSED(args))
-{
-    _PyTok_SourceText source;
-    _PyTok_SourceInit(&source);
-    if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
-            _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
-        goto error;
-    }
-
-    _PyTok_Cursor cursor;
-    _PyTok_CursorInit(&cursor, &source);
-    if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
-            check(cursor.lineno == 3 && cursor.pos == source.len,
-                  "wrong cursor at virtual EOF") < 0 ||
-            _PyTok_CursorSetLine(&cursor, 1) < 0) {
-        goto error;
-    }
-
-    char large[BUFSIZ + 1];
-    memset(large, 'z', sizeof(large));
-    large[sizeof(large) - 1] = '\n';
-    if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
-        goto error;
-    }
-
-    if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
-              "wrong cursor peek after relocation") < 0 ||
-            check(_PyTok_CursorPeek(&cursor, 1) == 'b',
-                  "wrong distant cursor peek") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'a',
-                  "wrong first cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'b',
-                  "wrong second cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
-                  "wrong final cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == EOF,
-                  "cursor advanced past line") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
-                  "cannot seek cursor offset") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
-                  "wrong cursor byte after seek") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
-                  "cannot seek line boundary") < 0 ||
-            check(cursor.lineno == 2 && cursor.line_start == 3 &&
-                      _PyTok_CursorAdvance(&cursor) == 'c',
-                  "wrong cursor at line boundary") < 0 ||
-            check(_PyTok_CursorSetLine(&cursor, 3) == 0,
-                  "cannot advance cursor to final line") < 0 ||
-            check(cursor.line_start == 6 &&
-                      _PyTok_CursorAdvance(&cursor) == 'z',
-                  "wrong cursor byte on final line") < 0) {
-        goto error;
-    }
-
-    _PyTok_Cursor saved = cursor;
-    if (check_system_error(
-            _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
-            "accepted invalid cursor offset") < 0 ||
-            check(same_cursor(&cursor, &saved),
-                  "invalid offset changed cursor") < 0 ||
-            check_system_error(
-                _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
-                "accepted invalid cursor line") < 0 ||
-            check(same_cursor(&cursor, &saved),
-                  "invalid line changed cursor") < 0 ||
-            check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
-                  "cannot set cursor to EOF") < 0 ||
-            check(cursor.lineno == 4 && cursor.pos == source.len,
-                  "wrong cursor at EOF") < 0) {
-        goto error;
-    }
-
-#if SIZEOF_VOID_P > 4
-    char byte = 0;
-    _PyTok_SourceText huge_source = {
-        .bytes = &byte,
-        .len = (_PyTok_Off)INT_MAX + 1,
-    };
-    _PyTok_Cursor huge_cursor = {
-        .source = &huge_source,
-        .pos = INT_MAX,
-        .line_end = (_PyTok_Off)INT_MAX + 1,
-        .lineno = 1,
-    };
-    if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
-                  huge_cursor.pos == INT_MAX,
-              "cursor advanced past maximum column") < 0) {
+                "appended after unterminated source line") < 0) {
         goto error;
     }
-#endif
 
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
@@ -306,7 +83,6 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
 
 static PyMethodDef test_methods[] = {
     {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
-    {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
     {NULL},
 };
 
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 33647ec284061f..93dd56a8ef166f 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -423,7 +423,6 @@
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
     <ClInclude Include="..\Parser\lexer\buffer.h" />
-    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
@@ -593,7 +592,6 @@
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\lexer\buffer.c" />
-    <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 434dd13267fe93..f97b51a0e48a3e 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -333,9 +333,6 @@
     <ClInclude Include="..\Parser\lexer\buffer.h">
       <Filter>Parser</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\tokenizer\cursor.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1361,9 +1358,6 @@
     <ClCompile Include="..\Parser\lexer\buffer.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\tokenizer\cursor.c">
-      <Filter>Parser</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\source.c">
       <Filter>Parser</Filter>
     </ClCompile>
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index cd6885a7d01040..97f5ad1f2872bd 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -1,62 +1,32 @@
 #include "Python.h"
-#include "errcode.h"
-
+#include "buffer.h"
 #include "state.h"
 
-/* Traverse and remember all f-string buffers, in order to be able to restore
-   them after reallocating tok->buf */
 void
-_PyLexer_remember_fstring_buffers(struct tok_state *tok)
+_PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base,
+                        _PyLexer_BufferSnapshot *snapshot)
 {
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
-        mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
-    }
+    snapshot->buf = tok->buf - base;
+    snapshot->cur = tok->cur - tok->buf;
+    snapshot->inp = tok->inp - tok->buf;
+    snapshot->start = tok->start == NULL ? -1 : tok->start - tok->buf;
+    snapshot->line_start = tok->line_start == NULL
+        ? -1 : tok->line_start - tok->buf;
+    snapshot->multi_line_start = tok->multi_line_start == NULL
+        ? -1 : tok->multi_line_start - tok->buf;
 }
 
-/* Traverse and restore all f-string buffers after reallocating tok->buf */
 void
-_PyLexer_restore_fstring_buffers(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
-        mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
-    }
-}
-
-int
-_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
+_PyLexer_RestoreBuffer(struct tok_state *tok, char *base,
+                       const _PyLexer_BufferSnapshot *snapshot)
 {
-    Py_ssize_t cur = tok->cur - tok->buf;
-    Py_ssize_t oldsize = tok->inp - tok->buf;
-    Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
-    if (newsize > tok->end - tok->buf) {
-        char *newbuf = tok->buf;
-        Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
-        Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
-        Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
-        _PyLexer_remember_fstring_buffers(tok);
-        newbuf = (char *)PyMem_Realloc(newbuf, newsize);
-        if (newbuf == NULL) {
-            tok->done = E_NOMEM;
-            return 0;
-        }
-        tok->buf = newbuf;
-        tok->cur = tok->buf + cur;
-        tok->inp = tok->buf + oldsize;
-        tok->end = tok->buf + newsize;
-        tok->start = start < 0 ? NULL : tok->buf + start;
-        tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
-        tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
-        _PyLexer_restore_fstring_buffers(tok);
-    }
-    return 1;
+    tok->buf = base + snapshot->buf;
+    tok->cur = tok->buf + snapshot->cur;
+    tok->inp = tok->buf + snapshot->inp;
+    tok->start = snapshot->start < 0
+        ? NULL : tok->buf + snapshot->start;
+    tok->line_start = snapshot->line_start < 0
+        ? NULL : tok->buf + snapshot->line_start;
+    tok->multi_line_start = snapshot->multi_line_start < 0
+        ? NULL : tok->buf + snapshot->multi_line_start;
 }
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
index bb218162ff4845..06c54ff8da944a 100644
--- a/Parser/lexer/buffer.h
+++ b/Parser/lexer/buffer.h
@@ -3,8 +3,20 @@
 
 #include "pyport.h"
 
-void _PyLexer_remember_fstring_buffers(struct tok_state *tok);
-void _PyLexer_restore_fstring_buffers(struct tok_state *tok);
-int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size);
+struct tok_state;
+
+typedef struct {
+    Py_ssize_t buf;
+    Py_ssize_t cur;
+    Py_ssize_t inp;
+    Py_ssize_t start;
+    Py_ssize_t line_start;
+    Py_ssize_t multi_line_start;
+} _PyLexer_BufferSnapshot;
+
+void _PyLexer_SnapshotBuffer(
+    struct tok_state *, const char *, _PyLexer_BufferSnapshot *);
+void _PyLexer_RestoreBuffer(
+    struct tok_state *, char *, const _PyLexer_BufferSnapshot *);
 
 #endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a96362c8961023..79b4de007c6daf 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -12,8 +12,21 @@
 
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
-#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
-                _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
+
+static int
+type_comment_token_setup(struct tok_state *tok, struct token *token, int type,
+                         int col_offset, int end_col_offset,
+                         const char *start, const char *end)
+{
+    _PyLexer_token_setup(tok, token, type, start, end);
+    token->start_loc = (_PyTok_Loc){tok->lineno, col_offset};
+    token->end_loc = (_PyTok_Loc){tok->lineno, end_col_offset};
+    return type;
+}
+
+#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) \
+    type_comment_token_setup(tok, token, token_type, col_offset, \
+                             end_col_offset, p_start, p_end)
 
 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
    tokenizing. */
@@ -542,8 +555,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
          int cursor_in_format_with_debug =
              cursor == 1 && (current_tok->in_debug || in_format_spec);
          int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
-            return MAKE_TOKEN(ENDMARKER);
+        if (cursor_valid) {
+            _PyLexer_update_ftstring_expr(tok, c);
         }
         if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
             return MAKE_TOKEN(ERRORTOKEN);
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 1d97ac57b745b0..7a851c34af3d47 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -3,8 +3,36 @@
 
 #include "state.h"
 
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
+static inline const char *
+_PyLexer_SpanView(const struct tok_state *tok, _PyTok_Span span,
+                  Py_ssize_t *length)
+{
+    assert(length != NULL);
+    assert(_PyTok_SpanIsValid(span));
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    if (span.start >= tok->buf_offset &&
+            span.end - tok->buf_offset <= tok->inp - tok->buf) {
+        *length = span.end - span.start;
+        return tok->buf + (span.start - tok->buf_offset);
+    }
+    return _PyTok_SourceSpanView(&tok->source, span, length);
+}
+
+/* The view points into the current input window. The next
+   _PyTokenizer_Get() call may discard it. */
+static inline const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+                  Py_ssize_t *length)
+{
+    assert(length != NULL);
+    if (!_PyTok_SpanIsValid(token->span)) {
+        *length = 0;
+        return "";
+    }
+    return _PyLexer_SpanView(tok, token->span, length);
+}
+
 #endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index c6d3b9045c7292..01825bb0bae3be 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -46,6 +46,7 @@ TOK_NEXT_MODE(struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
+void _PyLexer_update_ftstring_expr(struct tok_state *, char);
 int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2a6408bef927a3..3702e38b8ff74c 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void)
     tok->interactive_src_start = NULL;
     tok->interactive_src_end = NULL;
     tok->start = NULL;
-    tok->end = NULL;
     tok->done = E_OK;
     tok->fp = NULL;
     tok->tabsize = TABSIZE;
@@ -61,24 +60,6 @@ _PyTokenizer_tok_new(void)
     return tok;
 }
 
-static void
-free_fstring_expressions(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        if (mode->last_expr_buffer != NULL) {
-            PyMem_Free(mode->last_expr_buffer);
-            mode->last_expr_buffer = NULL;
-            mode->last_expr_size = 0;
-            mode->last_expr_end = -1;
-            mode->in_format_spec = 0;
-        }
-    }
-}
-
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)
@@ -90,7 +71,6 @@ _PyTokenizer_Free(struct tok_state *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
-    free_fstring_expressions(tok);
     PyMem_Free(tok);
 }
 
@@ -101,41 +81,46 @@ _PyToken_Free(struct token *token) {
 
 void
 _PyToken_Init(struct token *token) {
+#ifdef Py_DEBUG
+    token->span = (_PyTok_Span){-1, -1};
+    token->start_loc = (_PyTok_Loc){-1, -1};
+    token->end_loc = (_PyTok_Loc){-1, -1};
+#endif
     token->metadata = NULL;
 }
 
-int
-_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end)
+static inline _PyTok_Span
+buffer_span(const struct tok_state *tok, const char *start, const char *end)
 {
-    token->level = tok->level;
-    token->lineno = token->end_lineno = tok->lineno;
-    token->col_offset = col_offset;
-    token->end_col_offset = end_col_offset;
-    token->start = start;
-    token->end = end;
-    return type;
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    const char *base = tok->buf;
+    assert(base != NULL);
+    assert(tok->inp >= base);
+    Py_ssize_t start_offset = start - base;
+    Py_ssize_t end_offset = end - base;
+    assert(start_offset >= 0 && start_offset <= end_offset);
+    assert(end_offset <= tok->inp - base);
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
+    return _PyTok_SpanFromBounds(
+        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
 }
 
 int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
-    assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
     token->level = tok->level;
-    if (ISSTRINGLIT(type)) {
-        token->lineno = tok->first_lineno;
-    }
-    else {
-        token->lineno = tok->lineno;
-    }
-    token->end_lineno = tok->lineno;
-    token->col_offset = token->end_col_offset = -1;
-    token->start = start;
-    token->end = end;
+    token->span = buffer_span(tok, start, end);
+    int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
+    token->start_loc = (_PyTok_Loc){lineno, -1};
+    token->end_loc = (_PyTok_Loc){tok->lineno, -1};
 
     if (start != NULL && end != NULL) {
-        token->col_offset = tok->starting_col_offset;
-        token->end_col_offset = tok->col_offset;
+        token->start_loc.byte_col = tok->starting_col_offset;
+        token->end_loc.byte_col = tok->col_offset;
     }
     return type;
 }
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0824785195491e..9a2c442898ba38 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -23,8 +23,9 @@ enum interactive_underflow_t {
 
 struct token {
     int level;
-    int lineno, col_offset, end_lineno, end_col_offset;
-    const char *start, *end;
+    _PyTok_Span span;
+    _PyTok_Loc start_loc;
+    _PyTok_Loc end_loc;
     PyObject *metadata;
 };
 
@@ -49,16 +50,11 @@ typedef struct _tokenizer_mode {
     char quote;
     int quote_size;
     int raw;
-    const char* start;
-    const char* multi_line_start;
+    _PyTok_Off start;
+    _PyTok_Off multi_line_start;
     int first_line;
 
-    Py_ssize_t start_offset;
-    Py_ssize_t multi_line_start_offset;
-
-    Py_ssize_t last_expr_size;
-    Py_ssize_t last_expr_end;
-    char* last_expr_buffer;
+    _PyTok_Span debug_expr;
     int in_debug;
     int in_format_spec;
 
@@ -67,15 +63,15 @@ typedef struct _tokenizer_mode {
 
 /* Tokenizer state */
 struct tok_state {
-    /* Input state; buf <= cur <= inp <= end */
+    /* Input state; buf <= cur <= inp */
     /* NB an entire line is held in the buffer */
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
+    char *buf;          /* Owned for file/readline input; source-backed otherwise. */
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
+    _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
     int fp_interactive; /* If the file descriptor is interactive */
     char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
     char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
-    const char *end;    /* End of input buffer if buf != NULL */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
@@ -128,8 +124,6 @@ struct tok_state {
 #endif
 };
 
-int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end);
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index d67c48f7f678ed..c2c2b50fd053ca 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -7,6 +7,73 @@
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
+static _PyTok_Off
+current_offset(const struct tok_state *tok, const char *position)
+{
+    assert(position >= tok->buf && position <= tok->inp);
+    return tok->buf_offset + (position - tok->buf);
+}
+
+static char *
+offset_pointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+    if (offset >= tok->buf_offset &&
+            offset - tok->buf_offset <= tok->inp - tok->buf) {
+        return tok->buf + (offset - tok->buf_offset);
+    }
+    assert(offset >= 0 && offset <= tok->source.len);
+    return tok->source.bytes + offset;
+}
+
+static Py_ssize_t
+strip_expr_comments(const char *expr, Py_ssize_t len, char *result)
+{
+    Py_ssize_t output = 0;
+    char quote = 0;
+    int quote_size = 0;
+
+    for (Py_ssize_t i = 0; i < len;) {
+        char c = expr[i];
+        if (quote != 0) {
+            if (c == '\\' && i + 1 < len) {
+                result[output] = c;
+                result[output + 1] = expr[i + 1];
+                output += 2;
+                i += 2;
+                continue;
+            }
+            if (c == quote) {
+                if (quote_size == 1) {
+                    quote = 0;
+                }
+                else if (i + 2 < len && expr[i + 1] == quote &&
+                         expr[i + 2] == quote) {
+                    memcpy(result + output, expr + i, 3);
+                    output += 3;
+                    i += 3;
+                    quote = 0;
+                    continue;
+                }
+            }
+        }
+        else if (c == '#') {
+            while (i < len && expr[i] != '\n') {
+                i++;
+            }
+            continue;
+        }
+        else if (c == '\'' || c == '"') {
+            quote = c;
+            quote_size = i + 2 < len && expr[i + 1] == c &&
+                         expr[i + 2] == c ? 3 : 1;
+        }
+        result[output] = c;
+        output++;
+        i++;
+    }
+    return output;
+}
+
 int
 _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     assert(token != NULL);
@@ -16,101 +83,26 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
         return 0;
     }
-    PyObject *res = NULL;
-
-    // Look for a # character outside of string literals
-    int hash_detected = 0;
-    int in_string = 0;
-    char quote_char = 0;
-
-    for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
-        char ch = tok_mode->last_expr_buffer[i];
-
-        // Skip escaped characters
-        if (ch == '\\') {
-            i++;
-            continue;
-        }
-
-        // Handle quotes
-        if (ch == '"' || ch == '\'') {
-            // The following if/else block works becase there is an off number
-            // of quotes in STRING tokens and the lexer only ever reaches this
-            // function with valid STRING tokens.
-            // For example: """hello"""
-            // First quote: in_string = 1
-            // Second quote: in_string = 0
-            // Third quote: in_string = 1
-            if (!in_string) {
-                in_string = 1;
-                quote_char = ch;
-            }
-            else if (ch == quote_char) {
-                in_string = 0;
-            }
-            continue;
-        }
-
-        // Check for # outside strings
-        if (ch == '#' && !in_string) {
-            hash_detected = 1;
-            break;
-        }
+    Py_ssize_t expr_len;
+    const char *expr = _PyLexer_SpanView(
+        tok, tok_mode->debug_expr, &expr_len);
+    if (expr == NULL) {
+        return -1;
+    }
+    PyObject *res;
+    if (memchr(expr, '#', expr_len) == NULL) {
+        res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
     }
-    // If we found a # character in the expression, we need to handle comments
-    if (hash_detected) {
-        // Allocate buffer for processed result
-        char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
-        if (!result) {
+    else {
+        char *stripped = PyMem_Malloc((size_t)expr_len);
+        if (stripped == NULL) {
+            PyErr_NoMemory();
             return -1;
         }
-
-        Py_ssize_t i = 0;  // Input position
-        Py_ssize_t j = 0;  // Output position
-        in_string = 0;     // Whether we're in a string
-        quote_char = 0;    // Current string quote char
-
-        // Process each character
-        while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-            char ch = tok_mode->last_expr_buffer[i];
-
-            // Handle string quotes
-            if (ch == '"' || ch == '\'') {
-                // See comment above to understand this part
-                if (!in_string) {
-                    in_string = 1;
-                    quote_char = ch;
-                } else if (ch == quote_char) {
-                    in_string = 0;
-                }
-                result[j++] = ch;
-            }
-            // Skip comments
-            else if (ch == '#' && !in_string) {
-                while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
-                       tok_mode->last_expr_buffer[i] != '\n') {
-                    i++;
-                }
-                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-                    result[j++] = '\n';
-                }
-            }
-            // Copy other chars
-            else {
-                result[j++] = ch;
-            }
-            i++;
-        }
-
-        result[j] = '\0';  // Null-terminate the result string
-        res = PyUnicode_DecodeUTF8(result, j, NULL);
-        PyMem_Free(result);
-    } else {
-        res = PyUnicode_DecodeUTF8(
-            tok_mode->last_expr_buffer,
-            tok_mode->last_expr_size - tok_mode->last_expr_end,
-            NULL
-        );
+        Py_ssize_t stripped_len = strip_expr_comments(
+            expr, expr_len, stripped);
+        res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
+        PyMem_Free(stripped);
     }
 
     if (!res) {
@@ -120,59 +112,28 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     return 0;
 }
 
-int
+void
 _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
 {
-    assert(tok->cur != NULL);
-
-    Py_ssize_t size = strlen(tok->cur);
     tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
 
     switch (cur) {
-       case 0:
-            if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
-                return 1;
-            }
-            char *new_buffer = PyMem_Realloc(
-                tok_mode->last_expr_buffer,
-                tok_mode->last_expr_size + size
-            );
-            if (new_buffer == NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-                goto error;
-            }
-            tok_mode->last_expr_buffer = new_buffer;
-            strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
-            tok_mode->last_expr_size += size;
-            break;
         case '{':
-            if (tok_mode->last_expr_buffer != NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-            }
-            tok_mode->last_expr_buffer = PyMem_Malloc(size);
-            if (tok_mode->last_expr_buffer == NULL) {
-                goto error;
-            }
-            tok_mode->last_expr_size = size;
-            tok_mode->last_expr_end = -1;
-            strncpy(tok_mode->last_expr_buffer, tok->cur, size);
+            tok_mode->debug_expr = (_PyTok_Span){
+                current_offset(tok, tok->cur), -1};
             break;
         case '}':
         case '!':
-            tok_mode->last_expr_end = strlen(tok->start);
+            tok_mode->debug_expr.end = current_offset(tok, tok->start);
             break;
         case ':':
-            if (tok_mode->last_expr_end == -1) {
-               tok_mode->last_expr_end = strlen(tok->start);
+            if (tok_mode->debug_expr.end < 0) {
+                tok_mode->debug_expr.end = current_offset(tok, tok->start);
             }
             break;
         default:
             Py_UNREACHABLE();
     }
-    return 1;
-error:
-    tok->done = E_NOMEM;
-    return 0;
 }
 
 int
@@ -263,14 +224,10 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
     the_current_tok->kind = TOK_FSTRING_MODE;
     the_current_tok->quote = quote;
     the_current_tok->quote_size = quote_size;
-    the_current_tok->start = tok->start;
-    the_current_tok->multi_line_start = tok->line_start;
+    the_current_tok->start = current_offset(tok, tok->start);
+    the_current_tok->multi_line_start = current_offset(tok, tok->line_start);
     the_current_tok->first_line = tok->lineno;
-    the_current_tok->start_offset = -1;
-    the_current_tok->multi_line_start_offset = -1;
-    the_current_tok->last_expr_buffer = NULL;
-    the_current_tok->last_expr_size = 0;
-    the_current_tok->last_expr_end = -1;
+    the_current_tok->debug_expr = (_PyTok_Span){-1, -1};
     the_current_tok->in_format_spec = 0;
     the_current_tok->in_debug = 0;
 
@@ -460,13 +417,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
     }
 
-    if (current_tok->last_expr_buffer != NULL) {
-        PyMem_Free(current_tok->last_expr_buffer);
-        current_tok->last_expr_buffer = NULL;
-        current_tok->last_expr_size = 0;
-        current_tok->last_expr_end = -1;
-    }
-
     p_start = tok->start;
     p_end = tok->cur;
     tok->tok_mode_stack_index--;
@@ -518,9 +468,9 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
             // shift the tok_state's location into
             // the start of string, and report the error
             // from the initial quote character
-            tok->cur = (char *)current_tok->start;
-            tok->cur++;
-            tok->line_start = current_tok->multi_line_start;
+            tok->cur = offset_pointer(tok, current_tok->start) + 1;
+            tok->line_start = offset_pointer(
+                tok, current_tok->multi_line_start);
             int start = tok->lineno;
 
             tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
@@ -551,9 +501,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
 
         if (c == '{') {
-            if (!_PyLexer_update_ftstring_expr(tok, c)) {
-                return MAKE_TOKEN(ENDMARKER);
-            }
+            _PyLexer_update_ftstring_expr(tok, c);
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
diff --git a/Parser/pegen.c b/Parser/pegen.c
index fcec810037e98d..d86dd22444e6a7 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) {
 }
 
 static int
-_get_keyword_or_name_type(Parser *p, struct token *new_token)
+_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length)
 {
-    Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset;
-    assert(name_len > 0);
+    assert(length > 0);
 
-    if (name_len >= p->n_keyword_lists ||
-        p->keywords[name_len] == NULL ||
-        p->keywords[name_len]->type == -1) {
+    if (length >= p->n_keyword_lists ||
+        p->keywords[length] == NULL ||
+        p->keywords[length]->type == -1) {
         return NAME;
     }
-    for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
-        if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) {
+    for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) {
+        if (memcmp(k->str, text, (size_t)length) == 0) {
             return k->type;
         }
     }
@@ -193,8 +192,11 @@ static int
 initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
     assert(parser_token != NULL);
 
-    parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
-    parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
+    Py_ssize_t length;
+    const char *text = _PyToken_TextView(p->tok, new_token, &length);
+    parser_token->type = token_type == NAME
+        ? _get_keyword_or_name_type(p, text, length) : token_type;
+    parser_token->bytes = PyBytes_FromStringAndSize(text, length);
     if (parser_token->bytes == NULL) {
         return -1;
     }
@@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
     }
 
     parser_token->level = new_token->level;
-    parser_token->lineno = new_token->lineno;
-    parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
-                                                                    : new_token->col_offset;
-    parser_token->end_lineno = new_token->end_lineno;
-    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset
-                                                                 : new_token->end_col_offset;
+    parser_token->lineno = new_token->start_loc.lineno;
+    parser_token->col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->start_loc.byte_col
+        : new_token->start_loc.byte_col;
+    parser_token->end_lineno = new_token->end_loc.lineno;
+    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->end_loc.byte_col
+        : new_token->end_loc.byte_col;
 
     p->fill += 1;
 
@@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p)
 
     // Record and skip '# type: ignore' comments
     while (type == TYPE_IGNORE) {
-        Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;
+        Py_ssize_t len;
+        const char *text = _PyToken_TextView(p->tok, &new_token, &len);
         char *tag = PyMem_Malloc((size_t)len + 1);
         if (tag == NULL) {
             PyErr_NoMemory();
             goto error;
         }
-        strncpy(tag, new_token.start, (size_t)len);
+        memcpy(tag, text, (size_t)len);
         tag[len] = '\0';
         // Ownership of tag passes to the growable array
         if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
deleted file mode 100644
index 698a26a740fd24..00000000000000
--- a/Parser/tokenizer/cursor.c
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "Python.h"
-
-#include "cursor.h"
-
-static void
-set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start,
-         _PyTok_Off end)
-{
-    cursor->pos = start;
-    cursor->line_start = start;
-    cursor->line_end = end;
-    cursor->lineno = lineno;
-}
-
-int
-_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
-{
-    if (cursor->source == NULL) {
-        PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
-        return -1;
-    }
-    const _PyTok_SourceText *source = cursor->source;
-    if (lineno > 0 && cursor->lineno == lineno - 1 &&
-            lineno <= source->nlines) {
-        _PyTok_Off start = cursor->line_end;
-        _PyTok_Off end = source->len;
-        if (lineno < source->nlines) {
-            end = _PyTok_SourceFindLineEnd(source, start);
-            if (end < 0) {
-                return -1;
-            }
-        }
-        set_line(cursor, lineno, start, end);
-        return 0;
-    }
-
-    _PyTok_Line line;
-    if (_PyTok_SourceLine(source, lineno, &line) < 0) {
-        return -1;
-    }
-    set_line(cursor, lineno, line.start, line.end);
-    return 0;
-}
-
-int
-_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
-{
-    if (cursor->source == NULL) {
-        PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
-        return -1;
-    }
-    const _PyTok_SourceText *source = cursor->source;
-    int stays_on_line = cursor->lineno > 0 &&
-        offset >= cursor->line_start && offset < cursor->line_end;
-    if (!stays_on_line && cursor->lineno > 0 &&
-            offset == cursor->line_end && offset == source->len &&
-            (offset == 0 || source->bytes[offset - 1] != '\n')) {
-        stays_on_line = 1;
-    }
-    if (stays_on_line) {
-        cursor->pos = offset;
-        return 0;
-    }
-
-    _PyTok_Loc loc;
-    if (_PyTok_SourceLocation(
-            source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) {
-        return -1;
-    }
-    _PyTok_Off start = offset - loc.byte_col;
-    _PyTok_Off end = source->len;
-    if (loc.lineno < source->nlines) {
-        end = _PyTok_SourceFindLineEnd(source, start);
-        if (end < 0) {
-            return -1;
-        }
-    }
-    set_line(cursor, loc.lineno, start, end);
-    cursor->pos = offset;
-    return 0;
-}
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
deleted file mode 100644
index d0fd9cf80b77b7..00000000000000
--- a/Parser/tokenizer/cursor.h
+++ /dev/null
@@ -1,65 +0,0 @@
-#ifndef Py_TOKENIZER_CURSOR_H
-#define Py_TOKENIZER_CURSOR_H
-
-#include "source.h"
-
-typedef struct {
-    /* The source must remain initialized at this address while in use. */
-    const _PyTok_SourceText *source;
-    _PyTok_Off pos;
-    _PyTok_Off line_start;
-    _PyTok_Off line_end;
-    int lineno;
-} _PyTok_Cursor;
-
-/* Move to the start of a 1-based line. Both setters preserve the cursor on
-   error. */
-PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
-/* Move to an offset. A line boundary selects the following line. */
-PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
-
-static inline void
-_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
-{
-    *cursor = (_PyTok_Cursor){
-        .source = source,
-    };
-}
-
-/* Read one byte from the current line, including its terminating newline.
-   EOF marks the line boundary, not necessarily the end of the source. It is
-   also returned if advancing would make the byte column unrepresentable. */
-static inline int
-_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
-{
-    assert(cursor->source != NULL);
-    assert(cursor->pos >= cursor->line_start);
-    assert(cursor->pos <= cursor->line_end);
-    assert(cursor->line_end <= cursor->source->len);
-    if (cursor->pos >= cursor->line_end) {
-        return EOF;
-    }
-    if (cursor->pos - cursor->line_start >= INT_MAX) {
-        return EOF;
-    }
-    return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
-}
-
-/* Return the byte at a nonnegative distance within the current line, or EOF
-   if the distance reaches or crosses the line boundary. */
-static inline int
-_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
-{
-    assert(cursor->source != NULL);
-    assert(cursor->pos >= cursor->line_start);
-    assert(cursor->pos <= cursor->line_end);
-    assert(cursor->line_end <= cursor->source->len);
-    assert(distance >= 0);
-    if (distance < 0 ||
-            distance >= cursor->line_end - cursor->pos) {
-        return EOF;
-    }
-    return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);
-}
-
-#endif
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 82b824f56374fc..f93090dd30e343 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -13,6 +13,12 @@
 #  include <unistd.h>
 #endif
 
+static inline int
+reader_is_streaming(_PyTok_ReaderKind kind)
+{
+    return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE;
+}
+
 void
 _PyTok_ReaderFree(struct tok_state *tok)
 {
@@ -28,14 +34,27 @@ _PyTok_ReaderFree(struct tok_state *tok)
     }
     PyMem_Free(reader->file_buffer);
     PyMem_Free(reader->decoded);
-    if (reader->kind != _PYTOK_READER_PREPARED) {
+    if (reader_is_streaming(reader->kind)) {
         PyMem_Free(tok->buf);
-        tok->buf = NULL;
     }
+    tok->buf = NULL;
     PyMem_Free(reader);
     tok->reader = NULL;
 }
 
+static int
+resize_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t new_capacity)
+{
+    char *resized = PyMem_Realloc(*buffer, new_capacity);
+    if (resized == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    *buffer = resized;
+    *capacity = new_capacity;
+    return 0;
+}
+
 static int
 reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
 {
@@ -50,13 +69,31 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
         }
         cap *= 2;
     }
-    char *resized = PyMem_Realloc(*buffer, cap);
-    if (resized == NULL) {
-        PyErr_NoMemory();
+    return resize_buffer(buffer, capacity, cap);
+}
+
+static int
+reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
+{
+    _PyTok_Reader *reader = tok->reader;
+    if (needed <= reader->input_buffer_cap) {
+        return 0;
+    }
+    assert(tok->buf != NULL);
+    assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+    assert(tok->inp - tok->buf <= reader->input_buffer_cap);
+    Py_ssize_t used = tok->inp - tok->buf;
+    Py_ssize_t growth = used >> 1;
+    Py_ssize_t capacity = used > PY_SSIZE_T_MAX - growth
+        ? needed : Py_MAX(needed, used + growth);
+    _PyLexer_BufferSnapshot snapshot;
+    _PyLexer_SnapshotBuffer(tok, tok->buf, &snapshot);
+    char *buffer = tok->buf;
+    if (resize_buffer(
+            &buffer, &reader->input_buffer_cap, capacity) < 0) {
         return -1;
     }
-    *buffer = resized;
-    *capacity = cap;
+    _PyLexer_RestoreBuffer(tok, buffer, &snapshot);
     return 0;
 }
 
@@ -529,19 +566,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
     Py_UNREACHABLE();
 }
 
+static void
+reset_streaming_buffer(struct tok_state *tok)
+{
+    assert(tok->buf != NULL);
+    assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+    Py_ssize_t consumed = tok->inp - tok->buf;
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed);
+    tok->buf_offset += consumed;
+    tok->cur = tok->inp = tok->buf;
+}
+
 int
 _PyTok_ReaderUnderflow(struct tok_state *tok)
 {
-    int prepared = tok->reader->kind == _PYTOK_READER_PREPARED;
+    _PyTok_ReaderKind kind = tok->reader->kind;
+    int prepared = kind == _PYTOK_READER_PREPARED;
+    int streaming = reader_is_streaming(kind);
     int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
 
-    if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
-    }
-
     _PyTok_Chunk chunk;
     _PyTok_ReadResult result = reader_next(tok, &chunk);
     if (result != _PYTOK_READ_LINE) {
+        if (reset_buffer && streaming) {
+            reset_streaming_buffer(tok);
+        }
         if (result == _PYTOK_READ_EOF) {
             tok->done = E_EOF;
         }
@@ -558,34 +607,68 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
                     ? E_NOMEM : E_ERROR;
             }
         }
-        if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+        if (kind == _PYTOK_READER_INTERACTIVE &&
                 result != _PYTOK_READ_STOPPED) {
             PySys_WriteStderr("\n");
         }
         return 0;
     }
 
-    Py_ssize_t copy_len = chunk.len;
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+    Py_ssize_t scan_len = chunk.len;
+    if (kind == _PYTOK_READER_INTERACTIVE &&
             chunk.implicit_newline) {
-        copy_len--;
+        scan_len--;
     }
-    if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
-    }
-    if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
+    if (streaming) {
+        if (reset_buffer) {
+            reset_streaming_buffer(tok);
+        }
+        Py_ssize_t used = tok->inp - tok->buf;
+        int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 ||
+                       tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len;
+        if (overflow) {
+            PyErr_NoMemory();
+        }
+        if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) {
+            _PyTok_ChunkClear(&chunk);
+            tok->done = E_NOMEM;
+            tok->input_error = 1;
+            return 0;
+        }
+        memcpy(tok->inp, chunk.data, (size_t)scan_len);
+        tok->inp += scan_len;
+        *tok->inp = '\0';
     }
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
-            _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len,
-                                    chunk.implicit_newline) < 0) {
-        _PyTok_ChunkClear(&chunk);
-        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
-            ? E_NOMEM : E_ERROR;
-        tok->input_error = 1;
-        return 0;
+    else if (!prepared) {
+        int source_will_grow =
+            chunk.len > tok->source.cap - tok->source.len - 1;
+        _PyLexer_BufferSnapshot snapshot;
+        if (!reset_buffer && source_will_grow) {
+            _PyLexer_SnapshotBuffer(
+                tok, tok->source.bytes, &snapshot);
+        }
+        _PyTok_Off source_start = _PyTok_SourceAppendLine(
+            &tok->source, chunk.data, chunk.len,
+            chunk.implicit_newline);
+        if (source_start < 0) {
+            _PyTok_ChunkClear(&chunk);
+            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
+                ? E_NOMEM : E_ERROR;
+            tok->input_error = 1;
+            return 0;
+        }
+        if (reset_buffer) {
+            tok->buf = tok->cur = tok->source.bytes + source_start;
+            tok->buf_offset = source_start;
+            tok->line_start = tok->buf;
+            tok->start = NULL;
+            tok->multi_line_start = NULL;
+        }
+        else if (source_will_grow) {
+            _PyLexer_RestoreBuffer(
+                tok, tok->source.bytes, &snapshot);
+        }
+        tok->inp = tok->source.bytes + source_start + scan_len;
     }
     if (tok->fp_interactive) {
         tok->interactive_src_start = tok->source.bytes;
@@ -594,24 +677,14 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     if (prepared) {
         if (tok->start == NULL) {
             tok->buf = tok->cur;
+            tok->buf_offset = chunk.data - tok->source.bytes;
         }
         tok->inp = chunk.data + chunk.len;
     }
-    else {
-        memcpy(tok->inp, chunk.data, (size_t)copy_len);
-        tok->inp += copy_len;
-        *tok->inp = '\0';
-    }
     tok->implicit_newline = chunk.implicit_newline;
 
-    if (!prepared && tok->tok_mode_stack_index &&
-            !_PyLexer_update_ftstring_expr(tok, 0)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
-    }
     ADVANCE_LINENO();
-    if (tok->reader->kind == _PYTOK_READER_FILE &&
+    if (kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
             !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
         _PyTok_ChunkClear(&chunk);
@@ -639,14 +712,15 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
     if (kind == _PYTOK_READER_PREPARED) {
         return tok;
     }
-    tok->buf = PyMem_Malloc(BUFSIZ);
-    if (tok->buf == NULL) {
-        PyErr_NoMemory();
-        _PyTokenizer_Free(tok);
-        return NULL;
+    if (reader_is_streaming(kind)) {
+        if (reserve_buffer(
+                &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) {
+            _PyTokenizer_Free(tok);
+            return NULL;
+        }
+        tok->cur = tok->inp = tok->buf;
+        tok->buf[0] = '\0';
     }
-    tok->cur = tok->inp = tok->buf;
-    tok->end = tok->buf + BUFSIZ;
     return tok;
 }
 
@@ -664,7 +738,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
         return NULL;
     }
     tok->buf = tok->cur = tok->inp = tok->str;
-    tok->end = tok->buf;
     return tok;
 }
 
diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h
index 121d0f96f6698a..49a6f04ec60af2 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -44,6 +44,8 @@ typedef struct _PyTok_Reader {
     PyObject *decoder;
     const char *nextprompt;
 
+    Py_ssize_t input_buffer_cap;
+
     char *file_buffer;
     Py_ssize_t file_buffer_cap;
     _PyTok_Chunk prefetched_lines[2];
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
index c0f7925e33f8b9..e876be6e002651 100644
--- a/Parser/tokenizer/source.c
+++ b/Parser/tokenizer/source.c
@@ -2,8 +2,6 @@
 
 #include "source.h"
 
-#define LINE_CHECKPOINT_INTERVAL 256
-
 void
 _PyTok_SourceInit(_PyTok_SourceText *source)
 {
@@ -14,7 +12,6 @@ void
 _PyTok_SourceClear(_PyTok_SourceText *source)
 {
     PyMem_Free(source->bytes);
-    PyMem_Free(source->line_checkpoints);
     PyMem_Free(source->implicit_lines);
     _PyTok_SourceInit(source);
 }
@@ -58,34 +55,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
     return 0;
 }
 
-static int
-reserve_checkpoints(_PyTok_SourceText *source, int needed)
-{
-    if (needed <= source->checkpoints_cap) {
-        return 0;
-    }
-    int cap;
-    if (source->checkpoints_cap == 0) {
-        cap = 16;
-    }
-    else if (source->checkpoints_cap <= INT_MAX / 2) {
-        cap = source->checkpoints_cap * 2;
-    }
-    else {
-        PyErr_NoMemory();
-        return -1;
-    }
-    _PyTok_Off *checkpoints = source->line_checkpoints;
-    PyMem_Resize(checkpoints, _PyTok_Off, cap);
-    if (checkpoints == NULL) {
-        PyErr_NoMemory();
-        return -1;
-    }
-    source->line_checkpoints = checkpoints;
-    source->checkpoints_cap = cap;
-    return 0;
-}
-
 static int
 reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
 {
@@ -148,11 +117,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
         return -1;
     }
     int nlines = source->nlines + 1;
-    int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0;
-    int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
-    if ((checkpoint &&
-         reserve_checkpoints(source, checkpoint_count) < 0) ||
-            (implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
+    if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
             reserve_bytes(source, source->len + len + 1) < 0) {
         return -1;
     }
@@ -161,9 +126,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
     memcpy(source->bytes + start, bytes, len);
     source->len += len;
     source->bytes[source->len] = '\0';
-    if (checkpoint) {
-        source->line_checkpoints[checkpoint_count - 1] = start;
-    }
     if (implicit_newline) {
         source->implicit_lines[(nlines - 1) / 8] |=
             (unsigned char)(1U << ((nlines - 1) & 7));
@@ -194,121 +156,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
     return (source->implicit_lines[(lineno - 1) / 8] >>
             ((lineno - 1) & 7)) & 1;
 }
-
-static int
-source_ends_in_newline(const _PyTok_SourceText *source)
-{
-    return source->len > 0 && source->bytes[source->len - 1] == '\n';
-}
-
-static int
-eof_lineno(const _PyTok_SourceText *source)
-{
-    if (source->nlines == 0) {
-        return 1;
-    }
-    return source->nlines + source_ends_in_newline(source);
-}
-
-int
-_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
-                  _PyTok_Line *line)
-{
-    if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) {
-        PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
-        return -1;
-    }
-    if (lineno > source->nlines) {
-        *line = (_PyTok_Line){
-            .start = source->len,
-            .end = source->len,
-        };
-        return 0;
-    }
-
-    int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL;
-    int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
-    _PyTok_Off start = source->line_checkpoints[checkpoint];
-    while (current < lineno) {
-        start = _PyTok_SourceFindLineEnd(source, start);
-        if (start < 0) {
-            return -1;
-        }
-        current++;
-    }
-    _PyTok_Off end = source->len;
-    if (lineno < source->nlines) {
-        end = _PyTok_SourceFindLineEnd(source, start);
-        if (end < 0) {
-            return -1;
-        }
-    }
-    *line = (_PyTok_Line){
-        .start = start,
-        .end = end,
-        .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
-        .contains_nul = memchr(
-            source->bytes + start, 0, end - start) != NULL,
-    };
-    return 0;
-}
-
-int
-_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
-                      _PyTok_Affinity affinity, _PyTok_Loc *loc)
-{
-    if (offset < 0 || offset > source->len || loc == NULL ||
-            (affinity != _PYTOK_AFFINITY_LEFT &&
-             affinity != _PYTOK_AFFINITY_RIGHT)) {
-        PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
-        return -1;
-    }
-    if (source->nlines == 0 ||
-            (offset == source->len && source_ends_in_newline(source) &&
-             affinity == _PYTOK_AFFINITY_RIGHT)) {
-        *loc = (_PyTok_Loc){eof_lineno(source), 0};
-        return 0;
-    }
-
-    _PyTok_Off key = offset;
-    if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) {
-        key--;
-    }
-    int low = 0;
-    int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
-    while (low < high) {
-        int middle = low + (high - low) / 2;
-        if (source->line_checkpoints[middle] <= key) {
-            low = middle + 1;
-        }
-        else {
-            high = middle;
-        }
-    }
-    int checkpoint = low - 1;
-    if (checkpoint < 0) {
-        PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index");
-        return -1;
-    }
-    int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
-    _PyTok_Off start = source->line_checkpoints[checkpoint];
-    while (lineno < source->nlines) {
-        _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start);
-        if (end < 0) {
-            return -1;
-        }
-        if (offset < end ||
-                (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
-            break;
-        }
-        start = end;
-        lineno++;
-    }
-    _PyTok_Off byte_col = offset - start;
-    if (byte_col > INT_MAX) {
-        PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large");
-        return -1;
-    }
-    *loc = (_PyTok_Loc){lineno, (int)byte_col};
-    return 0;
-}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index b42ecda1b31aa5..fac17183ceb734 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -5,7 +5,8 @@
 
 typedef Py_ssize_t _PyTok_Off;
 
-/* Half-open byte offsets into a _PyTok_SourceText. */
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+   storage may retain only the current input window. */
 typedef struct {
     _PyTok_Off start;
     _PyTok_Off end;
@@ -17,32 +18,17 @@ typedef struct {
     int byte_col;
 } _PyTok_Loc;
 
-typedef enum {
-    _PYTOK_AFFINITY_LEFT,
-    _PYTOK_AFFINITY_RIGHT,
-} _PyTok_Affinity;
-
-/* The half-open range includes the terminating newline when present. */
-typedef struct {
-    _PyTok_Off start;
-    _PyTok_Off end;
-    unsigned implicit_newline : 1;
-    unsigned contains_nul : 1;
-} _PyTok_Line;
-
 typedef struct {
     char *bytes;
     _PyTok_Off len;
     _PyTok_Off cap;
-    _PyTok_Off *line_checkpoints;
     unsigned char *implicit_lines;
     int nlines;
-    int checkpoints_cap;
     Py_ssize_t implicit_cap;
 } _PyTok_SourceText;
 
 PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
-/* Clear invalidates all cursors, spans, and views for the source. */
+/* Clear invalidates all spans and views for the source. */
 PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
 /* Append one nonempty logical line and return its start offset. The input may
    contain one newline, as its final byte. An unterminated line must be the
@@ -54,17 +40,9 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
 /* The returned view is invalidated by SourceAppendLine and SourceClear. */
 PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
     const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
-/* Look up a 1-based line. Empty and newline-terminated sources have an empty
-   virtual line at EOF. */
-PyAPI_FUNC(int) _PyTok_SourceLine(
-    const _PyTok_SourceText *, int, _PyTok_Line *);
 /* Return false for invalid line numbers and the virtual EOF line. */
 PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
     const _PyTok_SourceText *, int);
-/* At a line boundary, left affinity selects the preceding line at its end;
-   right affinity selects the following line at byte column zero. */
-PyAPI_FUNC(int) _PyTok_SourceLocation(
-    const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
 
 static inline _PyTok_Span
 _PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
@@ -78,22 +56,4 @@ _PyTok_SpanIsValid(_PyTok_Span span)
     return span.start >= 0 && span.end >= span.start;
 }
 
-static inline _PyTok_Off
-_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
-{
-    if (source->bytes == NULL || start < 0 || start >= source->len) {
-        PyErr_SetString(PyExc_SystemError,
-                        "corrupt tokenizer source line index");
-        return -1;
-    }
-    const char *newline = memchr(
-        source->bytes + start, '\n', source->len - start);
-    if (newline == NULL) {
-        PyErr_SetString(PyExc_SystemError,
-                        "corrupt tokenizer source line index");
-        return -1;
-    }
-    return newline - source->bytes + 1;
-}
-
 #endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 762b7b3e4c8d71..eee1fc86ded781 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -203,14 +203,15 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
 }
 
 static void
-_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start,
-                 PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
+_get_col_offsets(tokenizeriterobject *it, const char *token_start,
+                 const char *token_end, const char *line_start, PyObject *line,
+                 int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
                  Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
 {
     _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
     Py_ssize_t byte_offset = -1;
-    if (token.start != NULL && token.start >= line_start) {
-        byte_offset = token.start - line_start;
+    if (token_start != NULL && token_start >= line_start) {
+        byte_offset = token_start - line_start;
         if (line_changed) {
             *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
             it->byte_col_offset_diff = byte_offset - *col_offset;
@@ -220,15 +221,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
         }
     }
 
-    if (token.end != NULL && token.end >= it->tok->line_start) {
-        Py_ssize_t end_byte_offset = token.end - it->tok->line_start;
+    if (token_end != NULL && token_end >= it->tok->line_start) {
+        Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
         if (lineno == end_lineno) {
-            // If the whole token is at the same line, we can just use the token.start
-            // buffer for figuring out the new column offset, since using line is not
-            // performant for very long lines.
+            // Avoid rescanning the prefix of a very long line.
             Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
             *end_col_offset = *col_offset + token_col_offset;
-            it->byte_col_offset_diff += token.end - token.start - token_col_offset;
+            it->byte_col_offset_diff += token_end - token_start - token_col_offset;
         }
         else {
             *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
@@ -263,12 +262,18 @@ tokenizeriter_next(PyObject *op)
         it->done = 1;
         goto exit;
     }
-    PyObject *str = NULL;
-    if (token.start == NULL || token.end == NULL) {
+    const char *token_start = NULL;
+    const char *token_end = NULL;
+    PyObject *str;
+    if (!_PyTok_SpanIsValid(token.span)) {
         str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     }
     else {
-        str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
+        Py_ssize_t token_length;
+        token_start = _PyToken_TextView(
+            it->tok, &token, &token_length);
+        token_end = token_start + token_length;
+        str = PyUnicode_FromStringAndSize(token_start, token_length);
     }
     if (str == NULL) {
         goto exit;
@@ -297,11 +302,11 @@ tokenizeriter_next(PyObject *op)
         goto exit;
     }
 
-    Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
-    Py_ssize_t end_lineno = it->tok->lineno;
+    Py_ssize_t lineno = token.start_loc.lineno;
+    Py_ssize_t end_lineno = token.end_loc.lineno;
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;
-    _get_col_offsets(it, token, line_start, line, line_changed,
+    _get_col_offsets(it, token_start, token_end, line_start, line, line_changed,
                      lineno, end_lineno, &col_offset, &end_col_offset);
 
     if (it->tok->tok_extra_tokens) {
@@ -317,7 +322,8 @@ tokenizeriter_next(PyObject *op)
         else if (type == NEWLINE) {
             Py_DECREF(str);
             if (!it->tok->implicit_newline) {
-                if (it->tok->start[0] == '\r') {
+                assert(token_start != NULL);
+                if (token_start[0] == '\r') {
                     str = PyUnicode_FromString("\r\n");
                 } else {
                     str = PyUnicode_FromString("\n");