diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 7e02191db86be5..0b5bdcdeda3c19 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_buffer_reallocation(self):
+        padding = " " * 9000
+        expression_line = ")=:>{2}}\n"
+        physical_lines = [
+            'f"""\n',
+            "{(\n",
+            padding + "1\n",
+            expression_line,
+            '"""\n',
+        ]
+        source = "".join(physical_lines)
+        chunks = iter([
+            "".join(physical_lines[:2]),
+            "".join(physical_lines[2:4]),
+            physical_lines[4],
+            "",
+        ])
+
+        expected = self._get_tokens(source, extra_tokens=True)
+        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+            chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(tokens, expected)
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index cd6885a7d01040..4bee1f85d47836 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -1,62 +1,45 @@
 #include "Python.h"
-#include "errcode.h"
-
+#include "buffer.h"
 #include "state.h"
 
-/* Traverse and remember all f-string buffers, in order to be able to restore
-   them after reallocating tok->buf */
 void
-_PyLexer_remember_fstring_buffers(struct tok_state *tok)
+_PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base,
+                        _PyLexer_BufferSnapshot *snapshot)
 {
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
+    snapshot->buf = tok->buf - base;
+    snapshot->cur = tok->cur - tok->buf;
+    snapshot->inp = tok->inp - tok->buf;
+    snapshot->start = tok->start == NULL ? -1 : tok->start - tok->buf;
+    snapshot->line_start = tok->line_start == NULL
+        ? -1 : tok->line_start - tok->buf;
+    snapshot->multi_line_start = tok->multi_line_start == NULL
+        ? -1 : tok->multi_line_start - tok->buf;
+    for (int index = tok->tok_mode_stack_index; index >= 0; --index) {
+        tokenizer_mode *mode = &tok->tok_mode_stack[index];
         mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
-        mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
+        mode->multi_line_start_offset = mode->multi_line_start == NULL
+            ? -1 : mode->multi_line_start - tok->buf;
     }
 }
 
-/* Traverse and restore all f-string buffers after reallocating tok->buf */
 void
-_PyLexer_restore_fstring_buffers(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
-        mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
-    }
-}
-
-int
-_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
+_PyLexer_RestoreBuffer(struct tok_state *tok, char *base,
+                       const _PyLexer_BufferSnapshot *snapshot)
 {
-    Py_ssize_t cur = tok->cur - tok->buf;
-    Py_ssize_t oldsize = tok->inp - tok->buf;
-    Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
-    if (newsize > tok->end - tok->buf) {
-        char *newbuf = tok->buf;
-        Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
-        Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
-        Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
-        _PyLexer_remember_fstring_buffers(tok);
-        newbuf = (char *)PyMem_Realloc(newbuf, newsize);
-        if (newbuf == NULL) {
-            tok->done = E_NOMEM;
-            return 0;
-        }
-        tok->buf = newbuf;
-        tok->cur = tok->buf + cur;
-        tok->inp = tok->buf + oldsize;
-        tok->end = tok->buf + newsize;
-        tok->start = start < 0 ? NULL : tok->buf + start;
-        tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
-        tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
-        _PyLexer_restore_fstring_buffers(tok);
+    tok->buf = base + snapshot->buf;
+    tok->cur = tok->buf + snapshot->cur;
+    tok->inp = tok->buf + snapshot->inp;
+    tok->start = snapshot->start < 0
+        ? NULL : tok->buf + snapshot->start;
+    tok->line_start = snapshot->line_start < 0
+        ? NULL : tok->buf + snapshot->line_start;
+    tok->multi_line_start = snapshot->multi_line_start < 0
+        ? NULL : tok->buf + snapshot->multi_line_start;
+    for (int index = tok->tok_mode_stack_index; index >= 0; --index) {
+        tokenizer_mode *mode = &tok->tok_mode_stack[index];
+        mode->start = mode->start_offset < 0
+            ? NULL : tok->buf + mode->start_offset;
+        mode->multi_line_start = mode->multi_line_start_offset < 0
+            ? NULL : tok->buf + mode->multi_line_start_offset;
     }
-    return 1;
 }
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
index bb218162ff4845..06c54ff8da944a 100644
--- a/Parser/lexer/buffer.h
+++ b/Parser/lexer/buffer.h
@@ -3,8 +3,20 @@
 
 #include "pyport.h"
 
-void _PyLexer_remember_fstring_buffers(struct tok_state *tok);
-void _PyLexer_restore_fstring_buffers(struct tok_state *tok);
-int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size);
+struct tok_state;
+
+typedef struct {
+    Py_ssize_t buf;
+    Py_ssize_t cur;
+    Py_ssize_t inp;
+    Py_ssize_t start;
+    Py_ssize_t line_start;
+    Py_ssize_t multi_line_start;
+} _PyLexer_BufferSnapshot;
+
+void _PyLexer_SnapshotBuffer(
+    struct tok_state *, const char *, _PyLexer_BufferSnapshot *);
+void _PyLexer_RestoreBuffer(
+    struct tok_state *, char *, const _PyLexer_BufferSnapshot *);
 
 #endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a96362c8961023..16f0a7b783f4b4 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -12,8 +12,21 @@
 
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
-#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
-                _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
+
+static int
+type_comment_token_setup(struct tok_state *tok, struct token *token, int type,
+                         int col_offset, int end_col_offset,
+                         const char *start, const char *end)
+{
+    _PyLexer_token_setup(tok, token, type, start, end);
+    token->start_loc = (_PyTok_Loc){tok->lineno, col_offset};
+    token->end_loc = (_PyTok_Loc){tok->lineno, end_col_offset};
+    return type;
+}
+
+#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) \
+    type_comment_token_setup(tok, token, token_type, col_offset, \
+                             end_col_offset, p_start, p_end)
 
 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
    tokenizing. */
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 1d97ac57b745b0..232dbaa287470e 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -7,4 +7,23 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
 
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
+/* The view points into the current input window. The next
+   _PyTokenizer_Get() call may discard it. */
+static inline const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+                  Py_ssize_t *length)
+{
+    assert(length != NULL);
+    if (!_PyTok_SpanIsValid(token->span)) {
+        *length = 0;
+        return "";
+    }
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(token->span.start >= tok->buf_offset);
+    assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
+    *length = token->span.end - token->span.start;
+    return tok->buf + (token->span.start - tok->buf_offset);
+}
+
 #endif
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2a6408bef927a3..d82a7d0f296bac 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void)
     tok->interactive_src_start = NULL;
     tok->interactive_src_end = NULL;
     tok->start = NULL;
-    tok->end = NULL;
     tok->done = E_OK;
     tok->fp = NULL;
     tok->tabsize = TABSIZE;
@@ -101,41 +100,46 @@ _PyToken_Free(struct token *token) {
 
 void
 _PyToken_Init(struct token *token) {
+#ifdef Py_DEBUG
+    token->span = (_PyTok_Span){-1, -1};
+    token->start_loc = (_PyTok_Loc){-1, -1};
+    token->end_loc = (_PyTok_Loc){-1, -1};
+#endif
     token->metadata = NULL;
 }
 
-int
-_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end)
+static inline _PyTok_Span
+buffer_span(const struct tok_state *tok, const char *start, const char *end)
 {
-    token->level = tok->level;
-    token->lineno = token->end_lineno = tok->lineno;
-    token->col_offset = col_offset;
-    token->end_col_offset = end_col_offset;
-    token->start = start;
-    token->end = end;
-    return type;
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    const char *base = tok->buf;
+    assert(base != NULL);
+    assert(tok->inp >= base);
+    Py_ssize_t start_offset = start - base;
+    Py_ssize_t end_offset = end - base;
+    assert(start_offset >= 0 && start_offset <= end_offset);
+    assert(end_offset <= tok->inp - base);
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
+    return _PyTok_SpanFromBounds(
+        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
 }
 
 int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
-    assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
     token->level = tok->level;
-    if (ISSTRINGLIT(type)) {
-        token->lineno = tok->first_lineno;
-    }
-    else {
-        token->lineno = tok->lineno;
-    }
-    token->end_lineno = tok->lineno;
-    token->col_offset = token->end_col_offset = -1;
-    token->start = start;
-    token->end = end;
+    token->span = buffer_span(tok, start, end);
+    int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
+    token->start_loc = (_PyTok_Loc){lineno, -1};
+    token->end_loc = (_PyTok_Loc){tok->lineno, -1};
 
     if (start != NULL && end != NULL) {
-        token->col_offset = tok->starting_col_offset;
-        token->end_col_offset = tok->col_offset;
+        token->start_loc.byte_col = tok->starting_col_offset;
+        token->end_loc.byte_col = tok->col_offset;
     }
     return type;
 }
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0824785195491e..496962fd0484f0 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -23,8 +23,9 @@ enum interactive_underflow_t {
 
 struct token {
     int level;
-    int lineno, col_offset, end_lineno, end_col_offset;
-    const char *start, *end;
+    _PyTok_Span span;
+    _PyTok_Loc start_loc;
+    _PyTok_Loc end_loc;
     PyObject *metadata;
 };
 
@@ -67,15 +68,15 @@ typedef struct _tokenizer_mode {
 
 /* Tokenizer state */
 struct tok_state {
-    /* Input state; buf <= cur <= inp <= end */
+    /* Input state; buf <= cur <= inp */
     /* NB an entire line is held in the buffer */
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
+    char *buf;          /* Owned for file/readline input; source-backed otherwise. */
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
+    _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
     int fp_interactive; /* If the file descriptor is interactive */
     char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
     char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
-    const char *end;    /* End of input buffer if buf != NULL */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
@@ -128,8 +129,6 @@ struct tok_state {
 #endif
 };
 
-int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end);
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index d67c48f7f678ed..fc0299c5c7c592 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
 {
     assert(tok->cur != NULL);
 
-    Py_ssize_t size = strlen(tok->cur);
+    Py_ssize_t size = cur == 0
+        ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur);
     tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
 
     switch (cur) {
@@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
                 goto error;
             }
             tok_mode->last_expr_buffer = new_buffer;
-            strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
+            memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size,
+                   tok->cur, size);
             tok_mode->last_expr_size += size;
             break;
         case '{':
@@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
             }
             tok_mode->last_expr_size = size;
             tok_mode->last_expr_end = -1;
-            strncpy(tok_mode->last_expr_buffer, tok->cur, size);
+            memcpy(tok_mode->last_expr_buffer, tok->cur, size);
             break;
         case '}':
         case '!':
diff --git a/Parser/pegen.c b/Parser/pegen.c
index fcec810037e98d..d86dd22444e6a7 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) {
 }
 
 static int
-_get_keyword_or_name_type(Parser *p, struct token *new_token)
+_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length)
 {
-    Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset;
-    assert(name_len > 0);
+    assert(length > 0);
 
-    if (name_len >= p->n_keyword_lists ||
-        p->keywords[name_len] == NULL ||
-        p->keywords[name_len]->type == -1) {
+    if (length >= p->n_keyword_lists ||
+        p->keywords[length] == NULL ||
+        p->keywords[length]->type == -1) {
         return NAME;
     }
-    for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
-        if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) {
+    for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) {
+        if (memcmp(k->str, text, (size_t)length) == 0) {
             return k->type;
         }
     }
@@ -193,8 +192,11 @@ static int
 initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
     assert(parser_token != NULL);
 
-    parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
-    parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
+    Py_ssize_t length;
+    const char *text = _PyToken_TextView(p->tok, new_token, &length);
+    parser_token->type = token_type == NAME
+        ? _get_keyword_or_name_type(p, text, length) : token_type;
+    parser_token->bytes = PyBytes_FromStringAndSize(text, length);
     if (parser_token->bytes == NULL) {
         return -1;
     }
@@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
     }
 
     parser_token->level = new_token->level;
-    parser_token->lineno = new_token->lineno;
-    parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
-                                                                    : new_token->col_offset;
-    parser_token->end_lineno = new_token->end_lineno;
-    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset
-                                                                 : new_token->end_col_offset;
+    parser_token->lineno = new_token->start_loc.lineno;
+    parser_token->col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->start_loc.byte_col
+        : new_token->start_loc.byte_col;
+    parser_token->end_lineno = new_token->end_loc.lineno;
+    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->end_loc.byte_col
+        : new_token->end_loc.byte_col;
 
     p->fill += 1;
 
@@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p)
 
     // Record and skip '# type: ignore' comments
     while (type == TYPE_IGNORE) {
-        Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;
+        Py_ssize_t len;
+        const char *text = _PyToken_TextView(p->tok, &new_token, &len);
         char *tag = PyMem_Malloc((size_t)len + 1);
         if (tag == NULL) {
             PyErr_NoMemory();
             goto error;
         }
-        strncpy(tag, new_token.start, (size_t)len);
+        memcpy(tag, text, (size_t)len);
         tag[len] = '\0';
         // Ownership of tag passes to the growable array
         if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 82b824f56374fc..408bfda147b4e9 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -13,6 +13,12 @@
 #  include <unistd.h>
 #endif
 
+static inline int
+reader_is_streaming(_PyTok_ReaderKind kind)
+{
+    return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE;
+}
+
 void
 _PyTok_ReaderFree(struct tok_state *tok)
 {
@@ -28,14 +34,27 @@ _PyTok_ReaderFree(struct tok_state *tok)
     }
     PyMem_Free(reader->file_buffer);
     PyMem_Free(reader->decoded);
-    if (reader->kind != _PYTOK_READER_PREPARED) {
+    if (reader_is_streaming(reader->kind)) {
         PyMem_Free(tok->buf);
-        tok->buf = NULL;
     }
+    tok->buf = NULL;
     PyMem_Free(reader);
     tok->reader = NULL;
 }
 
+static int
+resize_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t new_capacity)
+{
+    char *resized = PyMem_Realloc(*buffer, new_capacity);
+    if (resized == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    *buffer = resized;
+    *capacity = new_capacity;
+    return 0;
+}
+
 static int
 reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
 {
@@ -50,13 +69,31 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
         }
         cap *= 2;
     }
-    char *resized = PyMem_Realloc(*buffer, cap);
-    if (resized == NULL) {
-        PyErr_NoMemory();
+    return resize_buffer(buffer, capacity, cap);
+}
+
+static int
+reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
+{
+    _PyTok_Reader *reader = tok->reader;
+    if (needed <= reader->input_buffer_cap) {
+        return 0;
+    }
+    assert(tok->buf != NULL);
+    assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+    assert(tok->inp - tok->buf <= reader->input_buffer_cap);
+    Py_ssize_t used = tok->inp - tok->buf;
+    Py_ssize_t growth = used >> 1;
+    Py_ssize_t capacity = used > PY_SSIZE_T_MAX - growth
+        ? needed : Py_MAX(needed, used + growth);
+    _PyLexer_BufferSnapshot snapshot;
+    _PyLexer_SnapshotBuffer(tok, tok->buf, &snapshot);
+    char *buffer = tok->buf;
+    if (resize_buffer(
+            &buffer, &reader->input_buffer_cap, capacity) < 0) {
         return -1;
     }
-    *buffer = resized;
-    *capacity = cap;
+    _PyLexer_RestoreBuffer(tok, buffer, &snapshot);
     return 0;
 }
 
@@ -529,19 +566,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
     Py_UNREACHABLE();
 }
 
+static void
+reset_streaming_buffer(struct tok_state *tok)
+{
+    assert(tok->buf != NULL);
+    assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+    Py_ssize_t consumed = tok->inp - tok->buf;
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed);
+    tok->buf_offset += consumed;
+    tok->cur = tok->inp = tok->buf;
+}
+
 int
 _PyTok_ReaderUnderflow(struct tok_state *tok)
 {
-    int prepared = tok->reader->kind == _PYTOK_READER_PREPARED;
+    _PyTok_ReaderKind kind = tok->reader->kind;
+    int prepared = kind == _PYTOK_READER_PREPARED;
+    int streaming = reader_is_streaming(kind);
     int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
 
-    if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
-    }
-
     _PyTok_Chunk chunk;
     _PyTok_ReadResult result = reader_next(tok, &chunk);
     if (result != _PYTOK_READ_LINE) {
+        if (reset_buffer && streaming) {
+            reset_streaming_buffer(tok);
+        }
         if (result == _PYTOK_READ_EOF) {
             tok->done = E_EOF;
         }
@@ -558,34 +607,68 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
                     ? E_NOMEM : E_ERROR;
             }
         }
-        if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+        if (kind == _PYTOK_READER_INTERACTIVE &&
                 result != _PYTOK_READ_STOPPED) {
             PySys_WriteStderr("\n");
         }
         return 0;
     }
 
-    Py_ssize_t copy_len = chunk.len;
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+    Py_ssize_t scan_len = chunk.len;
+    if (kind == _PYTOK_READER_INTERACTIVE &&
             chunk.implicit_newline) {
-        copy_len--;
-    }
-    if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
+        scan_len--;
     }
-    if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
+    if (streaming) {
+        if (reset_buffer) {
+            reset_streaming_buffer(tok);
+        }
+        Py_ssize_t used = tok->inp - tok->buf;
+        int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 ||
+                       tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len;
+        if (overflow) {
+            PyErr_NoMemory();
+        }
+        if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) {
+            _PyTok_ChunkClear(&chunk);
+            tok->done = E_NOMEM;
+            tok->input_error = 1;
+            return 0;
+        }
+        memcpy(tok->inp, chunk.data, (size_t)scan_len);
+        tok->inp += scan_len;
+        *tok->inp = '\0';
     }
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
-            _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len,
-                                    chunk.implicit_newline) < 0) {
-        _PyTok_ChunkClear(&chunk);
-        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
-            ? E_NOMEM : E_ERROR;
-        tok->input_error = 1;
-        return 0;
+    else if (!prepared) {
+        int source_will_grow =
+            chunk.len > tok->source.cap - tok->source.len - 1;
+        _PyLexer_BufferSnapshot snapshot;
+        if (!reset_buffer && source_will_grow) {
+            _PyLexer_SnapshotBuffer(
+                tok, tok->source.bytes, &snapshot);
+        }
+        _PyTok_Off source_start = _PyTok_SourceAppendLine(
+            &tok->source, chunk.data, chunk.len,
+            chunk.implicit_newline);
+        if (source_start < 0) {
+            _PyTok_ChunkClear(&chunk);
+            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
+                ? E_NOMEM : E_ERROR;
+            tok->input_error = 1;
+            return 0;
+        }
+        if (reset_buffer) {
+            tok->buf = tok->cur = tok->source.bytes + source_start;
+            tok->buf_offset = source_start;
+            tok->line_start = tok->buf;
+            tok->start = NULL;
+            tok->multi_line_start = NULL;
+        }
+        else if (source_will_grow) {
+            _PyLexer_RestoreBuffer(
+                tok, tok->source.bytes, &snapshot);
+        }
+        tok->inp = tok->source.bytes + source_start + scan_len;
     }
     if (tok->fp_interactive) {
         tok->interactive_src_start = tok->source.bytes;
@@ -594,14 +677,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     if (prepared) {
         if (tok->start == NULL) {
             tok->buf = tok->cur;
+            tok->buf_offset = chunk.data - tok->source.bytes;
         }
         tok->inp = chunk.data + chunk.len;
     }
-    else {
-        memcpy(tok->inp, chunk.data, (size_t)copy_len);
-        tok->inp += copy_len;
-        *tok->inp = '\0';
-    }
     tok->implicit_newline = chunk.implicit_newline;
 
     if (!prepared && tok->tok_mode_stack_index &&
@@ -611,7 +690,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         return 0;
     }
     ADVANCE_LINENO();
-    if (tok->reader->kind == _PYTOK_READER_FILE &&
+    if (kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
             !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
         _PyTok_ChunkClear(&chunk);
@@ -639,14 +718,15 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
     if (kind == _PYTOK_READER_PREPARED) {
         return tok;
     }
-    tok->buf = PyMem_Malloc(BUFSIZ);
-    if (tok->buf == NULL) {
-        PyErr_NoMemory();
-        _PyTokenizer_Free(tok);
-        return NULL;
+    if (reader_is_streaming(kind)) {
+        if (reserve_buffer(
+                &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) {
+            _PyTokenizer_Free(tok);
+            return NULL;
+        }
+        tok->cur = tok->inp = tok->buf;
+        tok->buf[0] = '\0';
     }
-    tok->cur = tok->inp = tok->buf;
-    tok->end = tok->buf + BUFSIZ;
     return tok;
 }
 
@@ -664,7 +744,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
         return NULL;
     }
     tok->buf = tok->cur = tok->inp = tok->str;
-    tok->end = tok->buf;
     return tok;
 }
 
diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h
index 121d0f96f6698a..49a6f04ec60af2 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -44,6 +44,8 @@ typedef struct _PyTok_Reader {
     PyObject *decoder;
     const char *nextprompt;
 
+    Py_ssize_t input_buffer_cap;
+
     char *file_buffer;
     Py_ssize_t file_buffer_cap;
     _PyTok_Chunk prefetched_lines[2];
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index b42ecda1b31aa5..363475ff9015e3 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -5,7 +5,8 @@
 
 typedef Py_ssize_t _PyTok_Off;
 
-/* Half-open byte offsets into a _PyTok_SourceText. */
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+   storage may retain only the current input window. */
 typedef struct {
     _PyTok_Off start;
     _PyTok_Off end;
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 762b7b3e4c8d71..eee1fc86ded781 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -203,14 +203,15 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
 }
 
 static void
-_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start,
-                 PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
+_get_col_offsets(tokenizeriterobject *it, const char *token_start,
+                 const char *token_end, const char *line_start, PyObject *line,
+                 int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
                  Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
 {
     _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
     Py_ssize_t byte_offset = -1;
-    if (token.start != NULL && token.start >= line_start) {
-        byte_offset = token.start - line_start;
+    if (token_start != NULL && token_start >= line_start) {
+        byte_offset = token_start - line_start;
         if (line_changed) {
             *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
             it->byte_col_offset_diff = byte_offset - *col_offset;
@@ -220,15 +221,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
         }
     }
 
-    if (token.end != NULL && token.end >= it->tok->line_start) {
-        Py_ssize_t end_byte_offset = token.end - it->tok->line_start;
+    if (token_end != NULL && token_end >= it->tok->line_start) {
+        Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
         if (lineno == end_lineno) {
-            // If the whole token is at the same line, we can just use the token.start
-            // buffer for figuring out the new column offset, since using line is not
-            // performant for very long lines.
+            // Avoid rescanning the prefix of a very long line.
             Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
             *end_col_offset = *col_offset + token_col_offset;
-            it->byte_col_offset_diff += token.end - token.start - token_col_offset;
+            it->byte_col_offset_diff += token_end - token_start - token_col_offset;
         }
         else {
             *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
@@ -263,12 +262,18 @@ tokenizeriter_next(PyObject *op)
         it->done = 1;
         goto exit;
     }
-    PyObject *str = NULL;
-    if (token.start == NULL || token.end == NULL) {
+    const char *token_start = NULL;
+    const char *token_end = NULL;
+    PyObject *str;
+    if (!_PyTok_SpanIsValid(token.span)) {
         str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     }
     else {
-        str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
+        Py_ssize_t token_length;
+        token_start = _PyToken_TextView(
+            it->tok, &token, &token_length);
+        token_end = token_start + token_length;
+        str = PyUnicode_FromStringAndSize(token_start, token_length);
     }
     if (str == NULL) {
         goto exit;
@@ -297,11 +302,11 @@ tokenizeriter_next(PyObject *op)
         goto exit;
     }
 
-    Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
-    Py_ssize_t end_lineno = it->tok->lineno;
+    Py_ssize_t lineno = token.start_loc.lineno;
+    Py_ssize_t end_lineno = token.end_loc.lineno;
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;
-    _get_col_offsets(it, token, line_start, line, line_changed,
+    _get_col_offsets(it, token_start, token_end, line_start, line, line_changed,
                      lineno, end_lineno, &col_offset, &end_col_offset);
 
     if (it->tok->tok_extra_tokens) {
@@ -317,7 +322,8 @@ tokenizeriter_next(PyObject *op)
         else if (type == NEWLINE) {
             Py_DECREF(str);
             if (!it->tok->implicit_newline) {
-                if (it->tok->start[0] == '\r') {
+                assert(token_start != NULL);
+                if (token_start[0] == '\r') {
                     str = PyUnicode_FromString("\r\n");
                 } else {
                     str = PyUnicode_FromString("\n");