gh-153569: streamline tokenizer hot paths

Author: pablogsal

Lib/test/test_fstring.py

@@ -1303,6 +1303,13 @@ def test_nested_fstrings(self):
         self.assertEqual(f'{f"{0}"*3}', '000')
         self.assertEqual(f'{f"{y}"*3}', '555')
 
+    def test_deeply_nested_fstrings_with_leading_text(self):
+        source = "0"
+        for depth in range(20):
+            quote = '"' if depth % 2 == 0 else "'"
+            source = f"f{quote}x{{{source}}}{quote}"
+        self.assertEqual(eval(source), "x" * 20 + "0")
+
     def test_invalid_string_prefixes(self):
         single_quote_cases = ["fu''",
                              "uf''",

Lib/test/test_syntax.py

@@ -3106,7 +3106,7 @@ def test_tokenizer_eof_error_offsets_after_non_ascii(self):
             end_offset=-1,
         )
 
-    def test_error_line_excludes_old_newline(self):
+    def test_fstring_error_text_excludes_previous_newline(self):
         cases = [
             ('f"{a:{\n0\n\'\'=(\n="0("', (2, 1, "0", 3, 2)),
             ("f'{x!=[\n]a\n}(]==!", (2, 3, "]a", 2, 2)),
@@ -3115,7 +3115,14 @@ def test_error_line_excludes_old_newline(self):
             with self.subTest(source=source):
                 with self.assertRaises(SyntaxError) as caught:
                     compile(source, "<testcase>", "exec")
-                self.assertEqual(caught.exception.args[1][1:], expected)
+                actual = (
+                    caught.exception.lineno,
+                    caught.exception.offset,
+                    caught.exception.text,
+                    caught.exception.end_lineno,
+                    caught.exception.end_offset,
+                )
+                self.assertEqual(actual, expected)
 
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")

Modules/_testinternalcapi/tokenizer.c

@@ -30,7 +30,8 @@ same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
         left->pos == right->pos &&
         left->line_start == right->line_start &&
         left->line_end == right->line_end &&
-        left->lineno == right->lineno;
+        left->lineno == right->lineno &&
+        left->implicit_newline == right->implicit_newline;
 }
 
 static PyObject *
@@ -96,19 +97,16 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
             check(line.start == marker_start &&
                       line.end == marker_start + 7,
                   "wrong late source line") < 0 ||
-            check(!line.implicit_newline && !line.contains_nul,
+            check(!line.implicit_newline,
                   "wrong late source flags") < 0 ||
             check(_PyTok_SourceLine(&source, 2, &line) == 0,
                   "cannot find second source line") < 0 ||
             check(line.start == 6 && line.end == 9 &&
-                      line.implicit_newline && !line.contains_nul,
+                      line.implicit_newline,
                   "wrong second source line") < 0 ||
             check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
                       _PyTok_SourceLineIsImplicit(&source, 2),
                   "wrong early implicit newline flags") < 0 ||
-            check(_PyTok_SourceLine(&source, 3, &line) == 0,
-                  "cannot find third source line") < 0 ||
-            check(line.contains_nul, "missing null byte flag") < 0 ||
             check(_PyTok_SourceLine(&source, final_line, &line) == 0,
                   "cannot find final source line") < 0 ||
             check(line.implicit_newline &&
@@ -234,27 +232,19 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
               "wrong cursor peek after relocation") < 0 ||
             check(_PyTok_CursorPeek(&cursor, 1) == 'b',
                   "wrong distant cursor peek") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'a',
-                  "wrong first cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == 'b',
-                  "wrong second cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
-                  "wrong final cursor byte") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == EOF,
-                  "cursor advanced past line") < 0 ||
             check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
                   "cannot seek cursor offset") < 0 ||
-            check(_PyTok_CursorAdvance(&cursor) == '\n',
+            check(_PyTok_CursorPeek(&cursor, 0) == '\n',
                   "wrong cursor byte after seek") < 0 ||
             check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
                   "cannot seek line boundary") < 0 ||
             check(cursor.lineno == 2 && cursor.line_start == 3 &&
-                      _PyTok_CursorAdvance(&cursor) == 'c',
+                      _PyTok_CursorPeek(&cursor, 0) == 'c',
                   "wrong cursor at line boundary") < 0 ||
             check(_PyTok_CursorSetLine(&cursor, 3) == 0,
                   "cannot advance cursor to final line") < 0 ||
             check(cursor.line_start == 6 &&
-                      _PyTok_CursorAdvance(&cursor) == 'z',
+                      _PyTok_CursorPeek(&cursor, 0) == 'z',
                   "wrong cursor byte on final line") < 0) {
         goto error;
     }
@@ -277,25 +267,6 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
         goto error;
     }
 
-#if SIZEOF_VOID_P > 4
-    char byte = 0;
-    _PyTok_SourceText huge_source = {
-        .bytes = &byte,
-        .len = (_PyTok_Off)INT_MAX + 1,
-    };
-    _PyTok_Cursor huge_cursor = {
-        .source = &huge_source,
-        .pos = INT_MAX,
-        .line_end = (_PyTok_Off)INT_MAX + 1,
-        .lineno = 1,
-    };
-    if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
-                  huge_cursor.pos == INT_MAX,
-              "cursor advanced past maximum column") < 0) {
-        goto error;
-    }
-#endif
-
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
 

Parser/lexer/lexer.c

@@ -21,8 +21,7 @@ emit_type_comment(struct _PyTokenizer *tok, _PyTok_Token *token, int type,
     token->span = start < 0 || end < start
         ? _PyTok_InvalidSpan()
         : _PyTok_SpanFromBounds(start, end);
-    token->flags = _PyTok_SourceLineIsImplicit(
-        &tok->source, tok->cursor.lineno) ? _PYTOK_IMPLICIT_NL : 0;
+    token->flags = tok->cursor.implicit_newline ? _PYTOK_IMPLICIT_NL : 0;
     return type;
 }
 
@@ -55,6 +54,47 @@ source_column(struct _PyTokenizer *tok, _PyTok_Off offset)
 /* Spaces in the prefix match any sequence of spaces and tabs. */
 static const char type_comment_prefix[] = "# type: ";
 
+int
+_PyTok_EmitTokenWithLocationEnd(struct _PyTokenizer *tok,
+                                _PyTok_Token *token, int type,
+                                _PyTok_Off start, _PyTok_Off end,
+                                _PyTok_Off location_end)
+{
+    assert((start < 0 && end < 0) || (start >= 0 && end >= start));
+    assert(start < 0 || location_end >= end);
+    if (start < 0 && type == ENDMARKER) {
+        start = end = location_end = tok->cursor.pos;
+    }
+    token->type = type;
+    token->level = tok->level;
+    token->span = start < 0
+        ? _PyTok_InvalidSpan()
+        : _PyTok_SpanFromBounds(start, end);
+    token->start = (_PyTok_Loc){tok->cursor.lineno, -1};
+    token->end = token->start;
+    if (start >= 0) {
+        if (_PyTok_CursorLocation(
+                &tok->cursor, start, _PYTOK_AFFINITY_RIGHT,
+                &token->start) < 0) {
+            return record_location_error(tok);
+        }
+        _PyTok_Affinity end_affinity =
+            type == FSTRING_MIDDLE || type == TSTRING_MIDDLE ||
+            start == location_end
+            ? _PYTOK_AFFINITY_RIGHT
+            : _PYTOK_AFFINITY_LEFT;
+        if (_PyTok_CursorLocation(
+                &tok->cursor, location_end, end_affinity,
+                &token->end) < 0) {
+            return record_location_error(tok);
+        }
+    }
+    token->flags =
+        (tok->cursor.implicit_newline ? _PYTOK_IMPLICIT_NL : 0) |
+        (location_end != end ? _PYTOK_TEXT_END_DIFFERS : 0);
+    return type;
+}
+
 /* Verify that the identifier follows PEP 3131. */
 static int
 verify_identifier(struct _PyTokenizer *tok)

Parser/lexer/lexer_internal.h

@@ -21,15 +21,17 @@ static inline int
 _PyTok_LexerAdvance(struct _PyTokenizer *tok)
 {
     for (;;) {
-        if (tok->cursor.pos < tok->cursor.line_end &&
-                tok->cursor.pos - tok->cursor.line_start >= INT_MAX) {
-            _PyTok_RecordCurrentError(
-                tok, _PYTOK_ERR_COLUMN_OVERFLOW, NULL);
-            return EOF;
-        }
-        int c = _PyTok_CursorAdvance(&tok->cursor);
-        if (c != EOF) {
-            return c;
+        assert(tok->cursor.source != NULL);
+        assert(tok->cursor.pos >= tok->cursor.line_start);
+        assert(tok->cursor.pos <= tok->cursor.line_end);
+        assert(tok->cursor.line_end <= tok->source.len);
+        if (tok->cursor.pos < tok->cursor.line_end) {
+            if (tok->cursor.pos - tok->cursor.line_start >= INT_MAX) {
+                _PyTok_RecordCurrentError(
+                    tok, _PYTOK_ERR_COLUMN_OVERFLOW, NULL);
+                return EOF;
+            }
+            return Py_CHARMASK(tok->source.bytes[tok->cursor.pos++]);
         }
         if (_PyTok_HasError(tok) || !_PyTok_ReaderUnderflow(tok)) {
             return EOF;
@@ -52,6 +54,10 @@ _PyTok_LexerMark(const struct _PyTokenizer *tok)
 static inline void
 _PyTok_LexerReset(struct _PyTokenizer *tok, _PyTok_Off offset)
 {
+    if (offset >= tok->cursor.line_start && offset < tok->cursor.line_end) {
+        tok->cursor.pos = offset;
+        return;
+    }
     int result = _PyTok_CursorSetOffset(&tok->cursor, offset);
     if (result < 0) {
         _PyTok_RecordPending(tok, _PYTOK_ERR_PROPAGATE);
@@ -63,6 +69,7 @@ static inline _PyTok_Frame *
 _PyTok_CurrentFrame(struct _PyTokenizer *tok)
 {
     assert(tok->frame_index >= 0);
+    assert(tok->frame_index < tok->frame_capacity);
     assert(tok->frame_index < _PYTOK_MAX_FRAMES);
     return &tok->frames[tok->frame_index];
 }
@@ -91,50 +98,31 @@ _PyTok_LexerLocation(struct _PyTokenizer *tok, _PyTok_Off offset)
     return loc;
 }
 
-static inline int
+int
 _PyTok_EmitTokenWithLocationEnd(struct _PyTokenizer *tok,
                                 _PyTok_Token *token, int type,
                                 _PyTok_Off start, _PyTok_Off end,
-                                _PyTok_Off location_end)
-{
-    assert((start < 0 && end < 0) || (start >= 0 && end >= start));
-    assert(start < 0 || location_end >= end);
-    int synthetic = start < 0;
-    if (synthetic && type == ENDMARKER) {
-        start = end = location_end = tok->cursor.pos;
-    }
-    token->type = type;
-    token->level = tok->level;
-    token->span = start < 0
-        ? _PyTok_InvalidSpan()
-        : _PyTok_SpanFromBounds(start, end);
-    token->start = (_PyTok_Loc){tok->cursor.lineno, -1};
-    token->end = token->start;
-    if (start >= 0) {
-        if (_PyTok_CursorLocation(
-                &tok->cursor, start, _PYTOK_AFFINITY_RIGHT,
-                &token->start) < 0) {
-            return record_location_error(tok);
-        }
-        _PyTok_Affinity end_affinity =
-            type == FSTRING_MIDDLE || type == TSTRING_MIDDLE ||
-            start == location_end
-            ? _PYTOK_AFFINITY_RIGHT
-            : _PYTOK_AFFINITY_LEFT;
-        if (_PyTok_CursorLocation(
-                &tok->cursor, location_end, end_affinity, &token->end) < 0) {
-            return record_location_error(tok);
-        }
-    }
-    token->flags = _PyTok_SourceLineIsImplicit(
-        &tok->source, tok->cursor.lineno) ? _PYTOK_IMPLICIT_NL : 0;
-    return type;
-}
+                                _PyTok_Off location_end);
 
 static inline int
 _PyTok_EmitToken(struct _PyTokenizer *tok, _PyTok_Token *token, int type,
                  _PyTok_Off start, _PyTok_Off end)
 {
+    assert(start < 0 || end >= start);
+    if (start >= tok->cursor.line_start &&
+            start < tok->cursor.line_end &&
+            end <= tok->cursor.line_end &&
+            end - tok->cursor.line_start <= INT_MAX) {
+        token->type = type;
+        token->level = tok->level;
+        token->span = _PyTok_SpanFromBounds(start, end);
+        token->start = (_PyTok_Loc){
+            tok->cursor.lineno, (int)(start - tok->cursor.line_start)};
+        token->end = (_PyTok_Loc){
+            tok->cursor.lineno, (int)(end - tok->cursor.line_start)};
+        token->flags = tok->cursor.implicit_newline ? _PYTOK_IMPLICIT_NL : 0;
+        return type;
+    }
     return _PyTok_EmitTokenWithLocationEnd(
         tok, token, type, start, end, end);
 }

Parser/lexer/state.c

@@ -20,9 +20,13 @@ _PyTok_StateNew(void)
     _PyTok_CursorInit(&tok->cursor, &tok->source);
     tok->tabsize = TABSIZE;
     tok->atbol = 1;
-    tok->frames[0].body_start = -1;
-    tok->frames[0].debug_expr_start = -1;
-    tok->frames[0].debug_expr_end = -1;
+    tok->frames = tok->inline_frames;
+    tok->frame_capacity = _PYTOK_INLINE_FRAMES;
+    tok->frames[0] = (_PyTok_Frame){
+        .body_start = -1,
+        .debug_expr_start = -1,
+        .debug_expr_end = -1,
+    };
     return tok;
 }
 
@@ -34,5 +38,8 @@ _PyTok_StateFree(struct _PyTokenizer *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
+    if (tok->frames != tok->inline_frames) {
+        PyMem_Free(tok->frames);
+    }
     PyMem_Free(tok);
 }

Parser/lexer/state.h

@@ -11,6 +11,7 @@
 #define MAXLEVEL 200        /* Max parentheses level */
 #define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */
 #define _PYTOK_MAX_FRAMES (MAXFSTRINGLEVEL * 3)
+#define _PYTOK_INLINE_FRAMES 4
 
 #define INSIDE_FSTRING(tok) (tok->frame_index > 0)
 #define INSIDE_FSTRING_EXPR(mode) ((mode)->kind == _PYTOK_FRAME_FSTRING_EXPR)
@@ -71,14 +72,22 @@ struct _PyTokenizer {
 
     int type_comments;      /* Whether to look for type comments */
 
-    _PyTok_Frame frames[_PYTOK_MAX_FRAMES];
+    _PyTok_Frame *frames;
+    _PyTok_Frame inline_frames[_PYTOK_INLINE_FRAMES];
+    int frame_capacity;
     int frame_index;
     int extra_tokens;
     int comment_newline;
     int is_interactive;
     int is_prompting;
 };
 
+static inline int
+_PyTok_HasError(const struct _PyTokenizer *tok)
+{
+    return tok->error.kind != _PYTOK_ERR_NONE;
+}
+
 struct _PyTokenizer *_PyTok_StateNew(void);
 void _PyTok_StateFree(struct _PyTokenizer *);