gh-153569: strengthen tokenizer validation

Author: pablogsal

Modules/_xxtestfuzz/fuzzer.c

@@ -18,6 +18,13 @@
 #include <stdlib.h>
 #include <inttypes.h>
 
+#if defined(_Py_FUZZ_ONE) && defined(_Py_FUZZ_fuzz_tokenizer) && \
+        defined(HAVE_TMPFILE) && !defined(Py_ENABLE_SHARED)
+#  define FUZZ_TOKENIZER_FILE 1
+#  include "../../Include/internal/pycore_token.h"
+#  include "../../Parser/tokenizer/tokenizer.h"
+#endif
+
 /*  Fuzz PyFloat_FromString as a proxy for float(str). */
 static int fuzz_builtin_float(const char* data, size_t size) {
     PyObject* s = PyBytes_FromStringAndSize(data, size);
@@ -623,7 +630,6 @@ clear_expected_tokenizer_error(void)
     if (PyErr_ExceptionMatches(PyExc_SyntaxError) ||
             PyErr_ExceptionMatches(PyExc_UnicodeError) ||
             PyErr_ExceptionMatches(PyExc_LookupError) ||
-            PyErr_ExceptionMatches(PyExc_MemoryError) ||
             PyErr_ExceptionMatches(PyExc_Warning)) {
         PyErr_Clear();
         return;
@@ -632,6 +638,82 @@ clear_expected_tokenizer_error(void)
     abort();
 }
 
+#ifdef FUZZ_TOKENIZER_FILE
+static int
+fuzz_tokenizer_file(const char *data, size_t size, unsigned char options)
+{
+    FILE *fp = tmpfile();
+    if (fp == NULL) {
+        return 0;
+    }
+    if ((size != 0 && fwrite(data, 1, size, fp) != size) ||
+            fseek(fp, 0, SEEK_SET) != 0) {
+        fclose(fp);
+        return 0;
+    }
+
+    PyObject *filename = PyUnicode_FromString("<fuzz input>");
+    if (filename == NULL) {
+        fclose(fp);
+        clear_expected_tokenizer_error();
+        return 1;
+    }
+    _PyTok_Config config = {
+        .kind = _PYTOK_SOURCE_FILE,
+        .source.file = {fp, NULL, NULL, NULL},
+        .extra_tokens = options & 0x02,
+        .filename = filename,
+    };
+    PyTokenizer *tok = _PyTok_New(&config);
+    if (tok == NULL) {
+        _PyTok_RaiseInitError(filename);
+        Py_DECREF(filename);
+        fclose(fp);
+        clear_expected_tokenizer_error();
+        return 1;
+    }
+    Py_DECREF(filename);
+
+    int terminated = 0;
+    size_t limit = size * 4 + 32;
+    for (size_t i = 0; i < limit; i++) {
+        _PyTok_Token token;
+        _PyTok_TokenInit(&token);
+        if (_PyTok_Get(tok, &token) == _PYTOK_ERROR) {
+            _PyTok_TokenClear(&token);
+            _PyTok_RaiseError(tok);
+            clear_expected_tokenizer_error();
+            terminated = 1;
+            break;
+        }
+        int type = token.type;
+        Py_ssize_t token_len;
+        _PyTok_Loc start;
+        _PyTok_Loc end;
+        if (_PyTok_TokenView(tok, &token, &token_len) == NULL ||
+                token_len < 0 ||
+                _PyTok_TokenTextLocations(tok, &token, &start, &end) < 0 ||
+                start.lineno < 1 || start.byte_col < -1 ||
+                end.lineno < start.lineno || end.byte_col < -1 ||
+                (end.lineno == start.lineno &&
+                 end.byte_col < start.byte_col)) {
+            abort();
+        }
+        _PyTok_TokenClear(&token);
+        if (type == ENDMARKER) {
+            terminated = 1;
+            break;
+        }
+    }
+    _PyTok_Free(tok);
+    fclose(fp);
+    if (!terminated) {
+        abort();
+    }
+    return 1;
+}
+#endif
+
 static int
 fuzz_tokenizer(const char *data, size_t size)
 {
@@ -643,6 +725,12 @@ fuzz_tokenizer(const char *data, size_t size)
     data++;
     size--;
 
+#ifdef FUZZ_TOKENIZER_FILE
+    if ((options & 0x04) && fuzz_tokenizer_file(data, size, options)) {
+        return 0;
+    }
+#endif
+
     int bytes_mode = options & 0x01;
     PyObject *source;
     if (bytes_mode) {