gh-153569: add tokenizer validation tools

Author: pablogsal

Modules/_xxtestfuzz/fuzz_tests.txt

@@ -9,3 +9,4 @@ fuzz_struct_unpack
 fuzz_ast_literal_eval
 fuzz_elementtree_parsewhole
 fuzz_pycompile
+fuzz_tokenizer

Modules/_xxtestfuzz/fuzzer.c

@@ -441,6 +441,22 @@ static int fuzz_ast_literal_eval(const char* data, size_t size) {
 #define MAX_ELEMENTTREE_PARSEWHOLE_TEST_SIZE 0x100000
 PyObject* xmlparser_type = NULL;
 PyObject* bytesio_type = NULL;
+
+static int
+init_bytesio(void)
+{
+    if (bytesio_type != NULL) {
+        return 1;
+    }
+    PyObject *io_module = PyImport_ImportModule("_io");
+    if (io_module == NULL) {
+        return 0;
+    }
+    bytesio_type = PyObject_GetAttrString(io_module, "BytesIO");
+    Py_DECREF(io_module);
+    return bytesio_type != NULL;
+}
+
 /* Called by LLVMFuzzerTestOneInput for initialization */
 static int init_elementtree_parsewhole(void) {
     PyObject* elementtree_module = PyImport_ImportModule("_elementtree");
@@ -452,15 +468,7 @@ static int init_elementtree_parsewhole(void) {
     if (xmlparser_type == NULL) {
         return 0;
     }
-
-
-    PyObject* io_module = PyImport_ImportModule("io");
-    if (io_module == NULL) {
-        return 0;
-    }
-    bytesio_type = PyObject_GetAttrString(io_module, "BytesIO");
-    Py_DECREF(io_module);
-    if (bytesio_type == NULL) {
+    if (!init_bytesio()) {
         return 0;
     }
 
@@ -578,6 +586,136 @@ static int fuzz_pycompile(const char* data, size_t size) {
     return 0;
 }
 
+#define MAX_TOKENIZER_TEST_SIZE 16384
+
+static PyObject *tokenizer_iter_type;
+static PyObject *stringio_type;
+
+static int
+init_tokenizer(void)
+{
+    PyObject *tokenize_module = PyImport_ImportModule("_tokenize");
+    if (tokenize_module == NULL) {
+        return 0;
+    }
+    tokenizer_iter_type = PyObject_GetAttrString(
+        tokenize_module, "TokenizerIter");
+    Py_DECREF(tokenize_module);
+    if (tokenizer_iter_type == NULL) {
+        return 0;
+    }
+
+    PyObject *io_module = PyImport_ImportModule("_io");
+    if (io_module == NULL) {
+        return 0;
+    }
+    stringio_type = PyObject_GetAttrString(io_module, "StringIO");
+    Py_DECREF(io_module);
+    if (stringio_type == NULL) {
+        return 0;
+    }
+    return init_bytesio();
+}
+
+static void
+clear_expected_tokenizer_error(void)
+{
+    if (PyErr_ExceptionMatches(PyExc_SyntaxError) ||
+            PyErr_ExceptionMatches(PyExc_UnicodeError) ||
+            PyErr_ExceptionMatches(PyExc_LookupError) ||
+            PyErr_ExceptionMatches(PyExc_MemoryError) ||
+            PyErr_ExceptionMatches(PyExc_Warning)) {
+        PyErr_Clear();
+        return;
+    }
+    PyErr_Print();
+    abort();
+}
+
+static int
+fuzz_tokenizer(const char *data, size_t size)
+{
+    if (size < 1 || size > MAX_TOKENIZER_TEST_SIZE) {
+        return 0;
+    }
+
+    unsigned char options = (unsigned char)data[0];
+    data++;
+    size--;
+
+    int bytes_mode = options & 0x01;
+    PyObject *source;
+    if (bytes_mode) {
+        source = PyBytes_FromStringAndSize(data, size);
+    }
+    else {
+        source = PyUnicode_DecodeUTF8(data, size, "replace");
+    }
+    if (source == NULL) {
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+
+    PyObject *stream_type = bytes_mode ? bytesio_type : stringio_type;
+    PyObject *stream = PyObject_CallOneArg(stream_type, source);
+    Py_DECREF(source);
+    if (stream == NULL) {
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+    PyObject *readline = PyObject_GetAttrString(stream, "readline");
+    Py_DECREF(stream);
+    if (readline == NULL) {
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+
+    PyObject *args = PyTuple_Pack(1, readline);
+    Py_DECREF(readline);
+    if (args == NULL) {
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+    PyObject *extra_tokens = options & 0x02 ? Py_True : Py_False;
+    PyObject *kwargs = bytes_mode
+        ? Py_BuildValue(
+            "{s:O,s:s}", "extra_tokens", extra_tokens,
+            "encoding", "utf-8")
+        : Py_BuildValue("{s:O}", "extra_tokens", extra_tokens);
+    if (kwargs == NULL) {
+        Py_DECREF(args);
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+
+    PyObject *iterator = PyObject_Call(tokenizer_iter_type, args, kwargs);
+    Py_DECREF(args);
+    Py_DECREF(kwargs);
+    if (iterator == NULL) {
+        clear_expected_tokenizer_error();
+        return 0;
+    }
+
+    int terminated = 0;
+    size_t limit = size * 4 + 32;
+    for (size_t i = 0; i < limit; i++) {
+        PyObject *token = PyIter_Next(iterator);
+        if (token == NULL) {
+            terminated = 1;
+            if (PyErr_Occurred()) {
+                clear_expected_tokenizer_error();
+            }
+            break;
+        }
+        Py_DECREF(token);
+    }
+    Py_DECREF(iterator);
+    if (!terminated) {
+        abort();
+    }
+    return 0;
+}
+
 /* Run fuzzer and abort on failure. */
 static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) {
     int rv = fuzzer((const char*) data, size);
@@ -722,6 +860,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 #endif
 #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_pycompile)
     rv |= _run_fuzz(data, size, fuzz_pycompile);
+#endif
+#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_tokenizer)
+    static int TOKENIZER_INITIALIZED = 0;
+    if (!TOKENIZER_INITIALIZED && !init_tokenizer()) {
+        PyErr_Print();
+        abort();
+    }
+    TOKENIZER_INITIALIZED = 1;
+    rv |= _run_fuzz(data, size, fuzz_tokenizer);
 #endif
   return rv;
 }

Tools/README

@@ -47,6 +47,8 @@ scripts         A number of useful single-file programs, e.g. run_tests.py
 ssl             Scripts to generate ssl_data.h from OpenSSL sources, and run
                 tests against multiple installations of OpenSSL and LibreSSL.
 
+tokenizer       Tools for comparing tokenizer behavior between Python builds.
+
 tsan            Utilities for building CPython with thread-sanitizer.
 
 unicode         Tools for generating unicodedata and codecs from unicode.org

Tools/c-analyzer/cpython/ignored.tsv

@@ -652,13 +652,16 @@ Modules/_xxtestfuzz/fuzzer.c	-	xmlparser_type	-
 Modules/_xxtestfuzz/fuzzer.c	-	pycompile_scratch	-
 Modules/_xxtestfuzz/fuzzer.c	-	start_vals	-
 Modules/_xxtestfuzz/fuzzer.c	-	optimize_vals	-
+Modules/_xxtestfuzz/fuzzer.c	-	stringio_type	-
+Modules/_xxtestfuzz/fuzzer.c	-	tokenizer_iter_type	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	CSV_READER_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	JSON_LOADS_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	SRE_COMPILE_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	SRE_MATCH_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	STRUCT_UNPACK_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	AST_LITERAL_EVAL_INITIALIZED	-
 Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	ELEMENTTREE_PARSEWHOLE_INITIALIZED	-
+Modules/_xxtestfuzz/fuzzer.c	LLVMFuzzerTestOneInput	TOKENIZER_INITIALIZED	-
 
 ##-----------------------
 ## the analyzer should have ignored these