gh-153569: fix cross-platform tokenizer builds

Author: pablogsal

Lib/test/test_free_threading/test_tokenize.py

@@ -45,7 +45,7 @@ def next_token(it):
             tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line='  pass'),
             tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line='  pass'),
             tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line='  pass'),
-            tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line='  pass'),
+            tokenize.TokenInfo(type=0, string='', start=(3, 0), end=(3, 0), line=''),
         ]
 
         tokens.sort()

Makefile.pre.in

@@ -403,7 +403,7 @@ TOKENIZER_OBJS=		\
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
 		Parser/tokenizer/api.o \
-		Parser/tokenizer/errors.o \
+		Parser/tokenizer/tokenizer_errors.o \
 		Parser/tokenizer/helpers.o
 
 PEGEN_HEADERS= \

PCbuild/_freeze_module.vcxproj

@@ -188,7 +188,7 @@
     <ClCompile Include="..\Parser\tokenizer\api.c" />
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
-    <ClCompile Include="..\Parser\tokenizer\errors.c" />
+    <ClCompile Include="..\Parser\tokenizer\tokenizer_errors.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\helpers.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -481,7 +481,7 @@
     <ClCompile Include="..\Parser\tokenizer\decoder.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\tokenizer\errors.c">
+    <ClCompile Include="..\Parser\tokenizer\tokenizer_errors.c">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\reader.c">

PCbuild/pythoncore.vcxproj

@@ -596,7 +596,7 @@
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
-    <ClCompile Include="..\Parser\tokenizer\errors.c" />
+    <ClCompile Include="..\Parser\tokenizer\tokenizer_errors.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />
     <ClCompile Include="..\Parser\tokenizer\helpers.c" />
     <ClCompile Include="..\Parser\token.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1370,7 +1370,7 @@
     <ClCompile Include="..\Parser\tokenizer\decoder.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\tokenizer\errors.c">
+    <ClCompile Include="..\Parser\tokenizer\tokenizer_errors.c">
       <Filter>Parser</Filter>
     </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\reader.c">

Parser/tokenizer/reader_internal.h

@@ -37,7 +37,7 @@ typedef struct {
     _PyTok_ChunkOwnership ownership;
 } _PyTok_Chunk;
 
-typedef struct _PyTok_Reader {
+struct _PyTok_Reader {
     _PyTok_ReaderKind kind;
     PyObject *readline;
     PyObject *decoder;
@@ -63,7 +63,7 @@ typedef struct _PyTok_Reader {
     int decoder_finalized;
     int exhausted;
     int stopped;
-} _PyTok_Reader;
+};
 
 struct _PyTokenizer;
 

Parser/tokenizer/errors.c Parser/tokenizer/tokenizer_errors.cParser/tokenizer/errors.c renamed to Parser/tokenizer/tokenizer_errors.c

@@ -229,12 +229,12 @@ raise_syntax_error_object_at(PyTokenizer *tok, PyObject *type,
     PyObject *details;
     if (include_end) {
         details = Py_BuildValue(
-            "(OnnNnn)", tok->filename != NULL ? tok->filename : Py_None,
+            "(OinNin)", tok->filename != NULL ? tok->filename : Py_None,
             loc.lineno, col, text, end_loc.lineno, end_col);
     }
     else {
         details = Py_BuildValue(
-            "(OnnNOO)", tok->filename != NULL ? tok->filename : Py_None,
+            "(OinNOO)", tok->filename != NULL ? tok->filename : Py_None,
             loc.lineno, col, text, Py_None, Py_None);
     }
     if (details == NULL) {

Tools/peg_generator/pegen/build.py

@@ -133,7 +133,7 @@ def compile_c_extension(
         str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
         str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "source.c"),
         str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c"),
-        str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "errors.c"),
+        str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "tokenizer_errors.c"),
         str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
         str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
         str(MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c"),