Merge branch 'gh-153569-tokenizer-reader-cutover' into gh-153569-tokenizer-offset-state

# Conflicts:
#	Parser/tokenizer/decoder.c
#	Parser/tokenizer/reader.c
#	Parser/tokenizer/reader_internal.h

Author: pablogsal

Parser/tokenizer/decoder.c

@@ -279,10 +279,10 @@ _PyTok_DetectEncoding(struct _PyTokenizer *tok, const _PyTok_Chunk *first,
 
 int
 _PyTok_DecodeOnce(struct _PyTokenizer *tok, _PyTok_Chunk *chunk,
-                  const char *encoding)
+                  const char *encoding, const char *errors)
 {
     PyObject *unicode = PyUnicode_Decode(
-        chunk->data, chunk->len, encoding, NULL);
+        chunk->data, chunk->len, encoding, errors);
     if (unicode == NULL) {
         _PyTok_RecordPending(tok, _PYTOK_ERR_DECODE);
         return -1;
@@ -399,7 +399,8 @@ _PyTok_PrepareString(struct _PyTokenizer *tok, const char *input,
         .ownership = _PYTOK_CHUNK_BORROWED,
     };
     if (tok->reader->encoding != NULL && strcmp(tok->reader->encoding, "utf-8") != 0) {
-        if (_PyTok_DecodeOnce(tok, &decoded, tok->reader->encoding) < 0) {
+        if (_PyTok_DecodeOnce(
+                tok, &decoded, tok->reader->encoding, NULL) < 0) {
             return -1;
         }
     }

Parser/tokenizer/reader.c

@@ -376,15 +376,24 @@ next_readline(struct _PyTokenizer *tok, _PyTok_Chunk *chunk)
                 Py_DECREF(raw);
                 return _PYTOK_READ_ERROR;
             }
-            if (_PyTok_StartDecoder(tok, "replace") < 0) {
-                Py_DECREF(raw);
-                return _PYTOK_READ_ERROR;
-            }
             input.owner = raw;
             input.data = PyBytes_AS_STRING(raw);
             input.len = PyBytes_GET_SIZE(raw);
             input.ownership = _PYTOK_CHUNK_PYOBJECT;
-            if (input.len > 0 && _PyTok_DecodeChunk(tok, &input, 0) < 0) {
+            int decoded;
+            if (reader->decoder == NULL &&
+                    strcmp(reader->encoding, "utf-8") == 0 &&
+                    chunk_is_line(&input)) {
+                decoded = _PyTok_DecodeOnce(
+                    tok, &input, "utf-8", "replace");
+            }
+            else {
+                decoded = _PyTok_StartDecoder(tok, "replace");
+                if (decoded == 0 && input.len > 0) {
+                    decoded = _PyTok_DecodeChunk(tok, &input, 0);
+                }
+            }
+            if (decoded < 0) {
                 _PyTok_ChunkClear(&input);
                 return _PYTOK_READ_ERROR;
             }
@@ -468,7 +477,8 @@ next_interactive(struct _PyTokenizer *tok, _PyTok_Chunk *chunk)
         .ownership = _PYTOK_CHUNK_PYMEM,
     };
     if (reader->encoding != NULL &&
-            _PyTok_DecodeOnce(tok, &decoded, reader->encoding) < 0) {
+            _PyTok_DecodeOnce(
+                tok, &decoded, reader->encoding, NULL) < 0) {
         _PyTok_ChunkClear(&decoded);
         return _PYTOK_READ_ERROR;
     }

Parser/tokenizer/reader_internal.h

@@ -68,7 +68,8 @@ struct _PyTok_Reader {
 struct _PyTokenizer;
 
 char *_PyTok_CopyBytes(const char *, Py_ssize_t);
-int _PyTok_DecodeOnce(struct _PyTokenizer *, _PyTok_Chunk *, const char *);
+int _PyTok_DecodeOnce(
+    struct _PyTokenizer *, _PyTok_Chunk *, const char *, const char *);
 char *_PyTok_NormalizeNewlines(
     const char *, Py_ssize_t, int, int, Py_ssize_t *, int *);
 void _PyTok_ChunkClear(_PyTok_Chunk *);