gh-153569: speed up UTF-8 readline decoding

Author: pablogsal

Parser/tokenizer/decoder.c

@@ -267,10 +267,10 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
 
 int
 _PyTok_DecodeOnce(struct tok_state *tok, _PyTok_Chunk *chunk,
-                  const char *encoding)
+                  const char *encoding, const char *errors)
 {
     PyObject *unicode = PyUnicode_Decode(
-        chunk->data, chunk->len, encoding, NULL);
+        chunk->data, chunk->len, encoding, errors);
     if (unicode == NULL) {
         tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
             ? E_NOMEM : E_DECODE;
@@ -387,7 +387,8 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
         .ownership = _PYTOK_CHUNK_BORROWED,
     };
     if (tok->encoding != NULL && strcmp(tok->encoding, "utf-8") != 0) {
-        if (_PyTok_DecodeOnce(tok, &decoded, tok->encoding) < 0) {
+        if (_PyTok_DecodeOnce(
+                tok, &decoded, tok->encoding, NULL) < 0) {
             return -1;
         }
     }

Parser/tokenizer/reader.c

@@ -381,12 +381,11 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
                 Py_DECREF(raw);
                 return _PYTOK_READ_ERROR;
             }
-            if (_PyTok_StartDecoder(tok, "replace") < 0) {
-                Py_XDECREF(raw);
-                return _PYTOK_READ_ERROR;
-            }
             if (PyBytes_GET_SIZE(raw) == 0) {
                 Py_DECREF(raw);
+                if (_PyTok_StartDecoder(tok, "replace") < 0) {
+                    return _PYTOK_READ_ERROR;
+                }
                 if (finalize_decoding(tok) < 0) {
                     return _PYTOK_READ_ERROR;
                 }
@@ -396,7 +395,20 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
             input.data = PyBytes_AS_STRING(raw);
             input.len = PyBytes_GET_SIZE(raw);
             input.ownership = _PYTOK_CHUNK_PYOBJECT;
-            if (_PyTok_DecodeChunk(tok, &input, 0) < 0) {
+            int decoded;
+            if (reader->decoder == NULL &&
+                    strcmp(tok->encoding, "utf-8") == 0 &&
+                    chunk_is_line(&input)) {
+                decoded = _PyTok_DecodeOnce(
+                    tok, &input, "utf-8", "replace");
+            }
+            else {
+                decoded = _PyTok_StartDecoder(tok, "replace");
+                if (decoded == 0) {
+                    decoded = _PyTok_DecodeChunk(tok, &input, 0);
+                }
+            }
+            if (decoded < 0) {
                 _PyTok_ChunkClear(&input);
                 return _PYTOK_READ_ERROR;
             }
@@ -482,7 +494,8 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
         .ownership = _PYTOK_CHUNK_PYMEM,
     };
     if (tok->encoding != NULL &&
-            _PyTok_DecodeOnce(tok, &decoded, tok->encoding) < 0) {
+            _PyTok_DecodeOnce(
+                tok, &decoded, tok->encoding, NULL) < 0) {
         _PyTok_ChunkClear(&decoded);
         return _PYTOK_READ_ERROR;
     }

Parser/tokenizer/reader_internal.h

@@ -64,7 +64,8 @@ typedef struct _PyTok_Reader {
 struct tok_state;
 
 char *_PyTok_CopyBytes(const char *, Py_ssize_t);
-int _PyTok_DecodeOnce(struct tok_state *, _PyTok_Chunk *, const char *);
+int _PyTok_DecodeOnce(
+    struct tok_state *, _PyTok_Chunk *, const char *, const char *);
 char *_PyTok_NormalizeNewlines(
     const char *, Py_ssize_t, int, int, Py_ssize_t *, int *);
 void _PyTok_ChunkClear(_PyTok_Chunk *);