From 0fc5ebc7a1b7475e746e34204dbc9ad386a3b030 Mon Sep 17 00:00:00 2001 From: Bhuvansh Date: Tue, 15 Sep 2026 17:11:44 +0530 Subject: [PATCH 01/11] gh-155875: Fix use-after-free in curses after new_prescr() (GH-155914) initscr() and newterm() adopt the screen created by new_prescr(), so the screen object returned by new_prescr() no longer owns it after that, and new_prescr() returns the same object while the screen is pending. --- Lib/test/test_curses.py | 38 ++++++++++++++++++++++++++++++++++++++ Modules/_cursesmodule.c | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index d779955e236228..9cc6eee266bdd8 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -3481,6 +3481,44 @@ def test_use_prescr_screen(self): # The current screen is unchanged. screen.stdscr.refresh() + @unittest.skipUnless(hasattr(curses, 'new_prescr'), + 'requires curses.new_prescr()') + def test_new_prescr_returns_existing_screen(self): + pre1 = curses.new_prescr() + pre2 = curses.new_prescr() + self.assertIs(pre1, pre2) + + @unittest.skipUnless(hasattr(curses, 'new_prescr'), + 'requires curses.new_prescr()') + def test_newterm_after_new_prescr_keeps_screen_alive(self): + # newterm() adopts the SCREEN created by new_prescr(). Dropping the + # pre-screen wrapper must not delete the live screen. + s = self.make_pty() + pre = curses.new_prescr() + screen = curses.newterm('xterm', s, s) + del pre + gc_collect() + screen.stdscr.addstr(0, 0, 'x') + screen.stdscr.refresh() + + @unittest.skipUnless(hasattr(curses, 'new_prescr'), + 'requires curses.new_prescr()') + def test_initscr_after_new_prescr_keeps_screen_alive(self): + # initscr() adopts the SCREEN created by new_prescr(). Dropping the + # pre-screen wrapper must not delete the live screen. + s = self.make_pty() + saved = os.dup(1) + self.addCleanup(os.close, saved) + self.addCleanup(os.dup2, saved, 1) + os.dup2(s, 1) + + pre = curses.new_prescr() + stdscr = curses.initscr() + del pre + gc_collect() + stdscr.addstr(0, 0, 'x') + stdscr.refresh() + def test_initscr_after_newterm_keeps_screen_alive(self): # initscr() called while a newterm() screen is current returns that # screen's own standard window, so the window keeps the screen alive. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 24cfbcdd503cee..fd65cbb0bf67a8 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -159,6 +159,8 @@ typedef struct { PyTypeObject *complexstr_type; // _curses.complexstr PyObject *topscreen; // owned ref to the current screen object, // or NULL for the initscr() screen + PyObject *prescreen; // owned ref to the pending new_prescr() screen, + // or NULL if there is no pending pre-screen } cursesmodule_state; static inline cursesmodule_state * @@ -6979,13 +6981,21 @@ _curses_initscr_impl(PyObject *module) return NULL; } + cursesmodule_state *state = get_cursesmodule_state(module); + if (state->prescreen != NULL) { + PyCursesScreenObject *prescreen = + _PyCursesScreenObject_CAST(state->prescreen); + assert(prescreen->screen != NULL); + prescreen->screen = NULL; + Py_CLEAR(state->prescreen); + } + curses_initscr_called = curses_setupterm_called = TRUE; if (curses_init_dict(module) < 0) { return NULL; } - cursesmodule_state *state = get_cursesmodule_state(module); PyObject *winobj = PyCursesWindow_New(state, win, NULL, NULL, NULL); if (winobj == NULL) { return NULL; @@ -7161,6 +7171,13 @@ _curses_newterm_impl(PyObject *module, const char *type, PyObject *fd, cursesmodule_state *state = get_cursesmodule_state(module); /* The screen object owns the SCREEN and the streams; deleting it (when it is no longer referenced) calls delscreen() and closes the streams. */ + if (state->prescreen != NULL) { + PyCursesScreenObject *prescreen = + _PyCursesScreenObject_CAST(state->prescreen); + assert(prescreen->screen == screen); + prescreen->screen = NULL; + Py_CLEAR(state->prescreen); + } PyObject *screenobj = PyCursesScreen_New(state, screen, outfp, infp, NULL); if (screenobj == NULL) { delscreen(screen); @@ -7252,13 +7269,25 @@ static PyObject * _curses_new_prescr_impl(PyObject *module) /*[clinic end generated code: output=e7de5031da7511e2 input=1a3a89d630b641c3]*/ { + cursesmodule_state *state = get_cursesmodule_state(module); + if (state->prescreen != NULL) { + return Py_NewRef(state->prescreen); + } + SCREEN *screen = new_prescr(); if (screen == NULL) { curses_set_null_error(module, "new_prescr", NULL); return NULL; } - cursesmodule_state *state = get_cursesmodule_state(module); - return PyCursesScreen_New(state, screen, NULL, NULL, NULL); + + PyObject *screenobj = PyCursesScreen_New(state, screen, NULL, NULL, NULL); + if (screenobj == NULL) { + delscreen(screen); + return NULL; + } + + state->prescreen = Py_NewRef(screenobj); + return screenobj; } #endif /* HAVE_CURSES_NEW_PRESCR */ @@ -9262,6 +9291,7 @@ cursesmodule_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->complexchar_type); Py_VISIT(state->complexstr_type); Py_VISIT(state->topscreen); + Py_VISIT(state->prescreen); return 0; } @@ -9275,6 +9305,7 @@ cursesmodule_clear(PyObject *mod) Py_CLEAR(state->complexchar_type); Py_CLEAR(state->complexstr_type); Py_CLEAR(state->topscreen); + Py_CLEAR(state->prescreen); return 0; } From 237e1dda035485022a56d3f396ceff5f16fed301 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2026 14:32:26 +0200 Subject: [PATCH 02/11] gh-155742: Use PyBytesWriter in CJK codecs (#157535) Replace soft deprecated PyBytes_FromStringAndSize() with PyBytesWriter. Replace PyBytes_FromStringAndSize(NULL, 0) with Py_GetConstant(Py_CONSTANT_EMPTY_BYTES). --- Modules/cjkcodecs/multibytecodec.c | 40 +++++++++++++----------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5eb1533bdfc118..5fb3ec45a38875 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -82,7 +82,8 @@ typedef struct { PyObject *inobj; Py_ssize_t inpos, inlen; unsigned char *outbuf, *outbuf_end; - PyObject *excobj, *outobj; + PyObject *excobj; + PyBytesWriter *writer; } MultibyteEncodeBuffer; typedef struct { @@ -209,8 +210,8 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) Py_ssize_t orgpos, orgsize, incsize; orgpos = (Py_ssize_t)((char *)buf->outbuf - - PyBytes_AS_STRING(buf->outobj)); - orgsize = PyBytes_GET_SIZE(buf->outobj); + (char *)PyBytesWriter_GetData(buf->writer)); + orgsize = PyBytesWriter_GetSize(buf->writer); incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); if (orgsize > PY_SSIZE_T_MAX - incsize) { @@ -218,12 +219,12 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) return -1; } - if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1) + if (PyBytesWriter_Resize(buf->writer, orgsize + incsize) == -1) return -1; - buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos; - buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj) - + PyBytes_GET_SIZE(buf->outobj); + unsigned char *data = PyBytesWriter_GetData(buf->writer); + buf->outbuf = data + orgpos; + buf->outbuf_end = data + PyBytesWriter_GetSize(buf->writer); return 0; } @@ -503,7 +504,7 @@ multibytecodec_encode(const MultibyteCodec *codec, PyObject *errors, int flags) { MultibyteEncodeBuffer buf; - Py_ssize_t finalsize, r = 0; + Py_ssize_t r = 0; Py_ssize_t datalen; int kind; const void *data; @@ -511,10 +512,10 @@ multibytecodec_encode(const MultibyteCodec *codec, datalen = PyUnicode_GET_LENGTH(text); if (datalen == 0 && !(flags & MBENC_RESET)) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); buf.excobj = NULL; - buf.outobj = NULL; + buf.writer = NULL; buf.inobj = text; /* borrowed reference */ buf.inpos = 0; buf.inlen = datalen; @@ -526,11 +527,11 @@ multibytecodec_encode(const MultibyteCodec *codec, goto errorexit; } - buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16); - if (buf.outobj == NULL) + buf.writer = PyBytesWriter_Create(datalen * 2 + 16); + if (buf.writer == NULL) goto errorexit; - buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj); - buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj); + buf.outbuf = (unsigned char *)PyBytesWriter_GetData(buf.writer); + buf.outbuf_end = buf.outbuf + PyBytesWriter_GetSize(buf.writer); while (buf.inpos < buf.inlen) { /* we don't reuse inleft and outleft here. @@ -563,21 +564,14 @@ multibytecodec_encode(const MultibyteCodec *codec, goto errorexit; } - finalsize = (Py_ssize_t)((char *)buf.outbuf - - PyBytes_AS_STRING(buf.outobj)); - - if (finalsize != PyBytes_GET_SIZE(buf.outobj)) - if (_PyBytes_Resize(&buf.outobj, finalsize) == -1) - goto errorexit; - if (inpos_t) *inpos_t = buf.inpos; Py_XDECREF(buf.excobj); - return buf.outobj; + return PyBytesWriter_FinishWithPointer(buf.writer, buf.outbuf); errorexit: Py_XDECREF(buf.excobj); - Py_XDECREF(buf.outobj); + PyBytesWriter_Discard(buf.writer); return NULL; } From 2fcb0e27d959345151750b756af79054c3230531 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2026 14:35:59 +0200 Subject: [PATCH 03/11] gh-156939: Document that PyBytesObject ends with a NUL byte (#157236) Document as an implementation detail that PyBytesObject and ByteArrayObject end with a NUL byte. Co-authored-by: Petr Viktorin --- Doc/c-api/bytearray.rst | 6 ++++++ Doc/c-api/bytes.rst | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 2b36da997d4295..8f3cb2d63cb3c8 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -12,6 +12,12 @@ Byte Array Objects This subtype of :c:type:`PyObject` represents a Python bytearray object. + .. impl-detail:: + + The internal buffer of :c:type:`PyByteArrayObject` always includes an + extra trailing null byte for compatibility with null terminated C + strings. This extra byte is not counted in :c:func:`PyByteArray_Size` + nor in the *len* arguments of the functions below. .. c:var:: PyTypeObject PyByteArray_Type diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 60a90b3f096912..72f4a2829d89c7 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -8,6 +8,13 @@ Bytes Objects These functions raise :exc:`TypeError` when expecting a bytes parameter and called with a non-bytes parameter. +.. impl-detail:: + + The internal buffer of :c:type:`PyBytesObject` always includes an extra + trailing null byte for compatibility with null terminated C strings. + This extra byte is not counted in :c:func:`PyBytes_Size` nor in the + various *length* and *size* arguments of the functions below. + .. index:: pair: object; bytes From cbb43b830e01fb558c28e69af3980a046025b6df Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 15 Sep 2026 15:24:28 +0200 Subject: [PATCH 04/11] gh-140550: Fix PyABIInfo_VAR macro: avoid ";" (#157539) Co-authored-by: Victor Stinner --- Include/modsupport.h | 2 +- .../next/C_API/2026-09-14-18-07-43.gh-issue-140550.fLVOGn.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-14-18-07-43.gh-issue-140550.fLVOGn.rst diff --git a/Include/modsupport.h b/Include/modsupport.h index cb47ad8cd2727f..504a19d341a78a 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -140,7 +140,7 @@ PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); ///////////////////////////////////////////////////////// #define PyABIInfo_VAR(NAME) \ - static PyABIInfo NAME = _PyABIInfo_DEFAULT; + static PyABIInfo NAME = _PyABIInfo_DEFAULT #undef _PyABIInfo_DEFAULT_STABLE #undef _PyABIInfo_DEFAULT_FT diff --git a/Misc/NEWS.d/next/C_API/2026-09-14-18-07-43.gh-issue-140550.fLVOGn.rst b/Misc/NEWS.d/next/C_API/2026-09-14-18-07-43.gh-issue-140550.fLVOGn.rst new file mode 100644 index 00000000000000..146ea1968a0816 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-14-18-07-43.gh-issue-140550.fLVOGn.rst @@ -0,0 +1,3 @@ +Fix the :c:macro:`PyABIInfo_VAR` macro: remove redundant ``;``. Before, +``PyABIInfo_VAR(abi_info);`` code added two ``;;`` which is illegal in C++03. +Patch by Victor Stinner. From d95f29589e03603aa13d8ca9d4f817dce77d357c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:50:40 +0300 Subject: [PATCH 05/11] gh-148603: Update docs now UTF-8 is default (PEP 686) (#157372) --- Doc/builtins/functions.rst | 19 +++++++++++----- Doc/library/csv.rst | 17 +++++++------- Doc/library/io.rst | 43 +++++++++++++----------------------- Doc/tutorial/inputoutput.rst | 30 +++++++++++-------------- Doc/using/windows.rst | 9 ++++---- 5 files changed, 54 insertions(+), 64 deletions(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 67893e670fdba7..5cce5e3c87628a 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -1,7 +1,7 @@ .. XXX document all delegations to __special__ methods .. _built-in-funcs: -Built-in Functions +Built-in functions ================== The Python interpreter has a number of functions and types built into it that @@ -1459,7 +1459,8 @@ are always available. They are listed here in alphabetical order. already exists), ``'x'`` for exclusive creation, and ``'a'`` for appending (which on *some* Unix systems, means that *all* writes append to the end of the file regardless of the current seek position). In text mode, if - *encoding* is not specified the encoding used is platform-dependent: + *encoding* is not specified, UTF-8 is used by default; if + :ref:`Python UTF-8 Mode ` is disabled, :func:`locale.getencoding` is called to get the current locale encoding. (For reading and writing raw bytes use binary mode and leave *encoding* unspecified.) The available modes are: @@ -1490,7 +1491,7 @@ are always available. They are listed here in alphabetical order. argument) return contents as :class:`bytes` objects without any decoding. In text mode (the default, or when ``'t'`` is included in the *mode* argument), the contents of the file are returned as :class:`str`, the bytes having been - first decoded using a platform-dependent encoding or using the specified + first decoded using the default encoding or using the specified *encoding* if given. .. note:: @@ -1519,9 +1520,11 @@ are always available. They are listed here in alphabetical order. described above for binary files. *encoding* is the name of the encoding used to decode or encode the file. - This should only be used in text mode. The default encoding is platform - dependent (whatever :func:`locale.getencoding` returns), but any - :term:`text encoding` supported by Python can be used. + This should only be used in text mode. The default encoding is UTF-8; + if :ref:`Python UTF-8 Mode ` is disabled, the default is + platform-dependent (whatever :func:`locale.getencoding` returns). + Any :term:`text encoding` supported by Python can be used, and + ``encoding="locale"`` specifies the current locale encoding explicitly. See the :mod:`codecs` module for the list of supported encodings. *errors* is an optional string that specifies how encoding and decoding @@ -1638,6 +1641,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. + .. versionchanged:: 3.15 + UTF-8 is now the default encoding, instead of the + platform-dependent locale encoding (:pep:`686`). + .. function:: ord(character, /) Return the ordinal value of a character. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 53288e810bffcf..869c6a5f96a1f2 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -1,4 +1,4 @@ -:mod:`!csv` --- CSV File Reading and Writing +:mod:`!csv` --- CSV file reading and writing ============================================ .. module:: csv @@ -42,7 +42,7 @@ using the :class:`DictReader` and :class:`DictWriter` classes. .. _csv-contents: -Module Contents +Module contents --------------- The :mod:`!csv` module defines the following functions: @@ -451,7 +451,7 @@ The :mod:`!csv` module defines the following exception: .. _csv-fmt-params: -Dialects and Formatting Parameters +Dialects and formatting parameters ---------------------------------- To make it easier to specify the format of input and output records, specific @@ -557,7 +557,7 @@ with the specified formatting parameters replaced. .. _reader-objects: -Reader Objects +Reader objects -------------- Reader objects (:class:`DictReader` instances and objects returned by the @@ -594,7 +594,7 @@ DictReader objects have the following public attribute: -Writer Objects +Writer objects -------------- :class:`writer` objects (:class:`DictWriter` instances and objects returned by @@ -673,17 +673,16 @@ The corresponding simplest possible writing example is:: writer.writerows(someiterable) Since :func:`open` is used to open a CSV file for reading, the file -will by default be decoded into unicode using the system default -encoding (see :func:`locale.getencoding`). To decode a file +will by default be decoded into Unicode using UTF-8. To decode a file using a different encoding, use the ``encoding`` argument of open:: import csv - with open('some.csv', newline='', encoding='utf-8') as f: + with open('some.csv', newline='', encoding='latin-1') as f: reader = csv.reader(f) for row in reader: print(row) -The same applies to writing in something other than the system default +The same applies to writing in something other than the default encoding: specify the encoding argument when opening the output file. Registering a new dialect:: diff --git a/Doc/library/io.rst b/Doc/library/io.rst index ecaa053b4e18b9..635a47ebeaba4a 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -115,34 +115,21 @@ The raw stream API is described in detail in the docs of :class:`RawIOBase`. .. _io-text-encoding: -Text Encoding +Text encoding ------------- -The default encoding of :class:`TextIOWrapper` and :func:`open` is -locale-specific (:func:`locale.getencoding`). - -However, many developers forget to specify the encoding when opening text files -encoded in UTF-8 (e.g. JSON, TOML, Markdown, etc...) since most Unix -platforms use UTF-8 locale by default. This causes bugs because the locale -encoding is not UTF-8 for most Windows users. For example:: - - # May not work on Windows when non-ASCII characters in the file. - with open("README.md") as f: - long_description = f.read() - -Accordingly, it is highly recommended that you specify the encoding -explicitly when opening text files. If you want to use UTF-8, pass -``encoding="utf-8"``. To use the current locale encoding, -``encoding="locale"`` is supported since Python 3.10. +The default encoding of :class:`TextIOWrapper` and :func:`open` is UTF-8. +If :ref:`Python UTF-8 Mode ` is disabled, the default encoding +is locale-specific (:func:`locale.getencoding`). .. seealso:: :ref:`utf8-mode` - Python UTF-8 Mode can be used to change the default encoding to - UTF-8 from locale-specific encoding. + Python UTF-8 Mode ignores the locale encoding and forces the use + of UTF-8. :pep:`686` - Python 3.15 will make :ref:`utf8-mode` default. + Python 3.15 made :ref:`utf8-mode` the default. .. _io-encoding-warning: @@ -152,7 +139,7 @@ Opt-in EncodingWarning .. versionadded:: 3.10 See :pep:`597` for more details. -To find where the default locale encoding is used, you can enable +To find where the default encoding is used, you can enable the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -165,7 +152,7 @@ please consider using UTF-8 by default (i.e. ``encoding="utf-8"``) for new APIs. -High-level Module Interface +High-level module interface --------------------------- .. data:: DEFAULT_BUFFER_SIZE @@ -315,7 +302,7 @@ ABC Inherits Stub Methods Mixin M ========================= ================== ======================== ================================================== -I/O Base Classes +I/O base classes ^^^^^^^^^^^^^^^^ .. class:: IOBase @@ -660,7 +647,7 @@ I/O Base Classes so the implementation should only access *b* during the method call. -Raw File I/O +Raw file I/O ^^^^^^^^^^^^ .. class:: FileIO(name, mode='r', closefd=True, opener=None) @@ -728,7 +715,7 @@ Raw File I/O given in the constructor. -Buffered Streams +Buffered streams ^^^^^^^^^^^^^^^^ Buffered I/O streams provide a higher-level interface to an I/O device @@ -1004,8 +991,8 @@ Text I/O :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or - encoded with. In :ref:`UTF-8 Mode `, this defaults to UTF-8. - Otherwise, it defaults to :func:`locale.getencoding`. + encoded with. This defaults to UTF-8; if :ref:`UTF-8 Mode ` is + disabled, it defaults to :func:`locale.getencoding`. ``encoding="locale"`` can be used to specify the current locale's encoding explicitly. See :ref:`io-text-encoding` for more information. @@ -1187,7 +1174,7 @@ Text I/O It inherits from :class:`codecs.IncrementalDecoder`. -Static Typing +Static typing ------------- The following protocols can be used for annotating function and method diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index a00f06cf46c41a..4caaecfd013555 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -1,7 +1,7 @@ .. _tut-io: **************** -Input and Output +Input and output **************** There are several ways to present the output of a program; data can be printed @@ -11,7 +11,7 @@ discuss some of the possibilities. .. _tut-formatting: -Fancier Output Formatting +Fancier output formatting ========================= So far we've encountered two ways of writing values: *expression statements* and @@ -111,7 +111,7 @@ This syntax is easy to use, although it offers much less control for formatting. .. _tut-f-strings: -Formatted String Literals +Formatted string literals ------------------------- :ref:`Formatted string literals ` (also called f-strings for @@ -163,7 +163,7 @@ the reference guide for the :ref:`formatspec`. .. _tut-string-format: -The String format() Method +The string format() method -------------------------- Basic usage of the :meth:`str.format` method looks like this:: @@ -240,7 +240,7 @@ For a complete overview of string formatting with :meth:`str.format`, see :ref:`formatstrings`. -Manual String Formatting +Manual string formatting ------------------------ Here's the same table of squares and cubes, formatted manually:: @@ -303,7 +303,7 @@ More information can be found in the :ref:`old-string-formatting` section. .. _tut-files: -Reading and Writing Files +Reading and writing files ========================= .. index:: @@ -311,12 +311,11 @@ Reading and Writing Files pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with -two positional arguments and one keyword argument: -``open(filename, mode, encoding=None)`` +two positional arguments: ``open(filename, mode)`` :: - >>> f = open('workfile', 'w', encoding="utf-8") + >>> f = open('workfile', 'w') .. XXX str(f) is @@ -334,10 +333,7 @@ omitted. Normally, files are opened in :dfn:`text mode`, that means, you read and write strings from and to the file, which are encoded in a specific *encoding*. -If *encoding* is not specified, the default is platform dependent -(see :func:`open`). -Because UTF-8 is the modern de-facto standard, ``encoding="utf-8"`` is -recommended unless you know that you need to use a different encoding. +If *encoding* is not specified, the default is UTF-8 (see :func:`open`). Appending a ``'b'`` to the mode opens the file in :dfn:`binary mode`. Binary mode data is read and written as :class:`bytes` objects. You can not specify *encoding* when opening file in binary mode. @@ -356,7 +352,7 @@ after its suite finishes, even if an exception is raised at some point. Using :keyword:`!with` is also much shorter than writing equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - >>> with open('workfile', encoding="utf-8") as f: + >>> with open('workfile') as f: ... read_data = f.read() >>> # We can check that the file has been automatically closed. @@ -389,7 +385,7 @@ automatically fail. :: .. _tut-filemethods: -Methods of File Objects +Methods of file objects ----------------------- The rest of the examples in this section will assume that a file object called @@ -532,8 +528,8 @@ To decode the object again, if ``f`` is a :term:`binary file` or x = json.load(f) .. note:: - JSON files must be encoded in UTF-8. Use ``encoding="utf-8"`` when opening - JSON file as a :term:`text file` for both of reading and writing. + JSON files must be encoded in UTF-8, the default encoding for + :term:`text files `. This simple serialization technique can handle lists and dictionaries, but serializing arbitrary class instances in JSON requires a bit of extra effort. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 8a7c0e921a1821..baa68f7af8aa40 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -1345,14 +1345,15 @@ UTF-8 mode Python UTF-8 mode is now enabled by default (:pep:`686`). Windows still uses legacy encodings for the system encoding (the ANSI Code -Page). Python uses it for the default encoding of text files (e.g. -:func:`locale.getencoding`). +Page). When the :ref:`Python UTF-8 Mode ` is disabled, Python +uses the ANSI Code Page as the default encoding of text files, as +returned by :func:`locale.getencoding`. This may cause issues because UTF-8 is widely used on the internet and most Unix systems, including WSL (Windows Subsystem for Linux). -The :ref:`Python UTF-8 Mode `, enabled by default, can help by -changing the default text encoding to UTF-8. +The :ref:`Python UTF-8 Mode `, enabled by default, ignores the +system encoding and uses UTF-8 as the default text encoding. When the :ref:`UTF-8 mode ` is enabled, you can still use the system encoding (the ANSI Code Page) via the "mbcs" codec. From 2d1007f931963ef907460ddaa110bac4e635eebd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2026 16:34:17 +0200 Subject: [PATCH 06/11] gh-156939: Detect buffer overflow in bytes and bytearray (#157529) When Python is built in debug mode, bytes an bytearray destructors now check if the trailing null byte has been overridden to detect overflow. Add bytes_dealloc() to implement the check. Add _PyBytes_CheckOverflow() to share code --- Include/internal/pycore_bytesobject.h | 7 ++ Lib/test/test_capi/test_bytearray.py | 23 +++++ Lib/test/test_capi/test_bytes.py | 20 +++++ ...-09-14-23-16-26.gh-issue-156939.oZqlSt.rst | 3 + Modules/_testcapi/bytes.c | 41 +++++++++ Modules/_testcapi/mem.c | 2 + Objects/bytearrayobject.c | 6 ++ Objects/bytesobject.c | 88 ++++++++++++++----- 8 files changed, 168 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-14-23-16-26.gh-issue-156939.oZqlSt.rst diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 443bdb26ff8738..8f764f0fa6d6e1 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -81,6 +81,13 @@ extern int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize); extern int _PyBytes_IsMutable(PyObject *obj); #endif +#ifdef Py_DEBUG +extern void _PyBytes_CheckOverflow( + PyObject *op, + void *addr, + const char *type_name); +#endif + /* --- PyBytesWriter ------------------------------------------------------ */ struct PyBytesWriter { diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index cb7ad8b22252d9..638a29f026cd97 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,6 +1,9 @@ import sys +import textwrap import unittest +from test import support from test.support import import_helper +from test.support.script_helper import assert_python_failure _testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX @@ -172,6 +175,26 @@ def test_resize(self): # CRASHES resize(object(), 0) # CRASHES resize(NULL, 0) + @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') + def test_detect_overflow(self): + # Test detection of buffer overflow + size = 123 # bytes + overflow = 1 # bytes + code = textwrap.dedent(f''' + from test.support import SuppressCrashReport + import _testcapi + + size = {size} + overflow = {overflow} + with SuppressCrashReport(): + # Trigger a buffer overflow in a new bytearray + ba = _testcapi.bytearray_overflow(size, overflow) + ba = None + ''') + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in bytearray object', proc.err) + self.assertIn(f'at position {size}'.encode(), proc.err) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index a0006ea35e21fe..12a1e88eac82d9 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -317,6 +317,26 @@ def test_join(self): with self.assertRaises(SystemError): bytes_join(b'', NULL) + @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') + def test_detect_overflow(self): + # Test detection of buffer overflow + size = 123 # bytes + overflow = 1 # bytes + code = textwrap.dedent(f''' + from test.support import SuppressCrashReport + import _testcapi + + size = {size} + overflow = {overflow} + with SuppressCrashReport(): + # Trigger a buffer overflow in a new bytes + ba = _testcapi.bytes_overflow(size, overflow) + ba = None + ''') + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in bytes object', proc.err) + self.assertIn(f'at position {size}'.encode(), proc.err) + def get_data_canary(writer): size = writer.get_size() + 1 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-14-23-16-26.gh-issue-156939.oZqlSt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-14-23-16-26.gh-issue-156939.oZqlSt.rst new file mode 100644 index 00000000000000..212679fc0a79bf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-14-23-16-26.gh-issue-156939.oZqlSt.rst @@ -0,0 +1,3 @@ +When Python is built in debug mode, :class:`bytes` and :class:`bytearray` +destructors now check if the trailing null byte has been overridden to detect +buffer overflow. Patch by Victor Stinner. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 79effcad40090e..e3d966b3bb1896 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -528,6 +528,45 @@ test_byteswriter_ptr(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +bytes_overflow(PyObject *Py_UNUSED(module), PyObject *args) +{ + Py_ssize_t alloc, overflow = 1; + if (!PyArg_ParseTuple(args, "n|n", &alloc, &overflow)) + return NULL; + + PyObject *bytes = PyObject_CallFunction((PyObject*)&PyBytes_Type, "n", alloc); + if (bytes == NULL) { + return NULL; + } + + char *data = PyBytes_AS_STRING(bytes); + Py_ssize_t size = PyBytes_GET_SIZE(bytes); + memset(data, 'x', size); + memset(data + size, '#', overflow); // Buffer overflow! + return bytes; +} + + +static PyObject * +bytearray_overflow(PyObject *Py_UNUSED(module), PyObject *args) +{ + Py_ssize_t alloc, overflow = 1; + if (!PyArg_ParseTuple(args, "n|n", &alloc, &overflow)) + return NULL; + + PyObject *bytearray = PyObject_CallFunction((PyObject*)&PyByteArray_Type, "n", alloc); + if (bytearray == NULL) { + return NULL; + } + + char *data = PyByteArray_AS_STRING(bytearray); + Py_ssize_t size = PyByteArray_GET_SIZE(bytearray); + memset(data + size, '#', overflow); // Buffer overflow! + return bytearray; +} + + static PyMethodDef test_methods[] = { {"bytes_resize", bytes_resize, METH_VARARGS}, {"bytes_join", bytes_join, METH_VARARGS}, @@ -535,6 +574,8 @@ static PyMethodDef test_methods[] = { {"byteswriter_resize", byteswriter_resize, METH_NOARGS}, {"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS}, {"test_byteswriter_ptr", test_byteswriter_ptr, METH_NOARGS}, + {"bytes_overflow", bytes_overflow, METH_VARARGS}, + {"bytearray_overflow", bytearray_overflow, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 4ae6a60ff39d15..ba1462481231b5 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -448,6 +448,7 @@ test_pyobject_new(PyObject *self, PyObject *Py_UNUSED(ignored)) if (obj == NULL) { goto alloc_failed; } + memset(PyBytes_AS_STRING(obj), 0, 3 + 1); // +1 for the null byte Py_DECREF(obj); // PyObject_NEW_VAR() @@ -455,6 +456,7 @@ test_pyobject_new(PyObject *self, PyObject *Py_UNUSED(ignored)) if (obj == NULL) { goto alloc_failed; } + memset(PyBytes_AS_STRING(obj), 0, 3 + 1); // +1 for the null byte Py_DECREF(obj); Py_RETURN_NONE; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index de30c6118ba176..16c38403547818 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1272,6 +1272,12 @@ static void bytearray_dealloc(PyObject *op) { PyByteArrayObject *self = _PyByteArray_CAST(op); +#ifdef Py_DEBUG + if (self->ob_bytes_object != NULL) { + _PyBytes_CheckOverflow(self->ob_bytes_object, op, "bytearray"); + } +#endif + if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated bytearray object has exported buffers"); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 117d8b56017b64..4f33b14a197eab 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3191,12 +3191,42 @@ bytes_iteritem(PyObject *obj, Py_ssize_t index) return (_PyObjectIndexPair) { .object = l, .index = index + 1 }; } +#ifdef Py_DEBUG +void +_PyBytes_CheckOverflow(PyObject *self, void *addr, const char *type_name) +{ + // Make sure that the trailing null byte was not modified + char *data = PyBytes_AS_STRING(self); + Py_ssize_t size = PyBytes_GET_SIZE(self); + if (data[size] != '\0') { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in %s object %p " + "at position %zd", + type_name, addr, size); + } +} + + +static void +bytes_dealloc(PyObject *op) +{ + PyBytesObject *self = _PyBytes_CAST(op); + _PyBytes_CheckOverflow(op, op, "bytes"); + Py_TYPE(self)->tp_free((PyObject *)self); +} +#endif + + PyTypeObject PyBytes_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "bytes", PyBytesObject_SIZE, sizeof(char), +#ifdef Py_DEBUG + bytes_dealloc, /* tp_dealloc */ +#else 0, /* tp_dealloc */ +#endif 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3665,6 +3695,18 @@ byteswriter_write_canary_byte(PyBytesWriter *writer) unsigned char *data = (unsigned char*)byteswriter_data(writer); data[writer->size] = PyBytesWriter_CANARY_BYTE; } + + +static void +byteswriter_reset_trailing_byte(PyBytesWriter *writer) +{ + // PyBytesWriter writes non-zero canary byte as the last byte. + // bytes/bytearray expects the last byte to be a null byte. + // Reset the last byte to null for bytes/bytearray. + Py_ssize_t allocated = byteswriter_allocated(writer); + char *data = byteswriter_data(writer); + data[allocated] = '\0'; +} #endif @@ -3814,6 +3856,9 @@ PyBytesWriter_Discard(PyBytesWriter *writer) #ifdef Py_DEBUG byteswriter_check_canary_byte(writer); + if (writer->obj != NULL) { + byteswriter_reset_trailing_byte(writer); + } #endif Py_XDECREF(writer->obj); @@ -3838,16 +3883,7 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) } #ifdef Py_DEBUG - // Check for buffer overflow byteswriter_check_canary_byte(writer); - - if (writer->obj != NULL) { - // byteswriter_write_canary_byte() can override the trailing NUL byte. - // So reset the trailing NUL byte to NUL. - Py_ssize_t allocated = byteswriter_allocated(writer); - char *data = byteswriter_data(writer); - data[allocated] = '\0'; - } #endif PyObject *result; @@ -3855,6 +3891,11 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) result = bytes_get_empty(); } else if (writer->obj != NULL) { + // Truncate the bytes/bytearray object if needed +#ifdef Py_DEBUG + byteswriter_reset_trailing_byte(writer); +#endif + if (writer->use_bytearray) { if (size != PyByteArray_GET_SIZE(writer->obj)) { if (PyByteArray_Resize(writer->obj, size)) { @@ -3868,25 +3909,28 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) goto error; } } + + if (size == 1) { + // Get the single byte singleton + unsigned char ch = PyBytes_AS_STRING(writer->obj)[0]; + PyObject *op = (PyObject*)CHARACTER(ch); + assert(_Py_IsImmortal(op)); + Py_SETREF(writer->obj, op); + } } result = writer->obj; writer->obj = NULL; - - if (size == 1 && !writer->use_bytearray) { - // Get the single byte singleton - unsigned char ch = PyBytes_AS_STRING(result)[0]; - PyObject *op = (PyObject*)CHARACTER(ch); - assert(_Py_IsImmortal(op)); - Py_SETREF(result, op); - } - } - else if (writer->use_bytearray) { - result = PyByteArray_FromStringAndSize(writer->small_buffer, size); } else { - // The function returns single byte singleton if size equals 1 - result = PyBytes_FromStringAndSize(writer->small_buffer, size); + // Create an object from the small buffer + if (writer->use_bytearray) { + result = PyByteArray_FromStringAndSize(writer->small_buffer, size); + } + else { + // The function returns single byte singleton if size equals 1 + result = PyBytes_FromStringAndSize(writer->small_buffer, size); + } } #ifdef Py_DEBUG From 3b1057d4b5b2dfa57f6dbd868ffe7f787e0a82c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2026 17:10:07 +0200 Subject: [PATCH 07/11] gh-156939: Clear newly allocated bytes in PyBytesWriter_Resize() (#157455) Adjust the logic to set newly allocated bytes to a known byte pattern (PyBytesWrite_NEW_BYTE). Only copy 'size' bytes from the small buffer to the new bytes/bytearray object. --- Lib/test/test_capi/test_bytes.py | 5 ++- Objects/bytesobject.c | 64 +++++++++++++++++--------------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 12a1e88eac82d9..b68412a02b3228 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -386,7 +386,7 @@ def test_get_data(self): writer.write(0, b's' * small) self.assertEqual(writer.get_data(), b's' * small) writer.resize(large) - self.assertEqual(writer.get_data(), b's' * small + CANARY_BYTE + NEW_BYTE * (large - small - 1)) + self.assertEqual(writer.get_data(), b's' * small + NEW_BYTE * (large - small)) writer.write(small, b'L' * (large - small)) self.assertEqual(writer.get_data(), b's' * small + b'L' * (large - small)) @@ -475,6 +475,7 @@ def test_resize(self): @unittest.skipUnless(support.Py_DEBUG, 'need debug build') def test_resize_canary(self): CANARY_BYTE = self.CANARY_BYTE + for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): with self.subTest(size=size): # Truncate the last byte @@ -490,7 +491,7 @@ def test_resize_canary(self): writer = self.create_writer(size) writer.write(0, data) writer.resize(0) - self.assertEqual(writer.get_data(), b'') + self.assertEqual(get_data_canary(writer), CANARY_BYTE) self.assertEqual(writer.finish(), b'') @support.nomemtest diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4f33b14a197eab..38ec7a4aefcf56 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3719,25 +3719,26 @@ byteswriter_reset_trailing_byte(PyBytesWriter *writer) #endif static inline int -byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) +byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) { - assert(size >= 0); + assert(new_size >= 0); Py_ssize_t old_allocated = byteswriter_allocated(writer); - if (size <= old_allocated) { + if (new_size <= old_allocated) { // Do not shrink the buffer before PyBytesWriter_FinishWithSize() return 0; } + Py_ssize_t alloc = new_size; if (resize && writer->overallocate) { - if (size <= (PY_SSIZE_T_MAX - size / OVERALLOCATE_FACTOR)) { - size += size / OVERALLOCATE_FACTOR; + if (alloc <= (PY_SSIZE_T_MAX - alloc / OVERALLOCATE_FACTOR)) { + alloc += alloc / OVERALLOCATE_FACTOR; } } if (writer->obj != NULL) { if (writer->use_bytearray) { - if (PyByteArray_Resize(writer->obj, size)) { + if (PyByteArray_Resize(writer->obj, alloc)) { #ifdef Py_DEBUG // bytearray can override the canary byte on error byteswriter_write_canary_byte(writer); @@ -3747,7 +3748,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) } else { // Can raise MemoryError or OverflowError - if (_PyBytes_ResizeKeepOnError(&writer->obj, size)) { + if (_PyBytes_ResizeKeepOnError(&writer->obj, alloc)) { assert(writer->obj != NULL); return -1; } @@ -3755,37 +3756,40 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) } assert(writer->obj != NULL); } - else if (writer->use_bytearray) { - writer->obj = PyByteArray_FromStringAndSize(NULL, size); - if (writer->obj == NULL) { - return -1; - } - if (resize) { - assert((size_t)size > sizeof(writer->small_buffer)); - memcpy(PyByteArray_AS_STRING(writer->obj), - writer->small_buffer, - sizeof(writer->small_buffer)); - } - } else { - writer->obj = PyBytes_FromStringAndSize(NULL, size); - if (writer->obj == NULL) { - return -1; + char *data; + if (writer->use_bytearray) { + writer->obj = PyByteArray_FromStringAndSize(NULL, alloc); + if (writer->obj == NULL) { + return -1; + } + data = PyByteArray_AS_STRING(writer->obj); + } + else { + writer->obj = PyBytes_FromStringAndSize(NULL, alloc); + if (writer->obj == NULL) { + return -1; + } + assert(_PyBytes_IsMutable(writer->obj)); + data = PyBytes_AS_STRING(writer->obj); } + if (resize) { - assert((size_t)size > sizeof(writer->small_buffer)); - memcpy(PyBytes_AS_STRING(writer->obj), - writer->small_buffer, - sizeof(writer->small_buffer)); + // Copy data from the small buffer + Py_ssize_t old_size = writer->size; + assert((size_t)old_size <= sizeof(writer->small_buffer)); + assert(old_size <= alloc); + memcpy(data, writer->small_buffer, old_size); } - assert(_PyBytes_IsMutable(writer->obj)); } #ifdef Py_DEBUG Py_ssize_t allocated = byteswriter_allocated(writer); - if (resize && allocated > old_allocated) { - memset(byteswriter_data(writer) + old_allocated, PyBytesWrite_NEW_BYTE, - allocated - old_allocated); + if (resize) { + Py_ssize_t old_size = writer->size; + assert(allocated > old_size); + memset(byteswriter_data(writer) + old_size, PyBytesWrite_NEW_BYTE, + allocated - old_size); } #endif From fb123c860aad1e4312a9c488bfc8c28d52b0aabb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2026 17:17:43 +0200 Subject: [PATCH 08/11] gh-140550: Enable limited C API tests on Free Threading in test_cext (#157493) Enable limited C API tests on Free Threading in test_cext and test_cppext: test Py_TARGET_ABI3T macro. * Convert test_cppext to PySlot API using PySlot_PTR_STATIC(). * Define Py_MOD_GIL_NOT_USED in test_cext and test_cppext. * Disable C++ test_virtual_object() if Py_TARGET_ABI3T is defined. --- Lib/test/test_cext/__init__.py | 2 - Lib/test/test_cext/extension.c | 1 + Lib/test/test_cppext/__init__.py | 2 - Lib/test/test_cppext/extension.cpp | 58 ++++++++++++------- ...-09-14-14-50-42.gh-issue-140550.7jirEN.rst | 2 + 5 files changed, 41 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-09-14-14-50-42.gh-issue-140550.7jirEN.rst diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 4cc5f843dd388d..dfc7d230426cf8 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -110,11 +110,9 @@ def run_cmd(operation, cmd): class TestPublicCAPI(BaseTests, unittest.TestCase): - @support.requires_gil_enabled('incompatible with Free Threading') def test_build_limited(self): self.check_build('_test_limited_cext', limited=True) - @support.requires_gil_enabled('broken for now with Free Threading') def test_build_limited_c11(self): self.check_build('_test_limited_c11_cext', limited=True, std='c11') diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 543a8096f16f8a..b58c889ba8bcb3 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -139,6 +139,7 @@ static PySlot _testcext_slots[] = { PySlot_STATIC_DATA(Py_mod_doc, (void*)(char*)_testcext_doc), PySlot_FUNC(Py_mod_exec, (void*)_testcext_exec), PySlot_STATIC_DATA(Py_mod_methods, _testcext_methods), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), PySlot_END, }; diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 967feee6693c03..db7f41d9ef7a11 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -102,11 +102,9 @@ class TestPublicCAPI(BaseTests, unittest.TestCase): def test_build(self): self.check_build('_testcppext') - @support.requires_gil_enabled('incompatible with Free Threading') def test_build_limited_cpp03(self): self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) - @support.requires_gil_enabled('incompatible with Free Threading') def test_build_limited(self): self.check_build('_testcppext_limited', limited=True) diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 62ce81e2b510c7..7496581c6756da 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -1,4 +1,4 @@ -// gh-91321: Very basic C++ test extension to check that the Python C API is +// gh-91321: Basic C++ test extension to check that the Python C API is // compatible with C++ and does not emit C++ compiler warnings. // // The code is only built, not executed. @@ -159,6 +159,8 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +// VirtualPyObject is incompatible with opaque PyObject +#ifndef Py_TARGET_ABI3T /* Test a `new`-allocated object with a virtual method. * (https://github.com/python/cpython/issues/94731) */ @@ -237,6 +239,8 @@ test_virtual_object(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } Py_RETURN_NONE; } +#endif // Py_TARGET_ABI3T + static PyObject * test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) @@ -256,7 +260,9 @@ static PyMethodDef _testcppext_methods[] = { {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc}, {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL}, {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL}, +#ifndef Py_TARGET_ABI3T {"test_virtual_object", test_virtual_object, METH_NOARGS, _Py_NULL}, +#endif {"test_datetime", test_datetime, METH_NOARGS, _Py_NULL}, // Note: _testcppext_exec currently runs all test functions directly. // When adding a new one, add a call there. @@ -282,9 +288,11 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); +#ifndef Py_TARGET_ABI3T result = PyObject_CallMethod(module, "test_virtual_object", ""); if (!result) return -1; Py_DECREF(result); +#endif result = PyObject_CallMethod(module, "test_datetime", ""); if (!result) return -1; @@ -313,6 +321,10 @@ _testcppext_exec(PyObject *module) return 0; } + +PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); +PyABIInfo_VAR(abi_info); + // Need to ignore "-Wpedantic" warnings; see VirtualPyObject_Slots above _Py_COMP_DIAG_PUSH #if defined(__GNUC__) @@ -321,32 +333,38 @@ _Py_COMP_DIAG_PUSH #pragma clang diagnostic ignored "-Wpedantic" #endif -static PyModuleDef_Slot _testcppext_slots[] = { - {Py_mod_exec, reinterpret_cast(_testcppext_exec)}, - {0, _Py_NULL} +static PySlot _testcppext_slots[] = { + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + PySlot_PTR_STATIC(Py_mod_name, (void*)STR(MODULE_NAME)), + PySlot_PTR_STATIC(Py_mod_doc, (void*)(char*)_testcppext_doc), + PySlot_PTR_STATIC(Py_mod_exec, (void*)_testcppext_exec), + PySlot_PTR_STATIC(Py_mod_methods, _testcppext_methods), + PySlot_PTR_STATIC(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; _Py_COMP_DIAG_POP -PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); - -static struct PyModuleDef _testcppext_module = { - PyModuleDef_HEAD_INIT, // m_base - STR(MODULE_NAME), // m_name - _testcppext_doc, // m_doc - 0, // m_size - _testcppext_methods, // m_methods - _testcppext_slots, // m_slots - _Py_NULL, // m_traverse - _Py_NULL, // m_clear - _Py_NULL, // m_free -}; -#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define _FUNC_NAME(NAME) PyModExport_ ## NAME #define FUNC_NAME(NAME) _FUNC_NAME(NAME) -PyMODINIT_FUNC +PyMODEXPORT_FUNC FUNC_NAME(MODULE_NAME)(void) { - return PyModuleDef_Init(&_testcppext_module); + return _testcppext_slots; +} + +// Also define the soft-deprecated entrypoint to ensure it isn't called + +#define _INITFUNC_NAME(NAME) PyInit_ ## NAME +#define INITFUNC_NAME(NAME) _INITFUNC_NAME(NAME) + +PyMODINIT_FUNC +INITFUNC_NAME(MODULE_NAME)(void) +{ + PyErr_SetString( + PyExc_AssertionError, + "PyInit_* function called while a PyModExport_* one is available"); + return NULL; } diff --git a/Misc/NEWS.d/next/Tests/2026-09-14-14-50-42.gh-issue-140550.7jirEN.rst b/Misc/NEWS.d/next/Tests/2026-09-14-14-50-42.gh-issue-140550.7jirEN.rst new file mode 100644 index 00000000000000..6de18208c1979c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-09-14-14-50-42.gh-issue-140550.7jirEN.rst @@ -0,0 +1,2 @@ +Enable limited C API tests on Free Threading in test_cext and test_cppext: +test the :c:macro:`Py_TARGET_ABI3T` macro. Patch by Victor Stinner. From 37452c7db46b89a61f1d8322751f2bf0f5b05888 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 15 Sep 2026 16:18:39 +0100 Subject: [PATCH 09/11] gh-144133: Add warning about untrusted input to the `idna` codec (GH-155475) --- Doc/library/codecs.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index e0e69e71e843e7..311437a67f6e81 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1395,6 +1395,15 @@ encodings. | | | :mod:`encodings.idna`. | | | | Only ``errors='strict'`` | | | | is supported. | +| | | | +| | | .. warning:: | +| | | | +| | | This codec builds on | +| | | ``punycode``, whose | +| | | algorithms scale | +| | | poorly, so limit the | +| | | length of untrusted | +| | | input. | +--------------------+---------+---------------------------+ | mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | @@ -1646,6 +1655,11 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. +.. warning:: + + This module builds on ``punycode``, whose algorithms scale poorly, so limit + the length of untrusted input. + If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the third-party :pypi:`idna` module. From 5601e1ec3b5b4b94f189f8a457eea445cda6be45 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Tue, 15 Sep 2026 17:54:55 +0200 Subject: [PATCH 10/11] gh-157517: Don't copy HTML sources and fix sidebar sources links in docs builds (#157522) Co-authored-by: Stan Ulbrych --- Doc/conf.py | 9 ++++++--- Doc/tools/templates/customsourcelink.html | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index f803fb1ff44bef..084906ac17cf94 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -78,6 +78,8 @@ # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py version, release = get_version_info() +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -298,6 +300,7 @@ "repository_url": repository_url or None, "pr_id": os.getenv("READTHEDOCS_VERSION"), "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), + "source_branch": branch, } # This 'Last updated on:' timestamp is inserted at the bottom of every page. @@ -307,6 +310,9 @@ # Path to find HTML templates to override theme templates_path = ['tools/templates'] +# We link to sources on GitHub, so don't copy them into the HTML output. +html_copy_source = False + # Custom sidebar templates, filenames relative to this file. html_sidebars = { # Defaults taken from https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_sidebars @@ -571,9 +577,6 @@ # Options for sphinx.ext.extlinks # ------------------------------- -v = get_header_version_info() -branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" - # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 8feeed2fee3650..eb194aa038c1be 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -1,4 +1,4 @@ -{%- if show_source and has_source and sourcename %} +{%- if page_source_suffix is defined %}