From 084230eb2d46a701fe059aa7cec6b2aa1be75814 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sun, 16 Aug 2026 15:38:53 +0300 Subject: [PATCH 1/6] gh-148817: Fold long lists/sets of constant elements into constant tuples/frozensets (#149016) Fold long lists/sets of constant elements into constant tuples/frozensets. Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- .github/CODEOWNERS | 2 +- Lib/test/test_peepholer.py | 162 ++++++++++++++++++ ...-04-26-15-08-53.gh-issue-148817.cuN07H.rst | 5 + Python/flowgraph.c | 71 +++++--- 4 files changed, 218 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-26-15-08-53.gh-issue-148817.cuN07H.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 41b64d71b67ff2d..247b84ba37bbbd4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -223,7 +223,7 @@ Tools/cases_generator/ @markshannon Python/assemble.c @markshannon @iritkatriel Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel -Python/flowgraph.c @markshannon @iritkatriel +Python/flowgraph.c @markshannon @iritkatriel @eclips4 Python/instruction_sequence.c @iritkatriel Python/symtable.c @JelleZijlstra @carljm diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 28748009f731bc1..8727352a31a7a05 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -2470,6 +2470,168 @@ def test_list_to_tuple_get_iter_is_safe(self): self.assertEqual(b, [3, 2, 1, 0]) self.assertEqual(items, []) + def test_fold_constant_big_list_for_iter(self): + # for x in [c1, c2, ..., cN] (N > 30) should fold to LOAD_CONST tuple + consts = 35 + before = ( + [("BUILD_LIST", 0, 1)] + + [("LOAD_CONST", 0, 2), ("LIST_APPEND", 1, 3)] * consts + + [("GET_ITER", 0, 4), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 5), + ("STORE_FAST", 0, 6), + ("JUMP", top, 7), + end, + ("END_FOR", None, 8), + ("POP_ITER", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11)] + ) + after = [ + ("LOAD_CONST", 1, 3), + ("GET_ITER", 0, 4), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 5), + ("STORE_FAST", 0, 6), + ("JUMP", top, 7), + end, + ("END_FOR", None, 8), + ("POP_ITER", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + result_const = tuple(["test"] * consts) + self.cfg_optimization_test(before, after, consts=["test"], + expected_consts=["test", result_const]) + + def test_fold_constant_big_set_for_iter(self): + # for x in {c1, c2, ..., cN} (N > 30) should fold to LOAD_CONST frozenset + before = [ + ("BUILD_SET", 0, 1), + ("LOAD_SMALL_INT", 1, 2), ("SET_ADD", 1, 3), + ("LOAD_SMALL_INT", 2, 4), ("SET_ADD", 1, 5), + ("LOAD_SMALL_INT", 3, 6), ("SET_ADD", 1, 7), + ("GET_ITER", 0, 8), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 9), + ("STORE_FAST", 0, 10), + ("JUMP", top, 11), + end, + ("END_FOR", None, 12), + ("POP_ITER", None, 13), + ("LOAD_CONST", 0, 14), + ("RETURN_VALUE", None, 15), + ] + after = [ + ("LOAD_CONST", 1, 7), + ("GET_ITER", 0, 8), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 9), + ("STORE_FAST", 0, 10), + ("JUMP", top, 11), + end, + ("END_FOR", None, 12), + ("POP_ITER", None, 13), + ("LOAD_CONST", 0, 14), + ("RETURN_VALUE", None, 15), + ] + self.cfg_optimization_test(before, after, consts=["test"], + expected_consts=["test", frozenset({1, 2, 3})]) + + def test_fold_constant_list_to_tuple_for_iter(self): + INTRINSIC_LIST_TO_TUPLE = 6 + before = [ + ("BUILD_LIST", 0, 1), + ("LOAD_SMALL_INT", 1, 2), ("LIST_APPEND", 1, 3), + ("LOAD_SMALL_INT", 2, 4), ("LIST_APPEND", 1, 5), + ("LOAD_SMALL_INT", 3, 6), ("LIST_APPEND", 1, 7), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 8), + ("GET_ITER", 0, 9), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 10), + ("STORE_FAST", 0, 11), + ("JUMP", top, 12), + end, + ("END_FOR", None, 13), + ("POP_ITER", None, 14), + ("LOAD_CONST", 0, 15), + ("RETURN_VALUE", None, 16), + ] + after = [ + ("LOAD_CONST", 1, 8), + ("GET_ITER", 0, 9), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 10), + ("STORE_FAST", 0, 11), + ("JUMP", top, 12), + end, + ("END_FOR", None, 13), + ("POP_ITER", None, 14), + ("LOAD_CONST", 0, 15), + ("RETURN_VALUE", None, 16), + ] + self.cfg_optimization_test(before, after, consts=["test"], + expected_consts=["test", (1, 2, 3)]) + + def test_fold_constant_big_list_contains_op(self): + # x in [c1, c2, ..., cN] (N > 30) should fold to LOAD_CONST tuple + before = [ + ("LOAD_FAST", 0, 1), + ("BUILD_LIST", 0, 2), + ("LOAD_SMALL_INT", 1, 3), ("LIST_APPEND", 1, 4), + ("LOAD_SMALL_INT", 2, 5), ("LIST_APPEND", 1, 6), + ("LOAD_SMALL_INT", 3, 7), ("LIST_APPEND", 1, 8), + ("CONTAINS_OP", 0, 9), + ("RETURN_VALUE", None, 10), + ] + after = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_CONST", 1, 8), + ("CONTAINS_OP", 0, 9), + ("RETURN_VALUE", None, 10), + ] + self.cfg_optimization_test(before, after, consts=[None], + expected_consts=[None, (1, 2, 3)]) + + def test_fold_constant_big_set_contains_op(self): + # x in {c1, c2, ..., cN} (N > 30) should fold to LOAD_CONST frozenset + before = [ + ("LOAD_FAST", 0, 1), + ("BUILD_SET", 0, 2), + ("LOAD_SMALL_INT", 1, 3), ("SET_ADD", 1, 4), + ("LOAD_SMALL_INT", 2, 5), ("SET_ADD", 1, 6), + ("LOAD_SMALL_INT", 3, 7), ("SET_ADD", 1, 8), + ("CONTAINS_OP", 0, 9), + ("RETURN_VALUE", None, 10), + ] + after = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_CONST", 1, 8), + ("CONTAINS_OP", 0, 9), + ("RETURN_VALUE", None, 10), + ] + self.cfg_optimization_test(before, after, consts=[None], + expected_consts=[None, frozenset({1, 2, 3})]) + + def test_no_fold_big_list_for_iter_with_non_const(self): + same = [ + ("BUILD_LIST", 0, 1), + ("LOAD_SMALL_INT", 1, 2), ("LIST_APPEND", 1, 3), + ("LOAD_FAST_BORROW", 0, 4), ("LIST_APPEND", 1, 5), + ("LOAD_SMALL_INT", 3, 6), ("LIST_APPEND", 1, 7), + ("GET_ITER", 0, 8), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 9), + ("STORE_FAST", 1, 10), + ("JUMP", top, 11), + end, + ("END_FOR", None, 12), + ("POP_ITER", None, 13), + ("LOAD_CONST", 0, 14), + ("RETURN_VALUE", None, 15), + ] + self.cfg_optimization_test(same, same, consts=["test"]) + class OptimizeLoadFastTestCase(DirectCfgOptimizerTests): def make_bb(self, insts): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-26-15-08-53.gh-issue-148817.cuN07H.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-26-15-08-53.gh-issue-148817.cuN07H.rst new file mode 100644 index 000000000000000..87850754c85a140 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-26-15-08-53.gh-issue-148817.cuN07H.rst @@ -0,0 +1,5 @@ +Fold large constant list and set literals used as the iterable of a +:keyword:`for` loop or ``in``/``not in`` test into a constant +:class:`tuple` or :class:`frozenset`, restoring an optimization +previously done by the AST optimizer that was lost when constant +folding moved to the CFG. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9a7b0b1eda28d98..a5138d1a1fa2846 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1569,34 +1569,48 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, } /* Replace: - BUILD_LIST 0 + BUILD_LIST/BUILD_SET 0 LOAD_CONST c1 - LIST_APPEND 1 + LIST_APPEND/SET_ADD 1 LOAD_CONST c2 - LIST_APPEND 1 + LIST_APPEND/SET_ADD 1 ... LOAD_CONST cN - LIST_APPEND 1 - CALL_INTRINSIC_1 INTRINSIC_LIST_TO_TUPLE + LIST_APPEND/SET_ADD 1 + [CALL_INTRINSIC_1 INTRINSIC_LIST_TO_TUPLE] <-- optional with: LOAD_CONST (c1, c2, ... cN) + The instruction at `i` is either the LIST_TO_TUPLE intrinsic (so the + immediately preceding non-NOP instruction is expected to be a + LIST_APPEND, and only the BUILD_LIST/LIST_APPEND form is considered), + or the trailing LIST_APPEND or SET_ADD itself, in which case the + matching BUILD_LIST/BUILD_SET start is selected from its opcode, and + for sets the result is wrapped in a frozenset. */ static int -fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, - PyObject *consts, PyObject *const_cache, - _Py_hashtable_t *consts_index) +fold_constant_seq_into_load_const(basicblock *bb, int i, + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); assert(i >= 0); assert(i < bb->b_iused); - cfg_instr *intrinsic = &bb->b_instr[i]; - assert(intrinsic->i_opcode == CALL_INTRINSIC_1); - assert(intrinsic->i_oparg == INTRINSIC_LIST_TO_TUPLE); - + cfg_instr *target = &bb->b_instr[i]; + assert(target->i_opcode == LIST_APPEND || target->i_opcode == SET_ADD || + (target->i_opcode == CALL_INTRINSIC_1 && + target->i_oparg == INTRINSIC_LIST_TO_TUPLE)); + bool expected_append = target->i_opcode == CALL_INTRINSIC_1; + int append_op = expected_append ? LIST_APPEND : target->i_opcode; + assert(append_op == LIST_APPEND || append_op == SET_ADD); + int build_op = append_op == LIST_APPEND ? BUILD_LIST : BUILD_SET; int consts_found = 0; - bool expect_append = true; + /* Walking backward from `i`, we expect LIST_APPEND/SET_ADD and + LOAD_CONST to alternate. If `i` is the trailing LIST_TO_TUPLE + intrinsic, the next instruction back is an APPEND. If `i` is the + trailing APPEND itself, the next instruction back is a LOAD_CONST. */ + bool expect_append = expected_append; for (int pos = i - 1; pos >= 0; pos--) { cfg_instr *instr = &bb->b_instr[pos]; @@ -1607,7 +1621,7 @@ fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, continue; } - if (opcode == BUILD_LIST && oparg == 0) { + if (opcode == build_op && oparg == 0) { if (!expect_append) { /* Not a sequence start. */ return SUCCESS; @@ -1619,7 +1633,8 @@ fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, return ERROR; } - for (int newpos = i - 1; newpos >= pos; newpos--) { + int newpos_start = expected_append ? i - 1 : i; + for (int newpos = newpos_start; newpos >= pos; newpos--) { instr = &bb->b_instr[newpos]; if (instr->i_opcode == NOP) { continue; @@ -1636,11 +1651,20 @@ fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, nop_out(&instr, 1); } assert(consts_found == 0); - return instr_make_load_const(intrinsic, newconst, consts, const_cache, consts_index); + + if (build_op == BUILD_SET) { + PyObject *frozen = PyFrozenSet_New(newconst); + Py_DECREF(newconst); + if (frozen == NULL) { + return ERROR; + } + newconst = frozen; + } + return instr_make_load_const(target, newconst, consts, const_cache, consts_index); } if (expect_append) { - if (opcode != LIST_APPEND || oparg != 1) { + if (opcode != append_op || oparg != 1) { return SUCCESS; } } @@ -2579,17 +2603,22 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts, break; case CALL_INTRINSIC_1: if (oparg == INTRINSIC_LIST_TO_TUPLE) { - if (nextop == GET_ITER) { + RETURN_IF_ERROR(fold_constant_seq_into_load_const(bb, i, consts, const_cache, consts_index)); + if (inst->i_opcode == CALL_INTRINSIC_1 && nextop == GET_ITER) { INSTR_SET_OP0(inst, NOP); } - else { - RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache, consts_index)); - } } else if (oparg == INTRINSIC_UNARY_POSITIVE) { RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache, consts_index)); } break; + case LIST_APPEND: + case SET_ADD: + if (oparg == 1 && (nextop == GET_ITER || nextop == CONTAINS_OP)) { + RETURN_IF_ERROR(fold_constant_seq_into_load_const( + bb, i, consts, const_cache, consts_index)); + } + break; case BINARY_OP: RETURN_IF_ERROR(fold_const_binop(bb, i, consts, const_cache, consts_index)); break; From 125ca2699228379c9be80ad8a9d5c3631fcac44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Sun, 16 Aug 2026 15:25:03 +0200 Subject: [PATCH 2/6] gh-154196: Improve `AttributeError` messages from unresolved lazy imports (#154688) --- Lib/test/test_lazy_import/__init__.py | 17 +++++++++++++ ...-07-25-12-43-42.gh-issue-154196.0rAdob.rst | 2 ++ Objects/lazyimportobject.c | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-12-43-42.gh-issue-154196.0rAdob.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index b12e209707a9de3..9147e788d7a81f2 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -277,6 +277,23 @@ def test_lazy_import_type_attributes_accessible(self): proc = assert_python_ok("-c", code) self.assertIn(b"tp_free(op); } +/* Specialize the error message for failed attribute lookups. */ +static PyObject * +lazy_import_getattro(PyObject *op, PyObject *name) +{ + PyObject *value = _PyObject_GenericGetAttrWithDict(op, name, NULL, /* suppress */1); + if (value == NULL) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + PyObject *lz_name = _PyLazyImport_GetName(op); + if (lz_name == NULL) { + return NULL; + } + PyErr_Format(PyExc_AttributeError, + "cannot access attribute %R on unresolved lazy import %R", + name, lz_name); + Py_DECREF(lz_name); + return NULL; + } + return value; +} + static PyObject * lazy_import_name(PyLazyImportObject *m) { @@ -149,6 +172,7 @@ PyTypeObject PyLazyImport_Type = { .tp_repr = lazy_import_repr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_doc = lazy_import_doc, + .tp_getattro = lazy_import_getattro, .tp_traverse = lazy_import_traverse, .tp_clear = lazy_import_clear, .tp_methods = lazy_import_methods, From fbbd94eae374ed0392d399355134a73841742a6e Mon Sep 17 00:00:00 2001 From: hu-jeff <59699807+hu-jeff@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:31:34 +0000 Subject: [PATCH 3/6] Fix various typos in `Doc/library/stdtypes.rst` (#155879) Co-authored-by: Stan Ulbrych --- Doc/library/stdtypes.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 280a9f3b7d07518..02b47ed5ca1dde2 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -706,7 +706,7 @@ A hexadecimal string takes the form:: [sign] ['0x'] integer ['.' fraction] ['p' exponent] -where the optional ``sign`` may by either ``+`` or ``-``, ``integer`` +where the optional ``sign`` may be either ``+`` or ``-``, ``integer`` and ``fraction`` are strings of hexadecimal digits, and ``exponent`` is a decimal integer with an optional leading sign. Case is not significant, and there must be at least one hexadecimal digit in @@ -1345,7 +1345,7 @@ Mutable sequence types also support the following methods: :no-typesetting: .. method:: sequence.pop(index=-1, /) - Retrieve the item at *index* and also removes it from *sequence*. + Retrieve the item at *index* and also remove it from *sequence*. By default, the last item in *sequence* is removed and returned. .. method:: bytearray.remove(value, /) @@ -2120,7 +2120,7 @@ expression support in the :mod:`re` module). one character, ``False`` otherwise. Alphabetic characters are those characters defined in the Unicode character database as "Letter", i.e., those with general category property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different - from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and + from the `Alphabetic property defined in section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard `__. For example: @@ -3044,7 +3044,7 @@ replacement field. For example:: '0.333333' >>> f'{one_third:_^+10}' '___+1/3___' - >>> >>> f'{one_third!r:_^20}' + >>> f'{one_third!r:_^20}' '___Fraction(1, 3)___' >>> f'{one_third = :~>10}~' 'one_third = ~~~~~~~1/3~' @@ -3054,12 +3054,12 @@ replacement field. For example:: Template String Literals (t-strings) ------------------------------------ -An :dfn:`t-string` (formally a :dfn:`template string literal`) is +A :dfn:`t-string` (formally a :dfn:`template string literal`) is a string literal that is prefixed with ``t`` or ``T``. These strings follow the same syntax and evaluation rules as :ref:`formatted string literals `, -with for the following differences: +with the following differences: * Rather than evaluating to a ``str`` object, template string literals evaluate to a :class:`string.templatelib.Template` object. @@ -3086,7 +3086,7 @@ with for the following differences: The :class:`!Interpolation` instance for the expression will be created as normal, except that :attr:`~string.templatelib.Interpolation.conversion` will be set to '``r``' (:func:`repr`) by default. - If an explicit conversion or format specifier are provided, + If an explicit conversion or format specifier is provided, this will override the default behaviour. @@ -3463,7 +3463,7 @@ objects. .. classmethod:: fromhex(string, /) - This :class:`bytearray` class method returns bytearray object, decoding + This :class:`bytearray` class method returns a bytearray object, decoding the given string object. The string must contain two hexadecimal digits per byte, with ASCII whitespace being ignored. @@ -4427,7 +4427,7 @@ the ``%`` operator (modulo). This is also known as the bytes *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a bytes object), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -4628,7 +4628,7 @@ copying. underlying data. ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which - is the nested list representation of the view. If ``view.ndim = 1``, + is the nested list representation of the view. If ``view.ndim == 1``, this is equal to the number of elements in the view. .. versionchanged:: 3.12 @@ -4713,7 +4713,7 @@ copying. :class:`collections.abc.Sequence` .. versionchanged:: 3.5 - memoryviews can now be indexed with tuple of integers. + memoryviews can now be indexed with a tuple of integers. .. versionchanged:: 3.14 memoryview is now a :term:`generic type`. @@ -6182,7 +6182,7 @@ enables cleaner type hinting syntax compared to subscripting :class:`typing.Unio .. note:: - The ``|`` operand cannot be used at runtime to define unions where one or + The ``|`` operator cannot be used at runtime to define unions where one or more members is a forward reference. For example, ``int | "Foo"``, where ``"Foo"`` is a reference to a class not yet defined, will fail at runtime. For unions which include forward references, present the @@ -6341,7 +6341,7 @@ Methods Methods are functions that are called using the attribute notation. There are two flavors: :ref:`built-in methods ` (such as :meth:`~list.append` on lists) -and :ref:`class instance method `. +and :ref:`class instance methods `. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an From 70fdc966d0420e64e848eb09f7f695735f2a8b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Sun, 16 Aug 2026 21:43:24 +0800 Subject: [PATCH 4/6] gh-155477: Fix multiprocessing.Pool deadlock on close() with a buffersize imap (GH-155478) close() did not release the buffersize semaphores that throttle the task generator, so a partially-consumed imap left the task handler blocked and join() deadlocked. Release them in close() and stop the generator once the pool leaves the RUN state. --- Lib/multiprocessing/pool.py | 9 +++++++++ Lib/test/_test_multiprocessing.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 8fd0f98a02dd3a6..f50bcbe4451bea4 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -403,6 +403,11 @@ def _guarded_task_generation(self, result_job, func, iterable, sema=None): enumerated_iter = iter(enumerate(iterable)) while True: sema.acquire() + if self._state != RUN: + # The pool is closing or terminating; stop submitting + # the still-throttled tasks so the task handler can + # finish instead of blocking here forever. + break try: i, x = next(enumerated_iter) except StopIteration: @@ -661,6 +666,10 @@ def close(self): self._state = CLOSE self._worker_handler._state = CLOSE self._change_notifier.put(None) + # Wake any task generator throttled on a buffersize semaphore so + # it observes the CLOSE state and stops submitting. + for sema in list(self._taskqueue_buffersize_semaphores): + sema.release() def terminate(self): util.debug('terminating pool') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 338a31fd7f869ea..e5f618f5f2e84f4 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3229,6 +3229,27 @@ def produce_args(): p.terminate() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_with_buffersize_close_after_partial_consumption( + self, method_name + ): + # close()/join() must not deadlock when a buffersize iterator is + # only partially consumed (the throttled task generator must stop). + p = self.Pool(2) + method = getattr(p, method_name) + it = method(sqr, range(1000), buffersize=2) + next(it) + finished = threading.Event() + def finalize(): + p.close() + p.join() + finished.set() + t = threading.Thread(target=finalize) + t.start() + t.join(support.SHORT_TIMEOUT) + self.assertTrue(finished.is_set(), "close()/join() deadlocked") + @support.subTests('method_name', ("imap", "imap_unordered")) def test_imap_and_imap_unordered_with_buffersize_on_empty_iterable( self, method_name From 5b96d3914767dc71cb21fd3448f5ded4fd85d957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Mon, 17 Aug 2026 00:07:02 +0800 Subject: [PATCH 5/6] gh-151895: Fix marshal.loads() crash on dict reference-tracking failure (GH-151896) Loading a reference-tracked dictionary dereferenced a NULL pointer when the allocation that registers it for back-references failed under low memory. It now raises MemoryError, matching the tuple and list paths. --- .../2026-06-22-11-05-56.gh-issue-151895.QQsjUQ.rst | 2 ++ Python/marshal.c | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-11-05-56.gh-issue-151895.QQsjUQ.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-11-05-56.gh-issue-151895.QQsjUQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-11-05-56.gh-issue-151895.QQsjUQ.rst new file mode 100644 index 000000000000000..e17b340348b06b1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-11-05-56.gh-issue-151895.QQsjUQ.rst @@ -0,0 +1,2 @@ +Fixed a crash in :func:`marshal.loads` when an allocation failed while +loading a reference-tracked dictionary; it now raises :exc:`MemoryError`. diff --git a/Python/marshal.c b/Python/marshal.c index 25353f6e6896249..603697e9081c59a 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1471,6 +1471,9 @@ r_object(RFILE *p) } if (type == TYPE_DICT) { R_REF(v); + if (v == NULL) { + break; + } } else { idx = r_ref_reserve(flag, p); From 7a845ce16548bf94e777984458ea534c5a65a2a8 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Sun, 16 Aug 2026 20:24:49 +0300 Subject: [PATCH 6/6] gh-154842: Reject repack() while a reading handle is open (GH-154843) ZipFile.repack() moves member data, but a ZipExtFile from an earlier open() keeps its own absolute position, so it silently returned data from the wrong place and a full read failed with a misleading CRC error. Raise ValueError while _fileRefCnt shows an open reading handle, as the writing-handle case already does. --- Doc/library/zipfile.rst | 4 +++- Lib/test/test_zipfile/test_core.py | 12 ++++++++++++ Lib/zipfile/__init__.py | 15 ++++++++------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 65bc54e3856a945..cfbbc98a4739b8f 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -585,7 +585,9 @@ ZipFile objects strict_descriptor=True[, chunk_size]) Rewrites the archive to remove unreferenced local file entries, shrinking - its file size. The archive must be opened with mode ``'a'``. + its file size. The archive must be opened with mode ``'a'``, and any file + object returned by :meth:`ZipFile.open` must be closed first, since + repacking moves the member data such objects refer to. If *removed* is provided, it must be a sequence of :class:`ZipInfo` objects representing the recently removed members, and only their corresponding diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index d0ae7ce787bee32..1c6e3a9f0a9a2de 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2388,6 +2388,18 @@ def test_repack_writing(self, m_repack): zh.repack() m_repack.assert_not_called() + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_reading(self, m_repack): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + with zh.open(self.test_files[0][0]): + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + # Allowed once the reading handle is closed. + zh.repack() + m_repack.assert_called_once() + @mock.patch.object(zipfile, '_ZipRepacker') def test_repack_mode_r(self, m_repack): self._prepare_zip_from_test_files(TESTFN, self.test_files) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index dd1f7fb9e802048..7a81aa8f44c8f4c 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -2395,15 +2395,16 @@ def repack(self, removed=None, *, strict_descriptor=True, truncation.""" if self.mode != 'a': raise ValueError("repack() requires mode 'a'") - if not self.fp: - raise ValueError( - "Attempt to write to ZIP archive that was already closed") - if self._writing: - raise ValueError( - "Can't write to ZIP archive while an open writing handle exists" - ) with self._lock: + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing or self._fileRefCnt > 1: + raise ValueError( + "Can't repack ZIP archive while an open handle exists" + ) + self._writing = True try: repacker = _ZipRepacker(