diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f45ab397e93693..013150535cb089 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -65,14 +65,54 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(async_iterable, /) + aiter(callable, /, stop_value, *, stop_exception=StopAsyncIteration) + aiter(callable, /, *, stop_exception) + + Return an :term:`asynchronous iterator` object. + The first argument is interpreted very differently + depending on the presence of the other arguments. + Without other arguments, + the single argument must be an :term:`asynchronous iterable`, + and the result is equivalent to calling ``x.__aiter__()``. + + If *stop_value* or *stop_exception* is given, + then the first argument must be a callable object. + The asynchronous iterator created in this case + calls *callable* with no arguments and awaits the result + for each call to its :meth:`~object.__anext__` method; + if the awaited value is equal to *stop_value*, + or if the call raises an exception matching *stop_exception*, + :exc:`StopAsyncIteration` will be raised, + otherwise the value will be returned. + The callable is only called when the result of :meth:`~object.__anext__` + is awaited. + + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopAsyncIteration` + which does not match *stop_exception*, + it is replaced with a :exc:`RuntimeError`, + as for asynchronous generators (see :pep:`525`). + + For example, reading fixed-size chunks from an asynchronous stream + until the end of file is reached:: - Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. - Equivalent to calling ``x.__aiter__()``. + from functools import partial + async for chunk in aiter(partial(reader.read, 1024), b''): + process_chunk(chunk) + + Or consuming an :class:`asyncio.Queue` until it is shut down:: - Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. + from asyncio import QueueShutDown + async for item in aiter(queue.get, stop_exception=QueueShutDown): + process_item(item) .. versionadded:: 3.10 + .. versionchanged:: next + Added the *stop_value* and *stop_exception* parameters. + .. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable @@ -1143,22 +1183,34 @@ are always available. They are listed here in alphabetical order. .. function:: iter(iterable, /) - iter(callable, sentinel, /) + iter(callable, /, stop_value, *, stop_exception=StopIteration) + iter(callable, /, *, stop_exception) Return an :term:`iterator` object. The first argument is interpreted very - differently depending on the presence of the second argument. Without a - second argument, the single argument must be a collection object which supports the + differently depending on the presence of the other arguments. Without other + arguments, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, - :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, + :exc:`TypeError` is raised. + + If *stop_value* or *stop_exception* is given, then the first argument must be a callable object. The iterator created in this case will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will + *stop_value*, or if the call raises an exception matching *stop_exception*, + :exc:`StopIteration` will be raised, otherwise the value will be returned. + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopIteration` + which does not match *stop_exception*, + it is replaced with a :exc:`RuntimeError`, + as for generators (see :pep:`479`). + See also :ref:`typeiter`. One useful application of the second form of :func:`iter` is to build a @@ -1170,6 +1222,19 @@ are always available. They are listed here in alphabetical order. for block in iter(partial(f.read, 64), b''): process_block(block) + *stop_exception* is useful for callables + which report exhaustion by raising an exception + instead of returning a special value. + For example, draining a queue:: + + import queue + for item in iter(input_queue.get_nowait, stop_exception=queue.Empty): + process_item(item) + + .. versionchanged:: next + Added the *stop_exception* parameter + and allowed passing *stop_value* by keyword. + .. function:: len(object, /) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 310ccd651e18c7..f9ffb07ada88fd 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -711,16 +711,16 @@ Functions .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ xml_declaration=None, default_namespace=None, \ - short_empty_elements=True) + short_empty_elements=True, standalone=None) Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to generate a Unicode string (otherwise, a bytestring is generated). *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *xml_declaration*, *default_namespace* and *short_empty_elements* has the same - meaning as in :meth:`ElementTree.write`. Returns an (optionally) encoded string - containing the XML data. + *xml_declaration*, *default_namespace*, *short_empty_elements* and + *standalone* has the same meaning as in :meth:`ElementTree.write`. + Returns an (optionally) encoded string containing the XML data. .. versionchanged:: 3.4 Added the *short_empty_elements* parameter. @@ -732,19 +732,23 @@ Functions The :func:`tostring` function now preserves the attribute order specified by the user. + .. versionchanged:: next + Added the *standalone* parameter. + .. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ xml_declaration=None, default_namespace=None, \ - short_empty_elements=True) + short_empty_elements=True, standalone=None) Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to generate a Unicode string (otherwise, a bytestring is generated). *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *xml_declaration*, *default_namespace* and *short_empty_elements* has the same - meaning as in :meth:`ElementTree.write`. Returns a list of (optionally) encoded - strings containing the XML data. It does not guarantee any specific sequence, + *xml_declaration*, *default_namespace*, *short_empty_elements* and + *standalone* has the same meaning as in :meth:`ElementTree.write`. + Returns a list of (optionally) encoded strings containing the XML data. + It does not guarantee any specific sequence, except that ``b"".join(tostringlist(element)) == tostring(element)``. .. versionadded:: 3.2 @@ -759,6 +763,9 @@ Functions The :func:`tostringlist` function now preserves the attribute order specified by the user. + .. versionchanged:: next + Added the *standalone* parameter. + .. function:: XML(text, parser=None) @@ -1186,7 +1193,7 @@ ElementTree Objects .. method:: write(file, encoding="us-ascii", xml_declaration=None, \ default_namespace=None, method="xml", *, \ - short_empty_elements=True) + short_empty_elements=True, standalone=None) Writes the element tree to a file, as XML. *file* is a file name, or a :term:`file object` opened for writing. *encoding* [1]_ is the output @@ -1202,6 +1209,13 @@ ElementTree Objects emitted as a single self-closed tag, otherwise they are emitted as a pair of start/end tags. + The keyword-only *standalone* parameter is the value of the standalone + document declaration in the XML declaration. + Use ``True`` for ``standalone="yes"``, ``False`` for ``standalone="no"``, + and ``None`` (the default) to omit it. + An XML declaration is written if *standalone* is not ``None``; + combining it with ``xml_declaration=False`` raises a :exc:`ValueError`. + The output is either a string (:class:`str`) or binary (:class:`bytes`). This is controlled by the *encoding* argument. If *encoding* is ``"unicode"``, the output is a string; otherwise, it's binary. Note that @@ -1216,6 +1230,9 @@ ElementTree Objects The :meth:`write` method now preserves the attribute order specified by the user. + .. versionchanged:: next + Added the *standalone* parameter. + This is the XML file that is going to be manipulated:: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4c432fb249f246..3262acd87d6d49 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -75,6 +75,13 @@ New features Other language changes ====================== +* The :func:`iter` function now accepts the *stop_exception* parameter. + The created iterator stops when the callable raises the specified exception. + The second parameter is now named *stop_value* and can be passed by keyword. + :func:`aiter` now accepts the same *stop_value* and *stop_exception* + parameters, calling an asynchronous callable and awaiting the result. + (Contributed by Serhiy Storchaka in :gh:`64862`.) + * :meth:`memoryview.cast` now allows casting a multidimensional F-contiguous view to a one-dimensional view. (Contributed by Jaemin Park in :gh:`91484`.) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c86ae242feac1e..266add0fb7a9c3 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -29,6 +29,9 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); +// Set the exception passed to throw(typ[, val[, tb]]). +// Return 0 on success, -1 on failure. +extern int _PyGen_SetException(PyObject *typ, PyObject *val, PyObject *tb); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9ab20be70614de..4fd6c618fb7440 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -2108,6 +2108,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(step)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(steps)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_exception)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(store_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strategy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strftime)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 51d9fbe89b3423..5b35c53e0aa03b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -831,6 +831,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(stdout) STRUCT_FOR_ID(step) STRUCT_FOR_ID(steps) + STRUCT_FOR_ID(stop_exception) + STRUCT_FOR_ID(stop_value) STRUCT_FOR_ID(store_name) STRUCT_FOR_ID(strategy) STRUCT_FOR_ID(strftime) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 0623adce693d46..6c907e0cf79894 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -538,7 +538,7 @@ struct _py_func_state { If you add a new static type to the standard library, you may have to update one of these numbers. */ -#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 122 #define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 diff --git a/Include/internal/pycore_iterobject.h b/Include/internal/pycore_iterobject.h new file mode 100644 index 00000000000000..90b444976e2f19 --- /dev/null +++ b/Include/internal/pycore_iterobject.h @@ -0,0 +1,28 @@ +#ifndef Py_INTERNAL_ITEROBJECT_H +#define Py_INTERNAL_ITEROBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyTypeObject _PyACallIter_Type; +extern PyTypeObject _PyACallIterAwaitable_Type; + +// Like PyCallIter_New(), but the iteration also stops when *callable* raises +// an exception matching *stop_exc* (an exception class or a tuple of exception +// classes). *sentinel* can be NULL; NULL *stop_exc* means StopIteration. +extern PyObject *_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +// The asynchronous counterpart of _PyCallIter_NewEx(): the result of +// *callable* is awaited, and NULL *stop_exc* means StopAsyncIteration. +extern PyObject *_PyACallIter_New(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_ITEROBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 88ca09e6ba245f..c80925f020186b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -2106,6 +2106,8 @@ extern "C" { INIT_ID(stdout), \ INIT_ID(step), \ INIT_ID(steps), \ + INIT_ID(stop_exception), \ + INIT_ID(stop_value), \ INIT_ID(store_name), \ INIT_ID(strategy), \ INIT_ID(strftime), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 3c4d7d664537a8..b30cfc678de1cd 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -3104,6 +3104,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_exception); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_value); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 70a285dd91f385..cdae58b3e89ae3 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -789,6 +789,164 @@ async def gen(): applied_twice = aiter(applied_once) self.assertIs(applied_once, applied_twice) + def make_counter(self): + state = {'n': 0} + async def counter(): + state['n'] += 1 + return state['n'] + return counter + + def collect(self, ait): + async def consume(): + return [i async for i in ait] + return self.loop.run_until_complete(consume()) + + def test_aiter_callable_stop(self): + self.assertEqual(self.collect(aiter(self.make_counter(), 4)), [1, 2, 3]) + self.assertEqual(self.collect(aiter(self.make_counter(), stop_value=4)), + [1, 2, 3]) + + def test_aiter_callable_stop_exception(self): + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 3: + raise LookupError + return value + self.assertEqual(self.collect(aiter(spam, stop_exception=LookupError)), + [1, 2, 3]) + counter = self.make_counter() + self.assertEqual( + self.collect(aiter(spam, stop_exception=(ZeroDivisionError, + LookupError))), + [1, 2, 3]) + + def test_aiter_callable_stop_and_exception(self): + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 5: + raise LookupError + return value + self.assertEqual( + self.collect(aiter(spam, 3, stop_exception=LookupError)), [1, 2]) + counter = self.make_counter() + self.assertEqual( + self.collect(aiter(spam, 100, stop_exception=LookupError)), + [1, 2, 3, 4, 5]) + + def test_aiter_callable_stop_async_iteration(self): + # StopAsyncIteration is the default stop exception + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 3: + raise StopAsyncIteration + return value + self.assertEqual( + self.collect(aiter(spam, stop_exception=StopAsyncIteration)), + [1, 2, 3]) + + def test_aiter_callable_leak_from_await(self): + # A StopAsyncIteration leaking from the await is replaced with + # RuntimeError (see PEP 525) + async def spam(): + raise StopAsyncIteration + it = aiter(spam, 10, stop_exception=LookupError) + with self.assertRaisesRegex(RuntimeError, + 'callable raised StopAsyncIteration') as cm: + self.loop.run_until_complete(anext(it)) + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + # but if it matches stop_exception, it stops the iteration + it = aiter(spam, 10, stop_exception=(LookupError, StopAsyncIteration)) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_leak_from_call(self): + # StopIteration and StopAsyncIteration leaking from the call are + # replaced with RuntimeError (see PEP 525) + for exc in StopIteration, StopAsyncIteration: + with self.subTest(exc=exc): + def spam(): + raise exc + it = aiter(spam, 10, stop_exception=LookupError) + with self.assertRaisesRegex( + RuntimeError, f'callable raised {exc.__name__}') as cm: + self.loop.run_until_complete(anext(it)) + self.assertIsInstance(cm.exception.__cause__, exc) + # but if it matches stop_exception, it stops the iteration + it = aiter(spam, 10, stop_exception=(LookupError, exc)) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_other_exception(self): + async def spam(): + raise ZeroDivisionError + it = aiter(spam, stop_exception=LookupError) + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_exhausted(self): + it = aiter(self.make_counter(), 3) + self.assertEqual(self.collect(it), [1, 2]) + self.assertEqual(self.loop.run_until_complete(anext(it, 'default')), + 'default') + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_lazy(self): + # The callable is only called when the awaitable is awaited + calls = [] + async def spam(): + calls.append(1) + return len(calls) + it = aiter(spam, 10) + awaitable = it.__anext__() + self.assertEqual(calls, []) + self.assertEqual(self.loop.run_until_complete(awaitable), 1) + self.assertEqual(calls, [1]) + + def test_aiter_callable_awaitable(self): + it = aiter(self.make_counter(), 10) + awaitable = it.__anext__() + self.assertIsNone(awaitable.close()) + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(awaitable) + awaitable = it.__anext__() + with self.assertRaises(KeyError): + awaitable.throw(KeyError('injected')) + + def test_aiter_callable_cancel(self): + # Cancellation is delivered to the awaited callable result + cancelled = [] + async def spam(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled.append(1) + raise + async def consume(): + async for _ in aiter(spam, None): + pass + async def main(): + task = asyncio.ensure_future(consume()) + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.loop.run_until_complete(main()) + self.assertEqual(cancelled, [1]) + + def test_aiter_callable_errors(self): + async def gen(): + yield 1 + self.assertRaises(TypeError, aiter, gen(), 1) + self.assertRaises(TypeError, aiter, [1, 2], stop_exception=LookupError) + self.assertRaises(TypeError, aiter, len, stop_exception=42) + self.assertRaises(TypeError, aiter, len, + stop_exception=(LookupError, 42)) + self.assertRaises(TypeError, aiter, len, stop_exception=LookupError()) + def test_anext_bad_args(self): async def gen(): yield 1 diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index 8b25607b6d504f..91da2cf5ee9f61 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -226,6 +226,8 @@ def test_pack_unpack_roundtrip_for_nans(self): value = unpack(data1, endian) data2 = pack(size, value, endian) self.assertTrue(math.isnan(value)) + self.assertEqual(math.copysign(1.0, value), + -1.0 if sign else 1.0) self.assertEqual(data1, data2) @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 8930f6343ac299..df5843abfcb875 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6171,10 +6171,10 @@ def test_builtins_have_signatures(self): 'dict', 'frozendict', 'int', 'str'} # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} + "next", "vars"} no_signature |= needs_groups # These have unrepresentable parameter default values of NULL - unsupported_signature = {"anext"} + unsupported_signature = {"anext", "aiter", "iter"} # These need *args support in Argument Clinic needs_varargs = {"min", "max", "__build_class__"} no_signature |= needs_varargs diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 18e4b676c53236..be9d0a709f2f4a 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -93,7 +93,7 @@ def __call__(self): i = self.i self.i = i + 1 if i > 100: - raise IndexError # Emergency stop + raise IndexError # stops the iteration return i class EmptyIterClass: @@ -350,6 +350,97 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + # Test iter() with the stop value passed by keyword + def test_iter_keyword_stop(self): + self.check_iterator(iter(CallableIterClass(), stop_value=10), list(range(10))) + + # Test iter() with the exception argument + def test_iter_exception(self): + self.check_iterator(iter(CallableIterClass(), stop_exception=IndexError), + list(range(101))) + + def test_iter_exception_tuple(self): + self.check_iterator( + iter(CallableIterClass(), stop_exception=(ZeroDivisionError, IndexError)), + list(range(101))) + + # Test iter() with both the stop value and the exception argument + def test_iter_exception_and_stop(self): + self.check_iterator(iter(CallableIterClass(), 10, stop_exception=IndexError), + list(range(10))) + self.check_iterator(iter(CallableIterClass(), 200, stop_exception=IndexError), + list(range(101))) + + # A leaking StopIteration is replaced with RuntimeError (see PEP 479) + def test_iter_exception_stop_iteration_leak(self): + def spam(): + raise StopIteration + it = iter(spam, stop_exception=IndexError) + with self.assertRaisesRegex(RuntimeError, + 'callable raised StopIteration') as cm: + next(it) + self.assertIsInstance(cm.exception.__cause__, StopIteration) + # but if it matches stop_exception, it stops the iteration + it = iter(spam, stop_exception=(IndexError, StopIteration)) + self.assertRaises(StopIteration, next, it) + + # Other exceptions are propagated + def test_iter_exception_not_matching(self): + def spam(): + raise ZeroDivisionError + it = iter(spam, stop_exception=IndexError) + self.assertRaises(ZeroDivisionError, next, it) + + def test_iter_exception_errors(self): + self.assertRaises(TypeError, iter, [1, 2], stop_exception=IndexError) + self.assertRaises(TypeError, iter, len, stop_exception=42) + self.assertRaises(TypeError, iter, len, stop_exception=(IndexError, 42)) + self.assertRaises(TypeError, iter, len, stop_exception=IndexError()) + + # StopIteration is the default stop exception + def test_iter_exception_stop_iteration(self): + def spam(state=[0]): + i = state[0] + if i == 10: + raise StopIteration + state[0] = i+1 + return i + self.check_iterator(iter(spam, stop_exception=StopIteration), + list(range(10)), pickle=False) + + def test_calliter_reduce(self): + c = CallableIterClass() + # The form without the stop exception is pickled as iter(c, stop) + self.assertEqual(iter(c, 10).__reduce__(), (iter, (c, 10))) + self.assertEqual(iter(c, 10, stop_exception=StopIteration).__reduce__(), + (iter, (c, 10))) + self.assertEqual(iter(c, 10, stop_exception=()).__reduce__(), + (iter, (c, None), ((10,), ()))) + self.assertEqual(iter(c, stop_exception=StopIteration).__reduce__(), + (iter, (c, None), ((), StopIteration))) + self.assertEqual(iter(c, stop_exception=IndexError).__reduce__(), + (iter, (c, None), ((), IndexError))) + self.assertEqual(iter(c, 10, stop_exception=IndexError).__reduce__(), + (iter, (c, None), ((10,), IndexError))) + + def test_calliter_setstate(self): + c = CallableIterClass() + it = iter(c, stop_exception=IndexError) + self.assertRaises(TypeError, it.__setstate__, 42) + self.assertRaises(TypeError, it.__setstate__, ((), IndexError, ())) + self.assertRaises(TypeError, it.__setstate__, ([], IndexError)) + self.assertRaises(TypeError, it.__setstate__, ((1, 2), IndexError)) + self.assertRaises(TypeError, it.__setstate__, ((), 42)) + self.assertRaises(TypeError, it.__setstate__, ((), None)) + it.__setstate__(((10,), StopIteration)) + self.assertEqual(it.__reduce__(), (iter, (c, 10))) + it.__setstate__(((10,), ())) + self.assertEqual(it.__reduce__(), (iter, (c, None), ((10,), ()))) + it.__setstate__(((), IndexError)) + self.assertEqual(it.__reduce__(), (iter, (c, None), ((), IndexError))) + it.__setstate__(((10,), StopIteration)) + self.assertEqual(list(it), list(range(10))) + def test_iter_function_concealing_reentrant_exhaustion(self): # gh-101892: Test two-argument iter() with a function that # exhausts its associated iterator but forgets to either return diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f2adce532595e7..4308de227a46ce 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1726,7 +1726,7 @@ def get_gen(): yield 1 check(iter('abc'), size('lP')) # callable-iterator import re - check(re.finditer('',''), size('2P')) + check(re.finditer('',''), size('3P')) # list check(list([]), vsize('Pn')) check(list([1]), vsize('Pn') + 2*self.P) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index fb35bb6a5f442f..a0337de58b23ae 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -970,6 +970,88 @@ def test_tostring_xml_declaration_cases(self): expected_retval ) + def test_tostring_standalone(self): + elem = ET.XML('') + self.assertEqual( + ET.tostring(elem, encoding='unicode', standalone=True), + "\n" + "" + ) + self.assertEqual( + ET.tostring(elem, encoding='unicode', standalone=False), + "\n" + "" + ) + # the XML declaration is written even if it would be omitted + self.assertEqual( + ET.tostring(elem, standalone=True), + b"\n" + b"" + ) + self.assertEqual( + ET.tostring(elem, encoding='UTF-8', standalone=False), + b"\n" + b"" + ) + + def test_tostring_standalone_none(self): + elem = ET.XML('') + self.assertEqual( + ET.tostring(elem, encoding='unicode', standalone=None), + '' + ) + self.assertEqual( + ET.tostring(elem, encoding='unicode', xml_declaration=True, + standalone=None), + "\n" + ) + + def test_tostring_standalone_without_xml_declaration(self): + elem = ET.XML('') + for standalone in True, False: + for xml_declaration in False, 0, '': + with self.subTest(standalone=standalone, + xml_declaration=xml_declaration): + with self.assertRaises(ValueError): + ET.tostring(elem, xml_declaration=xml_declaration, + standalone=standalone) + + def test_tostring_standalone_text_method(self): + elem = ET.XML('text') + self.assertEqual( + ET.tostring(elem, encoding='unicode', method='text', + standalone=True), + 'text' + ) + + def test_tostringlist_standalone(self): + elem = ET.XML('') + self.assertEqual( + b''.join(ET.tostringlist(elem, standalone=True)), + b"\n" + b"" + ) + with self.assertRaises(ValueError): + ET.tostringlist(elem, xml_declaration=False, standalone=False) + + def test_write_standalone(self): + elem = ET.XML('') + tree = ET.ElementTree(elem) + for standalone, expected in [ + (True, "standalone='yes'"), (False, "standalone='no'")]: + with self.subTest(standalone=standalone): + file = io.StringIO() + tree.write(file, encoding='unicode', standalone=standalone) + self.assertEqual( + file.getvalue(), + "\n" + "" % expected + ) + file = io.StringIO() + with self.assertRaises(ValueError): + tree.write(file, encoding='unicode', xml_declaration=False, + standalone=True) + def test_tostringlist_default_namespace(self): elem = ET.XML('') self.assertEqual( @@ -1096,6 +1178,41 @@ def test_parse_text_source_multiple_chunks(self): xml = "%s" % body self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + def test_parse_input_larger_than_chunk(self): + # gh-83895: the C implementation feeds Expat in chunks of 1 MiB + size = 3 * (1 << 20) + xml = '%s' % ('x' * size) + for source in xml, xml.encode(): + with self.subTest(type=type(source).__name__): + root = ET.fromstring(source) + self.assertEqual(len(root[0].text), size) + self.assertEqual(root[1].tag, 'b') + + # gh-83895: input larger than INT_MAX is fed to Expat in chunks. + # memuse is 3 for the Python implementation, which joins the collected + # data, 2 would be enough for the C implementation. + @support.bigmemtest(size=support._2G + 100, memuse=3, dry_run=False) + def test_large_input(self, size): + data = b'' + b'x' * size + b'' + root = None + try: + parser = ET.XMLParser() + parser.feed(data) + data = None + root = parser.close() + self.assertEqual(len(root.text), size) + finally: + data = None + root = None + + def test_parse_error_after_chunk_boundary(self): + # the reported position accounts for the preceding chunks + size = 2 * (1 << 20) + with self.assertRaises(ET.ParseError) as cm: + ET.fromstring('%s<' % ('x' * size)) + self.assertEqual(cm.exception.position, (1, size + 4)) + + @support.subTests('sample,exception', [ (b' \xa1', UnicodeDecodeError), # crashed (b' \xa1\n" % ( - declared_encoding,)) + if standalone is None: + sddecl = "" + else: + sddecl = " standalone='%s'" % ( + "yes" if standalone else "no",) + write("\n" % ( + declared_encoding, sddecl)) if method == "text": _serialize_text(write, self._root) else: @@ -1085,7 +1100,7 @@ def _escape_attrib_html(text): def tostring(element, encoding=None, method=None, *, xml_declaration=None, default_namespace=None, - short_empty_elements=True): + short_empty_elements=True, standalone=None): """Generate string representation of XML element. All subelements are included. If encoding is "unicode", a string @@ -1094,7 +1109,9 @@ def tostring(element, encoding=None, method=None, *, *element* is an Element instance, *encoding* is an optional output encoding defaulting to US-ASCII, *method* is an optional output which can be one of "xml" (default), "html" or "text", - *default_namespace* sets the default XML namespace (for "xmlns"). + *default_namespace* sets the default XML namespace (for "xmlns"), + *standalone* is the value of the standalone document declaration + in the XML declaration (omitted if None). Returns an (optionally) encoded string containing the XML data. @@ -1104,7 +1121,8 @@ def tostring(element, encoding=None, method=None, *, xml_declaration=xml_declaration, default_namespace=default_namespace, method=method, - short_empty_elements=short_empty_elements) + short_empty_elements=short_empty_elements, + standalone=standalone) return stream.getvalue() class _ListDataStream(io.BufferedIOBase): @@ -1126,14 +1144,15 @@ def tell(self): def tostringlist(element, encoding=None, method=None, *, xml_declaration=None, default_namespace=None, - short_empty_elements=True): + short_empty_elements=True, standalone=None): lst = [] stream = _ListDataStream(lst) ElementTree(element).write(stream, encoding, xml_declaration=xml_declaration, default_namespace=default_namespace, method=method, - short_empty_elements=short_empty_elements) + short_empty_elements=short_empty_elements, + standalone=standalone) return lst diff --git a/Makefile.pre.in b/Makefile.pre.in index adcfe4c5259eb2..b2bd89039e1230 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1361,6 +1361,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_interpframe_structs.h \ $(srcdir)/Include/internal/pycore_interpolation.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_iterobject.h \ $(srcdir)/Include/internal/pycore_jit.h \ $(srcdir)/Include/internal/pycore_lazyimportobject.h \ $(srcdir)/Include/internal/pycore_list.h \ diff --git a/Misc/NEWS.d/next/C_API/2026-07-28-04-29-52.gh-issue-153740.7_LMSA.rst b/Misc/NEWS.d/next/C_API/2026-07-28-04-29-52.gh-issue-153740.7_LMSA.rst new file mode 100644 index 00000000000000..f01009f10e9b5b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-28-04-29-52.gh-issue-153740.7_LMSA.rst @@ -0,0 +1,3 @@ +If available on the platform, use native :c:type:`_Float16` in +:c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Unpack2` functions. Patch by +Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst new file mode 100644 index 00000000000000..61a42300994aa8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst @@ -0,0 +1,5 @@ +The :func:`iter` function now accepts the *stop_exception* parameter. +The created iterator stops when the callable raises the specified exception. +The second parameter is now named *stop_value* and can be passed by keyword. +:func:`aiter` now accepts the same *stop_value* and *stop_exception* +parameters, calling an asynchronous callable and awaiting the result. diff --git a/Misc/NEWS.d/next/Library/2026-08-31-19-30-00.gh-issue-89499.Kx8vQ1.rst b/Misc/NEWS.d/next/Library/2026-08-31-19-30-00.gh-issue-89499.Kx8vQ1.rst new file mode 100644 index 00000000000000..d0b8ead7506af0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-19-30-00.gh-issue-89499.Kx8vQ1.rst @@ -0,0 +1,5 @@ +:meth:`ElementTree.write() `, +:func:`~xml.etree.ElementTree.tostring` and +:func:`~xml.etree.ElementTree.tostringlist` now support the *standalone* +parameter, the value of the standalone document declaration +in the XML declaration. diff --git a/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst b/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst new file mode 100644 index 00000000000000..c6e2c77897cbc2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst @@ -0,0 +1,4 @@ +:mod:`xml.etree.ElementTree` now accepts input larger than 2 GiB +in the C implementation. +The data is fed to Expat in chunks, as :mod:`xml.parsers.expat` already did, +instead of raising :exc:`OverflowError`. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index a49811a338e625..fa178f53e9b3ff 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3940,6 +3940,27 @@ expat_parse(elementtreestate *st, XMLParserObject *self, const char *data, Py_RETURN_NONE; } +/* Expat takes the length as an int, feed larger data in chunks. */ +#define MAX_CHUNK_SIZE (1 << 20) + +LOCAL(PyObject*) +expat_parse_large(elementtreestate *st, XMLParserObject *self, + const char *data, Py_ssize_t data_len, int final) +{ + static_assert(MAX_CHUNK_SIZE <= INT_MAX, + "MAX_CHUNK_SIZE is larger than INT_MAX"); + while (data_len > MAX_CHUNK_SIZE) { + PyObject *res = expat_parse(st, self, data, MAX_CHUNK_SIZE, 0); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); + data += MAX_CHUNK_SIZE; + data_len -= MAX_CHUNK_SIZE; + } + return expat_parse(st, self, data, (int)data_len, final); +} + /*[clinic input] _elementtree.XMLParser.close @@ -4031,26 +4052,17 @@ _elementtree_XMLParser_feed_impl(XMLParserObject *self, PyObject *data) const char *data_ptr = PyUnicode_AsUTF8AndSize(data, &data_len); if (data_ptr == NULL) return NULL; - if (data_len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "size does not fit in an int"); - return NULL; - } /* Explicitly set UTF-8 encoding. Return code ignored. */ (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); - return expat_parse(st, self, data_ptr, (int)data_len, 0); + return expat_parse_large(st, self, data_ptr, data_len, 0); } else { Py_buffer view; PyObject *res; if (PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) < 0) return NULL; - if (view.len > INT_MAX) { - PyBuffer_Release(&view); - PyErr_SetString(PyExc_OverflowError, "size does not fit in an int"); - return NULL; - } - res = expat_parse(st, self, view.buf, (int)view.len, 0); + res = expat_parse_large(st, self, view.buf, view.len, 0); PyBuffer_Release(&view); return res; } @@ -4120,14 +4132,8 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, break; } - if (PyBytes_GET_SIZE(buffer) > INT_MAX) { - Py_DECREF(buffer); - Py_DECREF(reader); - PyErr_SetString(PyExc_OverflowError, "size does not fit in an int"); - return NULL; - } - res = expat_parse( - st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), + res = expat_parse_large( + st, self, PyBytes_AS_STRING(buffer), PyBytes_GET_SIZE(buffer), 0); first = 0; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index d90900457574d9..5eb1533bdfc118 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1482,23 +1482,25 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, endoffile = (PyBytes_GET_SIZE(cres) == 0); if (self->pendingsize > 0) { - PyObject *ctr; - char *ctrdata; - if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) { PyErr_NoMemory(); goto errorexit; } rsize = PyBytes_GET_SIZE(cres) + self->pendingsize; - ctr = PyBytes_FromStringAndSize(NULL, rsize); - if (ctr == NULL) + + PyBytesWriter *writer = PyBytesWriter_Create(rsize); + if (writer == NULL) { goto errorexit; - ctrdata = PyBytes_AS_STRING(ctr); + } + char *ctrdata = PyBytesWriter_GetData(writer); memcpy(ctrdata, self->pending, self->pendingsize); memcpy(ctrdata + self->pendingsize, PyBytes_AS_STRING(cres), PyBytes_GET_SIZE(cres)); - Py_SETREF(cres, ctr); + Py_SETREF(cres, PyBytesWriter_Finish(writer)); + if (cres == NULL) { + goto errorexit; + } self->pendingsize = 0; } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 17e6a729dcd83f..e379770e10031d 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1895,6 +1895,31 @@ int PyFloat_Pack2(double x, char *data, int le) { unsigned char *p = (unsigned char *)data; +#if HAVE_FLOAT16 + /* Conversion can change NaNs type or alter payload. Here we + just fallback to the generic code, instead of providing + workarounds as for single/double precision. */ + if (!isnan(x)) { + _Float16 y = (_Float16)x; + + if (isinf(y) && !isinf(x)) { + goto Overflow; + } + + unsigned char s[sizeof(_Float16)]; + + memcpy(s, &y, sizeof(_Float16)); + if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { + p[1] = s[0]; + p[0] = s[1]; + } + else { + p[0] = s[0]; + p[1] = s[1]; + } + return 0; + } +#endif unsigned char sign; int e; double f; @@ -2090,6 +2115,24 @@ double PyFloat_Unpack2(const char *data, int le) { unsigned char *p = (unsigned char *)data; +#if HAVE_FLOAT16 + _Float16 x16; + + if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { + char buf[2]; + + buf[1] = p[0]; + buf[0] = p[1]; + memcpy(&x16, buf, 2); + } + else { + memcpy(&x16, p, 2); + } + if (!isnan(x16)) { + return x16; + } + /* Fallback to the generic code for NaNs, see PyFloat_Pack2(). */ +#endif unsigned char sign; int e; unsigned int f; diff --git a/Objects/genobject.c b/Objects/genobject.c index 6529a66fc35a6b..6a96bc27d9a950 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -542,8 +542,8 @@ gen_close(PyObject *self, PyObject *args) // Set an exception for a gen.throw() call. // Return 0 on success, -1 on failure. -static int -gen_set_exception(PyObject *typ, PyObject *val, PyObject *tb) +int +_PyGen_SetException(PyObject *typ, PyObject *val, PyObject *tb) { /* First, check the traceback argument, replacing None with NULL. */ @@ -640,7 +640,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, "cannot reuse already awaited coroutine"); return NULL; } - gen_set_exception(typ, val, tb); + _PyGen_SetException(typ, val, tb); return NULL; } @@ -718,7 +718,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, throw_here: assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_EXECUTING); - if (gen_set_exception(typ, val, tb) < 0) { + if (_PyGen_SetException(typ, val, tb) < 0) { FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); return NULL; } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index e323987601d5d4..0394227cd482db 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -5,7 +5,10 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() +#include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyerrors.h" // _PyErr_FormatFromCause() +#include "pycore_pystate.h" // _PyThreadState_GET() typedef struct { @@ -185,22 +188,44 @@ PyTypeObject PySeqIter_Type = { typedef struct { PyObject_HEAD - PyObject *it_callable; /* Set to NULL when iterator is exhausted */ - PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */ + PyObject *it_callable; /* set to NULL when the iterator is exhausted */ + PyObject *it_sentinel; /* can be NULL, and is when exhausted */ + PyObject *it_stop_exc; /* never NULL */ } calliterobject; PyObject * -PyCallIter_New(PyObject *callable, PyObject *sentinel) +_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) { calliterobject *it; + if (stop_exc == NULL) { + stop_exc = PyExc_StopIteration; + } + else if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { + return NULL; + } it = PyObject_GC_New(calliterobject, &PyCallIter_Type); if (it == NULL) return NULL; it->it_callable = Py_NewRef(callable); - it->it_sentinel = Py_NewRef(sentinel); + it->it_sentinel = Py_XNewRef(sentinel); + it->it_stop_exc = Py_NewRef(stop_exc); _PyObject_GC_TRACK(it); return (PyObject *)it; } + +PyObject * +PyCallIter_New(PyObject *callable, PyObject *sentinel) +{ + return _PyCallIter_NewEx(callable, sentinel, NULL); +} + +static void +calliter_exhaust(calliterobject *it) +{ + Py_CLEAR(it->it_callable); + Py_CLEAR(it->it_sentinel); +} + static void calliter_dealloc(PyObject *op) { @@ -208,6 +233,7 @@ calliter_dealloc(PyObject *op) _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_callable); Py_XDECREF(it->it_sentinel); + Py_XDECREF(it->it_stop_exc); PyObject_GC_Del(it); } @@ -217,6 +243,7 @@ calliter_traverse(PyObject *op, visitproc visit, void *arg) calliterobject *it = (calliterobject*)op; Py_VISIT(it->it_callable); Py_VISIT(it->it_sentinel); + Py_VISIT(it->it_stop_exc); return 0; } @@ -231,23 +258,28 @@ calliter_iternext(PyObject *op) } result = _PyObject_CallNoArgs(it->it_callable); - if (result != NULL && it->it_sentinel != NULL){ - int ok; - - ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); + /* The call can exhaust the iterator re-entrantly. */ + if (result != NULL && it->it_callable != NULL) { + if (it->it_sentinel == NULL) { + return result; /* Common case, fast path */ + } + int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); if (ok == 0) { return result; /* Common case, fast path */ } if (ok > 0) { - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); + calliter_exhaust(it); } } - else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + else if (PyErr_ExceptionMatches(it->it_stop_exc)) { PyErr_Clear(); - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); + calliter_exhaust(it); + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the end of the iteration (see PEP 479). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); } Py_XDECREF(result); return NULL; @@ -263,14 +295,57 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) * call must be before access of iterator pointers. * see issue #101765 */ - if (it->it_callable != NULL && it->it_sentinel != NULL) - return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); - else + if (it->it_callable == NULL) { return Py_BuildValue("N(())", iter); + } + /* Only the sentinel can be passed as an argument of iter(), so other + attributes are restored from the state (see calliter_setstate()). */ + if (it->it_sentinel == NULL) { + return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, + it->it_stop_exc); + } + else if (it->it_stop_exc == PyExc_StopIteration) { + return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); + } + else { + return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, + it->it_sentinel, it->it_stop_exc); + } +} + +static PyObject * +calliter_setstate(PyObject *op, PyObject *state) +{ + calliterobject *it = (calliterobject*)op; + PyObject *sentinel, *stop_exc; + + if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != 2) { + goto error; + } + sentinel = PyTuple_GET_ITEM(state, 0); + stop_exc = PyTuple_GET_ITEM(state, 1); + if (!PyTuple_Check(sentinel) || PyTuple_GET_SIZE(sentinel) > 1) { + goto error; + } + if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { + return NULL; + } + if (it->it_callable != NULL) { + Py_XSETREF(it->it_sentinel, + PyTuple_GET_SIZE(sentinel) ? + Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL); + Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc)); + } + Py_RETURN_NONE; + +error: + PyErr_SetString(PyExc_TypeError, "invalid state for callable_iterator"); + return NULL; } static PyMethodDef calliter_methods[] = { {"__reduce__", calliter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", calliter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -337,10 +412,10 @@ anextawaitable_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -anextawaitable_getiter(anextawaitableobject *obj) +awaitable_getiter(PyObject *owner, PyObject *wrapped) { - assert(obj->wrapped != NULL); - PyObject *awaitable = _PyCoro_GetAwaitableIter(obj->wrapped); + assert(wrapped != NULL); + PyObject *awaitable = _PyCoro_GetAwaitableIter(wrapped); if (awaitable == NULL) { return NULL; } @@ -359,7 +434,7 @@ anextawaitable_getiter(anextawaitableobject *obj) if (!PyIter_Check(awaitable)) { PyErr_Format(PyExc_TypeError, "%T.__await__() must return an iterable, not %T", - obj, awaitable); + owner, awaitable); Py_DECREF(awaitable); return NULL; } @@ -391,7 +466,7 @@ anextawaitable_iternext(PyObject *op) * gen.__anext__().__next__() */ anextawaitableobject *obj = anextawaitableobject_CAST(op); - PyObject *awaitable = anextawaitable_getiter(obj); + PyObject *awaitable = awaitable_getiter(op, obj->wrapped); if (awaitable == NULL) { return NULL; } @@ -411,7 +486,7 @@ anextawaitable_iternext(PyObject *op) static PyObject * anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) { - PyObject *awaitable = anextawaitable_getiter(obj); + PyObject *awaitable = awaitable_getiter((PyObject *)obj, obj->wrapped); if (awaitable == NULL) { return NULL; } @@ -540,3 +615,328 @@ PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) _PyObject_GC_TRACK(anext); return (PyObject *)anext; } + + +/* -------------------------------------- */ + +/* The asynchronous counterpart of calliterobject: the callable is called + and its result is awaited for every __anext__(). */ + +typedef struct { + PyObject_HEAD + PyObject *it_callable; /* set to NULL when the iterator is exhausted */ + PyObject *it_sentinel; /* can be NULL, and is when exhausted */ + PyObject *it_stop_exc; /* never NULL */ +} acalliterobject; + +#define acalliterobject_CAST(op) ((acalliterobject *)(op)) + +/* The awaitable returned by acalliter_anext(). The callable is only + called when this object is awaited. */ +typedef struct { + PyObject_HEAD + PyObject *aw_iterator; /* the iterator which created this object */ + PyObject *aw_wrapped; /* the awaitable returned by the callable */ + bool aw_closed; +} acallawaitableobject; + +#define acallawaitableobject_CAST(op) ((acallawaitableobject *)(op)) + +PyObject * +_PyACallIter_New(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) +{ + if (stop_exc == NULL) { + stop_exc = PyExc_StopAsyncIteration; + } + else if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { + return NULL; + } + acalliterobject *it = PyObject_GC_New(acalliterobject, &_PyACallIter_Type); + if (it == NULL) { + return NULL; + } + it->it_callable = Py_NewRef(callable); + it->it_sentinel = Py_XNewRef(sentinel); + it->it_stop_exc = Py_NewRef(stop_exc); + _PyObject_GC_TRACK(it); + return (PyObject *)it; +} + +static void +acalliter_exhaust(acalliterobject *it) +{ + Py_CLEAR(it->it_callable); + Py_CLEAR(it->it_sentinel); +} + +static void +acalliter_dealloc(PyObject *op) +{ + acalliterobject *it = acalliterobject_CAST(op); + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_callable); + Py_XDECREF(it->it_sentinel); + Py_XDECREF(it->it_stop_exc); + PyObject_GC_Del(it); +} + +static int +acalliter_traverse(PyObject *op, visitproc visit, void *arg) +{ + acalliterobject *it = acalliterobject_CAST(op); + Py_VISIT(it->it_callable); + Py_VISIT(it->it_sentinel); + Py_VISIT(it->it_stop_exc); + return 0; +} + +static PyObject *acallawaitable_new(PyObject *iterator); + +static PyObject * +acalliter_anext(PyObject *op) +{ + return acallawaitable_new(op); +} + +static PyAsyncMethods acalliter_as_async = { + 0, /* am_await */ + PyObject_SelfIter, /* am_aiter */ + acalliter_anext, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject _PyACallIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "async_callable_iterator", + .tp_basicsize = sizeof(acalliterobject), + .tp_dealloc = acalliter_dealloc, + .tp_as_async = &acalliter_as_async, + .tp_getattro = PyObject_GenericGetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = acalliter_traverse, +}; + +/* -------------------------------------- */ + +static PyObject * +acallawaitable_new(PyObject *iterator) +{ + acallawaitableobject *aw = PyObject_GC_New( + acallawaitableobject, &_PyACallIterAwaitable_Type); + if (aw == NULL) { + return NULL; + } + aw->aw_iterator = Py_NewRef(iterator); + aw->aw_wrapped = NULL; + aw->aw_closed = false; + _PyObject_GC_TRACK(aw); + return (PyObject *)aw; +} + +static void +acallawaitable_dealloc(PyObject *op) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + _PyObject_GC_UNTRACK(aw); + Py_XDECREF(aw->aw_iterator); + Py_XDECREF(aw->aw_wrapped); + PyObject_GC_Del(aw); +} + +static int +acallawaitable_traverse(PyObject *op, visitproc visit, void *arg) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + Py_VISIT(aw->aw_iterator); + Py_VISIT(aw->aw_wrapped); + return 0; +} + +/* Call the callable. Return 0 on success, -1 on failure. */ +static int +acallawaitable_start(acallawaitableobject *aw) +{ + acalliterobject *it = acalliterobject_CAST(aw->aw_iterator); + + if (aw->aw_closed) { + PyErr_SetString(PyExc_RuntimeError, + "cannot reuse already awaited __anext__()"); + return -1; + } + if (it->it_callable == NULL) { + PyErr_SetNone(PyExc_StopAsyncIteration); + return -1; + } + PyObject *awaitable = _PyObject_CallNoArgs(it->it_callable); + if (awaitable == NULL) { + if (PyErr_ExceptionMatches(it->it_stop_exc)) { + PyErr_Clear(); + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the result of the await (PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); + } + else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + /* It would be mistaken for the end of the iteration (PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopAsyncIteration"); + } + return -1; + } + aw->aw_wrapped = awaitable; + return 0; +} + +/* Turn the exception raised by the wrapped awaitable into the result of + the await. Always returns NULL. */ +static PyObject * +acallawaitable_handle_error(acallawaitableobject *aw) +{ + acalliterobject *it = acalliterobject_CAST(aw->aw_iterator); + + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyObject *value; + if (_PyGen_FetchStopIterationValue(&value) < 0) { + return NULL; + } + int ok = 0; + if (it->it_sentinel != NULL) { + ok = PyObject_RichCompareBool(it->it_sentinel, value, Py_EQ); + } + if (ok == 0) { + (void)_PyGen_SetStopIterationValue(value); + } + else if (ok > 0) { + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + Py_DECREF(value); + return NULL; + } + if (PyErr_ExceptionMatches(it->it_stop_exc)) { + PyErr_Clear(); + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + /* It would be mistaken for the end of the iteration (see PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopAsyncIteration"); + } + return NULL; +} + +static PyObject * +acallawaitable_iternext(PyObject *op) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL && acallawaitable_start(aw) < 0) { + return NULL; + } + PyObject *awaitable = awaitable_getiter(op, aw->aw_wrapped); + if (awaitable == NULL) { + return NULL; + } + PyObject *result = (*Py_TYPE(awaitable)->tp_iternext)(awaitable); + Py_DECREF(awaitable); + if (result != NULL) { + return result; + } + return acallawaitable_handle_error(aw); +} + +static PyObject * +acallawaitable_proxy(acallawaitableobject *aw, char *meth, PyObject *arg) +{ + PyObject *awaitable = awaitable_getiter((PyObject *)aw, aw->aw_wrapped); + if (awaitable == NULL) { + return NULL; + } + // When specified, 'arg' may be a tuple (if coming from a METH_VARARGS + // method) or a single object (if coming from a METH_O method). + PyObject *ret = arg == NULL + ? PyObject_CallMethod(awaitable, meth, NULL) + : PyObject_CallMethod(awaitable, meth, "O", arg); + Py_DECREF(awaitable); + if (ret != NULL) { + return ret; + } + return acallawaitable_handle_error(aw); +} + +static PyObject * +acallawaitable_send(PyObject *op, PyObject *arg) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL && acallawaitable_start(aw) < 0) { + return NULL; + } + return acallawaitable_proxy(aw, "send", arg); +} + +static PyObject * +acallawaitable_throw(PyObject *op, PyObject *args) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL) { + /* Not started, so the exception is raised at the point of the + await, as for a not started coroutine. */ + PyObject *typ, *val = NULL, *tb = NULL; + if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) { + return NULL; + } + aw->aw_closed = true; + (void)_PyGen_SetException(typ, val, tb); + return NULL; + } + return acallawaitable_proxy(aw, "throw", args); +} + +static PyObject * +acallawaitable_close(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL) { + /* Not started, so there is nothing to close. */ + aw->aw_closed = true; + Py_RETURN_NONE; + } + PyObject *result = acallawaitable_proxy(aw, "close", NULL); + aw->aw_closed = true; + return result; +} + +static PyMethodDef acallawaitable_methods[] = { + {"send", acallawaitable_send, METH_O, send_doc}, + {"throw", acallawaitable_throw, METH_VARARGS, throw_doc}, + {"close", acallawaitable_close, METH_NOARGS, close_doc}, + {NULL, NULL} /* Sentinel */ +}; + +static PyAsyncMethods acallawaitable_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject _PyACallIterAwaitable_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "async_callable_iterator_awaitable", + .tp_basicsize = sizeof(acallawaitableobject), + .tp_dealloc = acallawaitable_dealloc, + .tp_as_async = &acallawaitable_as_async, + .tp_getattro = PyObject_GenericGetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = acallawaitable_traverse, + .tp_iter = PyObject_SelfIter, + .tp_iternext = acallawaitable_iternext, + .tp_methods = acallawaitable_methods, +}; diff --git a/Objects/object.c b/Objects/object.c index fadd9273a36607..c0cb0da7a0d92e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2519,6 +2519,8 @@ _PyObject_FiniState(PyInterpreterState *interp) } +extern PyTypeObject _PyACallIter_Type; +extern PyTypeObject _PyACallIterAwaitable_Type; extern PyTypeObject _PyAnextAwaitable_Type; extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; @@ -2612,6 +2614,8 @@ static PyTypeObject* static_types[_Py_NUM_MANAGED_PREINITIALIZED_TYPES] = { &PyWrapperDescr_Type, &PyZip_Type, &Py_GenericAliasType, + &_PyACallIter_Type, + &_PyACallIterAwaitable_Type, &_PyAnextAwaitable_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 33647ec284061f..79dfc9ccf39ec2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -276,6 +276,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 434dd13267fe93..765b4d46b12dd0 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -747,6 +747,9 @@ Include\cpython + + Include\cpython + Include\internal diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index cbe59c8883d5a5..d28e6fa9cd01ae 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -10,6 +10,7 @@ #include "pycore_floatobject.h" // _PyFloat_ExactDealloc() #include "pycore_interp.h" // _PyInterpreterState_GetConfig() #include "pycore_import.h" // _PyImport_LazyImportModuleLevelObject () +#include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -1893,50 +1894,70 @@ builtin_hex(PyObject *module, PyObject *integer) } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +@text_signature "($module, object, /, [stop_value], *, stop_exception=StopIteration)" +iter as builtin_iter + + object: object + / + stop_value: object = NULL + * + stop_exception: object = NULL + +Get an iterator from an object. + +In the first form, the argument must supply its own iterator, or be a +sequence. In the second form, the callable is called until it returns +the stop value or raises the specified exception. +[clinic start generated code]*/ + static PyObject * -builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception) +/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d3a2f767f29d9ae6]*/ { - PyObject *v; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) - return NULL; - v = args[0]; - if (nargs == 1) - return PyObject_GetIter(v); - if (!PyCallable_Check(v)) { + if (stop_value == NULL && stop_exception == NULL) { + return PyObject_GetIter(object); + } + if (!PyCallable_Check(object)) { PyErr_SetString(PyExc_TypeError, - "iter(v, w): v must be callable"); + "iter(): the first argument must be callable"); return NULL; } - PyObject *sentinel = args[1]; - return PyCallIter_New(v, sentinel); + return _PyCallIter_NewEx(object, stop_value, stop_exception); } -PyDoc_STRVAR(iter_doc, -"iter(iterable) -> iterator\n\ -iter(callable, sentinel) -> iterator\n\ -\n\ -Get an iterator from an object. In the first form, the argument must\n\ -supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the\n\ -sentinel."); - /*[clinic input] +@text_signature "($module, object, /, [stop_value], *, stop_exception=StopAsyncIteration)" aiter as builtin_aiter - async_iterable: object + object: object / + stop_value: object = NULL + * + stop_exception: object = NULL Return an AsyncIterator for an AsyncIterable object. + +In the second form, the callable is called and its result is awaited +until it returns the stop value or raises the specified exception. [clinic start generated code]*/ static PyObject * -builtin_aiter(PyObject *module, PyObject *async_iterable) -/*[clinic end generated code: output=1bae108d86f7960e input=473993d0cacc7d23]*/ +builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception) +/*[clinic end generated code: output=2865edb3fbc45693 input=2adb37d12adafd0c]*/ { - return PyObject_GetAIter(async_iterable); + if (stop_value == NULL && stop_exception == NULL) { + return PyObject_GetAIter(object); + } + if (!PyCallable_Check(object)) { + PyErr_SetString(PyExc_TypeError, + "aiter(): the first argument must be callable"); + return NULL; + } + return _PyACallIter_New(object, stop_value, stop_exception); } PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); @@ -3472,7 +3493,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, + BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4a38e0df61708c..c10bb03d817816 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -850,14 +850,166 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, +PyDoc_STRVAR(builtin_iter__doc__, +"iter($module, object, /, [stop_value], *, stop_exception=StopIteration)\n" +"--\n" +"\n" +"Get an iterator from an object.\n" +"\n" +"In the first form, the argument must supply its own iterator, or be a\n" +"sequence. In the second form, the callable is called until it returns\n" +"the stop value or raises the specified exception."); + +#define BUILTIN_ITER_METHODDEF \ + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL|METH_KEYWORDS, builtin_iter__doc__}, + +static PyObject * +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception); + +static PyObject * +builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(stop_value), &_Py_ID(stop_exception), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "stop_value", "stop_exception", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iter", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *object; + PyObject *stop_value = NULL; + PyObject *stop_exception = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + stop_value = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + stop_exception = args[2]; +skip_optional_kwonly: + return_value = builtin_iter_impl(module, object, stop_value, stop_exception); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, async_iterable, /)\n" +"aiter($module, object, /, [stop_value], *, stop_exception=StopAsyncIteration)\n" "--\n" "\n" -"Return an AsyncIterator for an AsyncIterable object."); +"Return an AsyncIterator for an AsyncIterable object.\n" +"\n" +"In the second form, the callable is called and its result is awaited\n" +"until it returns the stop value or raises the specified exception."); #define BUILTIN_AITER_METHODDEF \ - {"aiter", (PyCFunction)builtin_aiter, METH_O, builtin_aiter__doc__}, + {"aiter", _PyCFunction_CAST(builtin_aiter), METH_FASTCALL|METH_KEYWORDS, builtin_aiter__doc__}, + +static PyObject * +builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception); + +static PyObject * +builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(stop_value), &_Py_ID(stop_exception), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "stop_value", "stop_exception", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "aiter", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *object; + PyObject *stop_value = NULL; + PyObject *stop_exception = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + stop_value = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + stop_exception = args[2]; +skip_optional_kwonly: + return_value = builtin_aiter_impl(module, object, stop_value, stop_exception); + +exit: + return return_value; +} PyDoc_STRVAR(builtin_anext__doc__, "anext($module, async_iterator, default=, /)\n" @@ -1387,4 +1539,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=84efa9c5cc737ce5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5fb1ac6a4253ee2f input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index 6d1ae651fa0005..84a0589c8e8317 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -1426,19 +1426,19 @@ _PyCodec_SurrogateEscapeUnicodeEncodeError(PyObject *exc) return NULL; } - PyObject *res = PyBytes_FromStringAndSize(NULL, slen); - if (res == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(slen); + if (writer == NULL) { Py_DECREF(obj); return NULL; } - char *outp = PyBytes_AsString(res); + char *outp = PyBytesWriter_GetData(writer); for (Py_ssize_t i = start; i < end; i++) { Py_UCS4 ch = PyUnicode_READ_CHAR(obj, i); if (ch < 0xdc80 || ch > 0xdcff) { /* Not a UTF-8b surrogate, fail with original exception. */ Py_DECREF(obj); - Py_DECREF(res); + PyBytesWriter_Discard(writer); PyErr_SetObject(PyExceptionInstance_Class(exc), exc); return NULL; } @@ -1446,6 +1446,8 @@ _PyCodec_SurrogateEscapeUnicodeEncodeError(PyObject *exc) } Py_DECREF(obj); + PyObject *res = PyBytesWriter_Finish(writer); + // Py_BuildValue() propagates the exception if res is NULL return Py_BuildValue("(Nn)", res, end); } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index db575d870be5c5..148f6e68ab806e 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -58,6 +58,8 @@ Objects/genobject.c - _PyCoroWrapper_Type - Objects/interpolationobject.c - _PyInterpolation_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - +Objects/iterobject.c - _PyACallIter_Type - +Objects/iterobject.c - _PyACallIterAwaitable_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - Objects/lazyimportobject.c - PyLazyImport_Type - Objects/listobject.c - PyListIter_Type - @@ -73,6 +75,8 @@ Objects/moduleobject.c - PyModule_Type - Objects/namespaceobject.c - _PyNamespace_Type - Objects/object.c - _PyNone_Type - Objects/object.c - _PyNotImplemented_Type - +Objects/object.c - _PyACallIter_Type - +Objects/object.c - _PyACallIterAwaitable_Type - Objects/object.c - _PyAnextAwaitable_Type - Objects/odictobject.c - PyODictItems_Type - Objects/odictobject.c - PyODictIter_Type - diff --git a/configure b/configure index 6b560fe6841b72..e0ad04b036aeb1 100755 --- a/configure +++ b/configure @@ -16724,6 +16724,65 @@ printf "%s\n" "#define _Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h fi +# Check for native half-float type (_Float16). +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _Float16 support" >&5 +printf %s "checking for _Float16 support... " >&6; } +if test ${ac_cv_float16_supported+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + +CFLAGS="$CFLAGS -O0" +if test "$cross_compiling" = yes +then : + ac_cv_float16_supported=no +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int main(void) +{ + _Float16 val = 1.0f16; + double d = 3.14; + val = d; + return 0; +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_float16_supported=yes +else case e in #( + e) ac_cv_float16_supported=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_float16_supported" >&5 +printf "%s\n" "$ac_cv_float16_supported" >&6; } +if test "x$ac_cv_float16_supported" = xyes +then : + +printf "%s\n" "#define HAVE_FLOAT16 1" >>confdefs.h + +fi + pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 diff --git a/configure.ac b/configure.ac index 476f13c82bbb2b..2b8a5b052878c9 100644 --- a/configure.ac +++ b/configure.ac @@ -4469,6 +4469,25 @@ if test "$ac_cv_ffi_complex_double_supported" = "yes"; then [Defined if _Complex C type can be used with libffi.]) fi +# Check for native half-float type (_Float16). +AC_CACHE_CHECK([for _Float16 support], [ac_cv_float16_supported], +WITH_SAVE_ENV([ +CFLAGS="$CFLAGS -O0" +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +int main(void) +{ + _Float16 val = 1.0f16; + double d = 3.14; + val = d; + return 0; +} +]])], [ac_cv_float16_supported=yes], +[ac_cv_float16_supported=no], +[ac_cv_float16_supported=no])])) +AS_VAR_IF([ac_cv_float16_supported], [yes], + [AC_DEFINE([HAVE_FLOAT16], [1], + [Defined if _Float16 C type is supported])]) + dnl Check for libmpdec >= 2.5.0 PKG_CHECK_MODULES([LIBMPDEC], [libmpdec >= 2.5.0], [have_mpdec=yes], [ WITH_SAVE_ENV([ diff --git a/pyconfig.h.in b/pyconfig.h.in index 64b5c52790b458..c2a36afe334763 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -497,6 +497,9 @@ /* Define if you have the 'ffi_prep_closure_loc' function. */ #undef HAVE_FFI_PREP_CLOSURE_LOC +/* Defined if _Float16 C type is supported */ +#undef HAVE_FLOAT16 + /* Define to 1 if you have the 'flock' function. */ #undef HAVE_FLOCK