Skip to content

Commit e1c7fab

Browse files
committed
Merge branch 'main' into pydoc_dunders
2 parents b826b60 + 64018ac commit e1c7fab

13 files changed

Lines changed: 170 additions & 75 deletions

File tree

‎.github/dependabot.yml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,3 @@ updates:
117117
patterns: ["*"]
118118
cooldown:
119119
default-days: 14
120-
121-
- package-ecosystem: "pip"
122-
target-branch: "3.13"
123-
directory: "/Tools/"
124-
schedule:
125-
interval: "quarterly"
126-
labels: ["skip issue", "skip news"]
127-
groups:
128-
pip:
129-
patterns: ["*"]
130-
cooldown:
131-
default-days: 14

‎Doc/library/multiprocessing.rst‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,21 @@ to start a process. These *start methods* are
160160
No unnecessary resources are inherited.
161161

162162
Available on POSIX platforms which support passing file descriptors over
163-
Unix pipes such as Linux. The default on those.
163+
Unix pipes such as Linux. The default on those, unless there is no
164+
writable temporary directory (see :func:`tempfile.gettempdir`), which
165+
the fork server requires; *spawn* is the default in that case.
164166

165167
.. versionchanged:: 3.14
166168
This became the default start method on POSIX platforms.
167169

170+
.. versionchanged:: next
171+
*spawn* is now the default start method when there is no writable
172+
temporary directory, for example on a read-only file system.
173+
Previously *forkserver* remained the default there and starting a
174+
process failed with an :exc:`OSError`. The default is determined
175+
when it is first needed rather than when :mod:`!multiprocessing`
176+
is imported. See :gh:`155717`.
177+
168178
.. versionchanged:: 3.4
169179
*spawn* added on all POSIX platforms, and *forkserver* added for
170180
some POSIX platforms.

‎Doc/tools/removed-ids.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ using/windows.html: the-full-installer-deprecated
9393
using/windows.html: virtual-environments
9494
using/windows.html: windows-full
9595

96+
# Duplicated label from reference/datamodel:
97+
using/windows.html: customization
98+
9699
# Moved to library/stdtypes:
97100
reference/expressions.html: agen.__anext__
98101
reference/expressions.html: agen.aclose

‎Lib/annotationlib.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,9 +1105,17 @@ def type_repr(value):
11051105
11061106
"""
11071107
if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)):
1108-
if value.__module__ == "builtins":
1108+
module = value.__module__
1109+
if module is None:
1110+
# Built-in methods have __module__ set to None; recover the module
1111+
# from the bound object (the class itself for class-bound methods).
1112+
obj = getattr(value, "__self__", None)
1113+
if obj is not None:
1114+
cls = obj if isinstance(obj, type) else type(obj)
1115+
module = cls.__module__
1116+
if module is None or module == "builtins":
11091117
return value.__qualname__
1110-
return f"{value.__module__}.{value.__qualname__}"
1118+
return f"{module}.{value.__qualname__}"
11111119
elif isinstance(value, _Template):
11121120
tree = _template_to_ast(value)
11131121
return ast.unparse(tree)

‎Lib/multiprocessing/context.py‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
import functools
12
import os
23
import sys
34
import threading
45

56
from . import process
67
from . import reduction
7-
from . import util
88

99
__all__ = ()
1010

@@ -240,10 +240,19 @@ def _after_fork():
240240
class DefaultContext(BaseContext):
241241
Process = Process
242242

243-
def __init__(self, context):
244-
self._default_context = context
243+
def __init__(self, context=None):
244+
if context is not None:
245+
self._default_context = context
245246
self._actual_context = None
246247

248+
# The platform default is only determined when first needed, as doing
249+
# so can be costly; importing multiprocessing must stay cheap.
250+
# A cached_property rather than a property so that the attribute can
251+
# still be assigned, deleted and patched like a plain one.
252+
@functools.cached_property
253+
def _default_context(self):
254+
return _get_platform_default_context()
255+
247256
def get_context(self, method=None):
248257
if method is None:
249258
if self._actual_context is None:
@@ -331,18 +340,21 @@ def _check_available(self):
331340
if reduction.HAVE_SEND_HANDLE:
332341
_concrete_contexts['forkserver'] = ForkServerContext()
333342

334-
# bpo-33725: running arbitrary code after fork() is no longer reliable
335-
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
336-
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
337-
if (
338-
reduction.HAVE_SEND_HANDLE
339-
and sys.platform != 'darwin'
340-
# gh-155717: forkserver requires to write temporary files
341-
and util._has_writeable_tempdir()
342-
):
343-
_default_context = DefaultContext(_concrete_contexts['forkserver'])
344-
else:
345-
_default_context = DefaultContext(_concrete_contexts['spawn'])
343+
def _get_platform_default_context():
344+
# bpo-33725: running arbitrary code after fork() is no longer reliable
345+
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
346+
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
347+
if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
348+
# gh-155717: forkserver requires to write temporary files.
349+
#
350+
# Imported here rather than at the top level as util is costly
351+
# to import and is not otherwise needed to import this module.
352+
from . import util
353+
if util._has_writeable_tempdir():
354+
return _concrete_contexts['forkserver']
355+
return _concrete_contexts['spawn']
356+
357+
_default_context = DefaultContext()
346358

347359
else: # Windows
348360

@@ -365,7 +377,10 @@ class SpawnContext(BaseContext):
365377
_concrete_contexts = {
366378
'spawn': SpawnContext(),
367379
}
368-
_default_context = DefaultContext(_concrete_contexts['spawn'])
380+
def _get_platform_default_context():
381+
return _concrete_contexts['spawn']
382+
383+
_default_context = DefaultContext()
369384

370385
#
371386
# Force the start method

‎Lib/multiprocessing/util.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import os
1111
import itertools
1212
import sys
13-
import tempfile
1413
import weakref
1514
import atexit
1615
import threading # we want threading to install it's
@@ -155,7 +154,7 @@ def _remove_temp_dir(rmtree, tempdir):
155154
current_process._config['tempdir'] = None
156155

157156

158-
def _get_base_temp_dir():
157+
def _get_base_temp_dir(tempfile):
159158
"""Get a temporary directory where socket files will be created.
160159
161160
To prevent additional imports, pass a pre-imported 'tempfile' module.
@@ -216,8 +215,9 @@ def get_temp_dir():
216215
# get name of a temp directory which will be automatically cleaned up
217216
tempdir = process.current_process()._config.get('tempdir')
218217
if tempdir is None:
219-
import shutil
220-
base_tempdir = _get_base_temp_dir()
218+
# Deferred until first use to keep importing this module cheap.
219+
import shutil, tempfile
220+
base_tempdir = _get_base_temp_dir(tempfile)
221221
tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
222222
info('created temp directory %s', tempdir)
223223
# keep a strong reference to shutil.rmtree(), since the finalizer
@@ -234,7 +234,14 @@ def _has_writeable_tempdir():
234234
#
235235
# See: https://github.com/python/cpython/issues/155717.
236236

237-
path = _get_base_temp_dir()
237+
# Deferred until first use to keep importing this module cheap.
238+
import tempfile
239+
240+
try:
241+
path = _get_base_temp_dir(tempfile)
242+
except OSError:
243+
# No usable temporary directory at all.
244+
return False
238245
if path is None:
239246
return False
240247

‎Lib/test/_test_multiprocessing.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6516,6 +6516,62 @@ def test_forkserver_requires_writeable_tempdir(self, mode):
65166516
_, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
65176517
self.assertEqual(out.decode().strip(), "spawn")
65186518

6519+
@unittest.skipIf(os.name == "nt", "requires POSIX")
6520+
@support.requires_non_root_user
6521+
def test_default_start_method_without_usable_tempdir(self):
6522+
# Neither importing multiprocessing nor determining the default
6523+
# start method may fail when tempfile finds no usable temporary
6524+
# directory at all, and the default start method must be 'spawn'.
6525+
#
6526+
# See https://github.com/python/cpython/issues/155717.
6527+
6528+
cmd = '''if 1:
6529+
import os, tempfile
6530+
# Make TMPDIR the only candidate, so that tempfile.gettempdir()
6531+
# raises FileNotFoundError instead of falling back to /tmp.
6532+
tempfile._candidate_tempdir_list = lambda: [os.environ["TMPDIR"]]
6533+
try:
6534+
tempfile.gettempdir()
6535+
except FileNotFoundError:
6536+
pass
6537+
else:
6538+
raise AssertionError("TMPDIR is usable")
6539+
6540+
import multiprocessing
6541+
if __name__ == "__main__":
6542+
print(multiprocessing.get_start_method(),
6543+
multiprocessing.get_all_start_methods()[0])
6544+
'''
6545+
6546+
with support.os_helper.temp_dir() as root:
6547+
TMPDIR = pathlib.Path(root, "TMPDIR")
6548+
TMPDIR.mkdir(mode=os.R_OK | os.X_OK)
6549+
_, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
6550+
self.assertEqual(out.decode().split(), ["spawn", "spawn"])
6551+
6552+
def test_import_does_not_determine_default_start_method(self):
6553+
# Determining the default start method requires checking that the
6554+
# temporary directory is writeable. That must not happen, nor may
6555+
# the modules it needs be loaded, when importing multiprocessing.
6556+
import_helper.ensure_lazy_imports(
6557+
"multiprocessing", {"tempfile", "shutil", "subprocess"})
6558+
6559+
cmd = '''if 1:
6560+
import sys
6561+
import multiprocessing
6562+
print(multiprocessing.get_start_method(allow_none=True),
6563+
"tempfile" in sys.modules,
6564+
multiprocessing.get_start_method(),
6565+
multiprocessing.get_all_start_methods()[0])
6566+
'''
6567+
# -S: site customizations could import tempfile themselves.
6568+
_, out, err = script_helper.assert_python_ok('-S', '-c', cmd)
6569+
unset, probed, method, first = out.decode().split()
6570+
self.assertEqual(unset, "None")
6571+
self.assertEqual(probed, "False")
6572+
self.assertEqual(method, first)
6573+
self.assertIn(method, multiprocessing.get_all_start_methods())
6574+
65196575

65206576
@unittest.skipIf(sys.platform == "win32",
65216577
"test semantics don't make sense on Windows")

‎Lib/test/test_annotationlib.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1888,6 +1888,18 @@ def nested():
18881888
type_repr(nested), f"{__name__}.TestTypeRepr.test_type_repr.<locals>.nested"
18891889
)
18901890
self.assertEqual(type_repr(len), "len")
1891+
# gh-152692: built-in methods have __module__ set to None
1892+
self.assertEqual(type_repr([].append), "list.append")
1893+
self.assertEqual(type_repr((1).bit_length), "int.bit_length")
1894+
self.assertEqual(type_repr(dict.fromkeys), "dict.fromkeys")
1895+
# Methods of extension types also have __module__ set to None, but
1896+
# are not in the builtins module.
1897+
import datetime
1898+
self.assertEqual(
1899+
type_repr(datetime.datetime(2000, 1, 1).isoformat),
1900+
"datetime.datetime.isoformat",
1901+
)
1902+
self.assertEqual(type_repr(datetime.date.today), "datetime.date.today")
18911903
self.assertEqual(type_repr(type_repr), "annotationlib.type_repr")
18921904
self.assertEqual(type_repr(times_three), f"{__name__}.times_three")
18931905
self.assertEqual(type_repr(...), "...")
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix :func:`annotationlib.type_repr` returning a string such as
2+
``None.list.append`` for bound built-in methods, whose ``__module__`` is
3+
``None``. It now returns the qualified name, as it already did for other
4+
builtins.
Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1-
:mod:`multiprocessing`'s default start method on systems with non-writeable
2-
tempfile filesystem is now :ref:`"spawn" <multiprocessing-start-methods>`
3-
instead of ``"forkserver"``. Patch by Bénédikt Tran.
1+
:mod:`multiprocessing`'s default start method is now
2+
:ref:`"spawn" <multiprocessing-start-methods>` instead of ``"forkserver"``
3+
when there is no writable temporary directory, such as on a read-only file
4+
system. Previously starting a process failed with an :exc:`OSError` there.
5+
Patch by Bénédikt Tran.

0 commit comments

Comments
 (0)