From feefcf75b57a2db21f9377a7ed00cf9a25ffcbe1 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 11 Aug 2026 10:18:10 +0200 Subject: [PATCH] Use raw syscalls for copy_file_range() and memfd_create() glibc only grew the copy_file_range() and memfd_create() wrappers in 2.27, and CPython compiles os.copy_file_range() and os.memfd_create() out when the libc it is built against lacks them. We worked around that by forcing the configure checks on and weak linking the wrappers, which kept the functions out of the os module whenever the runtime glibc was older than 2.27, even on kernels implementing the syscalls. Backport https://github.com/python/cpython/pull/155520 instead, which calls the wrappers when they exist and issues the raw syscalls when they don't, so both functions work on any sufficiently new kernel regardless of the glibc in use. The UAPI header overlay already provides the __NR_ constants and the MFD_ flags for all glibc targets. A single patch covers 3.10 through 3.15, so the weak linking patches and the 3.10 specific configure patch go away. The distribution tests now assert that both functions are always present on Linux GNU targets instead of tying their availability to the runtime libc. Signed-off-by: Daan De Meyer --- cpython-unix/build-cpython.sh | 42 ++----- .../patch-configure-memfd-create-3.10.patch | 22 ---- ...osixmodule-copy-file-range-weak-3.13.patch | 31 ----- ...tch-posixmodule-copy-file-range-weak.patch | 31 ----- ...h-posixmodule-memfd-create-weak-3.13.patch | 35 ------ .../patch-posixmodule-memfd-create-weak.patch | 35 ------ .../patch-posixmodule-syscall-fallback.patch | 108 ++++++++++++++++++ pythonbuild/disttests/__init__.py | 20 +--- 8 files changed, 120 insertions(+), 204 deletions(-) delete mode 100644 cpython-unix/patch-configure-memfd-create-3.10.patch delete mode 100644 cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch delete mode 100644 cpython-unix/patch-posixmodule-copy-file-range-weak.patch delete mode 100644 cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch delete mode 100644 cpython-unix/patch-posixmodule-memfd-create-weak.patch create mode 100644 cpython-unix/patch-posixmodule-syscall-fallback.patch diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index efdf18480..932ea6d01 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -331,6 +331,14 @@ if [[ -n "${LINUX_UAPI_INCLUDE_ARCH:-}" ]]; then fi fi +# glibc only grew copy_file_range() and memfd_create() in 2.27 and CPython +# compiles os.copy_file_range() and os.memfd_create() out when the libc it is +# built against lacks them, which loses them for good in builds targeting an +# older glibc. Call the raw syscalls when the wrappers are missing so both +# functions keep working on any kernel that implements them. +# Backport of https://github.com/python/cpython/pull/155520. +patch -p1 -i "${ROOT}/patch-posixmodule-syscall-fallback.patch" + # Most bits look at CFLAGS. But setup.py only looks at CPPFLAGS. # So we need to set both. CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include -I${TOOLS_PATH}/deps/include/ncursesw" @@ -606,40 +614,6 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_explicit_bzero=no" fi -# The modern UAPI overlay provides the memfd constants, but some glibc sysroots -# predate the memfd_create() wrapper. Force the configure check on and weak-link -# the wrapper so the function is exposed only when runtime glibc provides it. -# This workaround is specific to glibc builds; the UAPI overlay itself can also -# be used with other Linux libcs. -if [[ -n "${LINUX_UAPI_INCLUDE_ARCH:-}" && "${TARGET_TRIPLE}" == *-linux-gnu* ]]; then - if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - patch -p1 -i "${ROOT}/patch-posixmodule-memfd-create-weak.patch" - else - patch -p1 -i "${ROOT}/patch-posixmodule-memfd-create-weak-3.13.patch" - fi - - # Python 3.10 checks for memfd_create with a custom compile test instead - # of the cached ac_cv_func_memfd_create check used by newer versions. - if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then - patch -p1 -i "${ROOT}/patch-configure-memfd-create-3.10.patch" - fi - - CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_memfd_create=yes" -fi - -# Some glibc sysroots predate the copy_file_range() wrapper. Force the -# configure check on and weak-link the wrapper so the function is exposed only -# when runtime glibc provides it. -if [[ "${TARGET_TRIPLE}" == *-linux-gnu* ]]; then - if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - patch -p1 -i "${ROOT}/patch-posixmodule-copy-file-range-weak.patch" - else - patch -p1 -i "${ROOT}/patch-posixmodule-copy-file-range-weak-3.13.patch" - fi - - CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_copy_file_range=yes" -fi - # Define the base PGO profiling task, which we'll extend below with ignores export PROFILE_TASK='-m test --pgo' diff --git a/cpython-unix/patch-configure-memfd-create-3.10.patch b/cpython-unix/patch-configure-memfd-create-3.10.patch deleted file mode 100644 index 0b27ad583..000000000 --- a/cpython-unix/patch-configure-memfd-create-3.10.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/configure.ac b/configure.ac ---- a/configure.ac -+++ b/configure.ac -@@ -3822,6 +3822,10 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[void *x=_dyl - ]) - - AC_MSG_CHECKING(for memfd_create) -+if test "$ac_cv_func_memfd_create" = yes; then -+ AC_DEFINE(HAVE_MEMFD_CREATE, 1, Define if you have the 'memfd_create' function.) -+ AC_MSG_RESULT(yes) -+else - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef HAVE_SYS_MMAN_H - #include -@@ -3834,6 +3838,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) - ]) -+fi - - AC_MSG_CHECKING(for eventfd) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch b/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch deleted file mode 100644 index d1e7e5e3f..000000000 --- a/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -10431,4 +10431,9 @@ - #ifdef HAVE_COPY_FILE_RANGE -+/* Weak references. */ -+__attribute__((weak)) -+ssize_t copy_file_range(int src, off_t *offset_src, int dst, -+ off_t *offset_dst, size_t count, unsigned int flags); -+ - /*[clinic input] - - os.copy_file_range -@@ -15737,5 +15742,17 @@ posixmodule_exec(PyObject *m) - #endif - -+#ifdef HAVE_COPY_FILE_RANGE -+ if (copy_file_range == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_DelItemString(dict, "copy_file_range") < 0) { -+ return -1; -+ } -+ } -+#endif -+ - /* Initialize environ dictionary */ - PyObject *v = convertenviron(); - Py_XINCREF(v); diff --git a/cpython-unix/patch-posixmodule-copy-file-range-weak.patch b/cpython-unix/patch-posixmodule-copy-file-range-weak.patch deleted file mode 100644 index 903e0f39b..000000000 --- a/cpython-unix/patch-posixmodule-copy-file-range-weak.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -12488,4 +12488,9 @@ - #ifdef HAVE_COPY_FILE_RANGE -+/* Weak references. */ -+__attribute__((weak)) -+ssize_t copy_file_range(int src, off_t *offset_src, int dst, -+ off_t *offset_dst, size_t count, unsigned int flags); -+ - /*[clinic input] - - os.copy_file_range -@@ -18147,5 +18152,17 @@ posixmodule_exec(PyObject *m) - #endif - -+#ifdef HAVE_COPY_FILE_RANGE -+ if (copy_file_range == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_PopString(dict, "copy_file_range", NULL) < 0) { -+ return -1; -+ } -+ } -+#endif -+ - /* Initialize environ dictionary */ - if (PyModule_Add(m, "environ", convertenviron()) != 0) { - return -1; diff --git a/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch b/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch deleted file mode 100644 index 7e115b101..000000000 --- a/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -528,6 +528,12 @@ extern char *ctermid_r(char *); - # include - #endif - -+#ifdef HAVE_MEMFD_CREATE -+/* Weak references. */ -+__attribute__((weak)) -+int memfd_create(const char *name, unsigned int flags); -+#endif -+ - /* eventfd() */ - #ifdef HAVE_SYS_EVENTFD_H - # include -@@ -15718,6 +15724,18 @@ posixmodule_exec(PyObject *m) - { - _posixstate *state = get_posix_state(m); - -+#ifdef HAVE_MEMFD_CREATE -+ if (memfd_create == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_DelItemString(dict, "memfd_create") < 0) { -+ return -1; -+ } -+ } -+#endif -+ - #if defined(HAVE_PWRITEV) - if (HAVE_PWRITEV_RUNTIME) {} else { - PyObject* dct = PyModule_GetDict(m); diff --git a/cpython-unix/patch-posixmodule-memfd-create-weak.patch b/cpython-unix/patch-posixmodule-memfd-create-weak.patch deleted file mode 100644 index 229f14ea9..000000000 --- a/cpython-unix/patch-posixmodule-memfd-create-weak.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -192,6 +192,12 @@ - # include // memfd_create(), MFD_CLOEXEC - #endif - -+#ifdef HAVE_MEMFD_CREATE -+/* Weak references. */ -+__attribute__((weak)) -+int memfd_create(const char *name, unsigned int flags); -+#endif -+ - #ifdef HAVE_SYS_EVENTFD_H - # include // eventfd() - #endif -@@ -18128,6 +18134,18 @@ posixmodule_exec(PyObject *m) - { - _posixstate *state = get_posix_state(m); - -+#ifdef HAVE_MEMFD_CREATE -+ if (memfd_create == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_PopString(dict, "memfd_create", NULL) < 0) { -+ return -1; -+ } -+ } -+#endif -+ - #if defined(HAVE_PWRITEV) - if (HAVE_PWRITEV_RUNTIME) {} else { - PyObject* dct = PyModule_GetDict(m); diff --git a/cpython-unix/patch-posixmodule-syscall-fallback.patch b/cpython-unix/patch-posixmodule-syscall-fallback.patch new file mode 100644 index 000000000..9174ac617 --- /dev/null +++ b/cpython-unix/patch-posixmodule-syscall-fallback.patch @@ -0,0 +1,108 @@ +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -8766,5 +8766,5 @@ exit: + #endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */ + +-#if defined(HAVE_COPY_FILE_RANGE) ++#if defined(_Py_HAVE_COPY_FILE_RANGE) + + PyDoc_STRVAR(os_copy_file_range__doc__, +@@ -8882,5 +8882,5 @@ exit: + } + +-#endif /* defined(HAVE_COPY_FILE_RANGE) */ ++#endif /* defined(_Py_HAVE_COPY_FILE_RANGE) */ + + #if ((defined(HAVE_SPLICE) && !defined(_AIX))) +@@ -11460,5 +11460,5 @@ exit: + } + +-#if defined(HAVE_MEMFD_CREATE) ++#if defined(_Py_HAVE_MEMFD_CREATE) + + PyDoc_STRVAR(os_memfd_create__doc__, +@@ -11546,5 +11546,5 @@ exit: + } + +-#endif /* defined(HAVE_MEMFD_CREATE) */ ++#endif /* defined(_Py_HAVE_MEMFD_CREATE) */ + + #if (defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC)) +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -13019,5 +13019,10 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + #endif /* HAVE_PWRITEV */ + +-#ifdef HAVE_COPY_FILE_RANGE ++#if defined(HAVE_COPY_FILE_RANGE) || \ ++ (defined(__linux__) && defined(__NR_copy_file_range)) ++# define _Py_HAVE_COPY_FILE_RANGE ++#endif ++ ++#ifdef _Py_HAVE_COPY_FILE_RANGE + /*[clinic input] + +@@ -13071,5 +13076,11 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + do { + Py_BEGIN_ALLOW_THREADS ++#ifdef HAVE_COPY_FILE_RANGE + ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags); ++#else ++ /* Largefile support makes off_t 64-bit, as the kernel expects. */ ++ ret = syscall(__NR_copy_file_range, src, p_offset_src, dst, p_offset_dst, ++ count, flags); ++#endif + Py_END_ALLOW_THREADS + } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); +@@ -13081,5 +13092,5 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + return PyLong_FromSsize_t(ret); + } +-#endif /* HAVE_COPY_FILE_RANGE*/ ++#endif /* _Py_HAVE_COPY_FILE_RANGE */ + + #if (defined(HAVE_SPLICE) && !defined(_AIX)) +@@ -15783,5 +15794,10 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) + } + +-#ifdef HAVE_MEMFD_CREATE ++#if defined(HAVE_MEMFD_CREATE) || \ ++ (defined(__linux__) && defined(__NR_memfd_create) && defined(MFD_CLOEXEC)) ++# define _Py_HAVE_MEMFD_CREATE ++#endif ++ ++#ifdef _Py_HAVE_MEMFD_CREATE + /*[clinic input] + os.memfd_create +@@ -15799,5 +15815,9 @@ os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) + const char *bytes = PyBytes_AS_STRING(name); + Py_BEGIN_ALLOW_THREADS ++#ifdef HAVE_MEMFD_CREATE + fd = memfd_create(bytes, flags); ++#else ++ fd = syscall(__NR_memfd_create, bytes, flags); ++#endif + Py_END_ALLOW_THREADS + if (fd == -1) { +@@ -18408,5 +18428,5 @@ all_ins(PyObject *m) + if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1; + #endif +-#ifdef HAVE_MEMFD_CREATE ++#ifdef _Py_HAVE_MEMFD_CREATE + if (PyModule_AddIntMacro(m, MFD_CLOEXEC)) return -1; + if (PyModule_AddIntMacro(m, MFD_ALLOW_SEALING)) return -1; +@@ -18456,5 +18476,5 @@ all_ins(PyObject *m) + if (PyModule_AddIntMacro(m, MFD_HUGE_16GB)) return -1; + #endif +-#endif /* HAVE_MEMFD_CREATE */ ++#endif /* _Py_HAVE_MEMFD_CREATE */ + + #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) +@@ -18712,5 +18732,5 @@ static const struct have_function { + #endif + +-#ifdef HAVE_MEMFD_CREATE ++#ifdef _Py_HAVE_MEMFD_CREATE + { "HAVE_MEMFD_CREATE", NULL }, + #endif diff --git a/pythonbuild/disttests/__init__.py b/pythonbuild/disttests/__init__.py index 98c8c6540..e64c87615 100644 --- a/pythonbuild/disttests/__init__.py +++ b/pythonbuild/disttests/__init__.py @@ -379,18 +379,12 @@ def test_os_getrandom(self): @unittest.skipUnless( "-linux-gnu" in os.environ["TARGET_TRIPLE"], - "memfd_create weak linking is enabled for Linux GNU targets", + "the memfd_create syscall fallback is enabled for Linux GNU targets", ) def test_os_memfd_create(self): - import ctypes import errno - libc = ctypes.CDLL(None) - libc_has_memfd_create = hasattr(libc, "memfd_create") - self.assertEqual(hasattr(os, "memfd_create"), libc_has_memfd_create) - - if not libc_has_memfd_create: - return + self.assertTrue(hasattr(os, "memfd_create")) try: fd = os.memfd_create("python-build-standalone-test") @@ -408,18 +402,12 @@ def test_os_memfd_create(self): @unittest.skipUnless( "-linux-gnu" in os.environ["TARGET_TRIPLE"], - "copy_file_range weak linking is enabled for Linux GNU targets", + "the copy_file_range syscall fallback is enabled for Linux GNU targets", ) def test_os_copy_file_range(self): - import ctypes import errno - libc = ctypes.CDLL(None) - libc_has_copy_file_range = hasattr(libc, "copy_file_range") - self.assertEqual(hasattr(os, "copy_file_range"), libc_has_copy_file_range) - - if not libc_has_copy_file_range: - return + self.assertTrue(hasattr(os, "copy_file_range")) data = b"copy_file_range" with tempfile.TemporaryFile() as src, tempfile.TemporaryFile() as dst: