diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index efdf18480..932ea6d01 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -331,6 +331,14 @@ if [[ -n "${LINUX_UAPI_INCLUDE_ARCH:-}" ]]; then fi fi +# glibc only grew copy_file_range() and memfd_create() in 2.27 and CPython +# compiles os.copy_file_range() and os.memfd_create() out when the libc it is +# built against lacks them, which loses them for good in builds targeting an +# older glibc. Call the raw syscalls when the wrappers are missing so both +# functions keep working on any kernel that implements them. +# Backport of https://github.com/python/cpython/pull/155520. +patch -p1 -i "${ROOT}/patch-posixmodule-syscall-fallback.patch" + # Most bits look at CFLAGS. But setup.py only looks at CPPFLAGS. # So we need to set both. CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include -I${TOOLS_PATH}/deps/include/ncursesw" @@ -606,40 +614,6 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_explicit_bzero=no" fi -# The modern UAPI overlay provides the memfd constants, but some glibc sysroots -# predate the memfd_create() wrapper. Force the configure check on and weak-link -# the wrapper so the function is exposed only when runtime glibc provides it. -# This workaround is specific to glibc builds; the UAPI overlay itself can also -# be used with other Linux libcs. -if [[ -n "${LINUX_UAPI_INCLUDE_ARCH:-}" && "${TARGET_TRIPLE}" == *-linux-gnu* ]]; then - if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - patch -p1 -i "${ROOT}/patch-posixmodule-memfd-create-weak.patch" - else - patch -p1 -i "${ROOT}/patch-posixmodule-memfd-create-weak-3.13.patch" - fi - - # Python 3.10 checks for memfd_create with a custom compile test instead - # of the cached ac_cv_func_memfd_create check used by newer versions. - if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then - patch -p1 -i "${ROOT}/patch-configure-memfd-create-3.10.patch" - fi - - CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_memfd_create=yes" -fi - -# Some glibc sysroots predate the copy_file_range() wrapper. Force the -# configure check on and weak-link the wrapper so the function is exposed only -# when runtime glibc provides it. -if [[ "${TARGET_TRIPLE}" == *-linux-gnu* ]]; then - if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - patch -p1 -i "${ROOT}/patch-posixmodule-copy-file-range-weak.patch" - else - patch -p1 -i "${ROOT}/patch-posixmodule-copy-file-range-weak-3.13.patch" - fi - - CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_copy_file_range=yes" -fi - # Define the base PGO profiling task, which we'll extend below with ignores export PROFILE_TASK='-m test --pgo' diff --git a/cpython-unix/patch-configure-memfd-create-3.10.patch b/cpython-unix/patch-configure-memfd-create-3.10.patch deleted file mode 100644 index 0b27ad583..000000000 --- a/cpython-unix/patch-configure-memfd-create-3.10.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/configure.ac b/configure.ac ---- a/configure.ac -+++ b/configure.ac -@@ -3822,6 +3822,10 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[void *x=_dyl - ]) - - AC_MSG_CHECKING(for memfd_create) -+if test "$ac_cv_func_memfd_create" = yes; then -+ AC_DEFINE(HAVE_MEMFD_CREATE, 1, Define if you have the 'memfd_create' function.) -+ AC_MSG_RESULT(yes) -+else - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef HAVE_SYS_MMAN_H - #include -@@ -3834,6 +3838,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) - ]) -+fi - - AC_MSG_CHECKING(for eventfd) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch b/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch deleted file mode 100644 index d1e7e5e3f..000000000 --- a/cpython-unix/patch-posixmodule-copy-file-range-weak-3.13.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -10431,4 +10431,9 @@ - #ifdef HAVE_COPY_FILE_RANGE -+/* Weak references. */ -+__attribute__((weak)) -+ssize_t copy_file_range(int src, off_t *offset_src, int dst, -+ off_t *offset_dst, size_t count, unsigned int flags); -+ - /*[clinic input] - - os.copy_file_range -@@ -15737,5 +15742,17 @@ posixmodule_exec(PyObject *m) - #endif - -+#ifdef HAVE_COPY_FILE_RANGE -+ if (copy_file_range == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_DelItemString(dict, "copy_file_range") < 0) { -+ return -1; -+ } -+ } -+#endif -+ - /* Initialize environ dictionary */ - PyObject *v = convertenviron(); - Py_XINCREF(v); diff --git a/cpython-unix/patch-posixmodule-copy-file-range-weak.patch b/cpython-unix/patch-posixmodule-copy-file-range-weak.patch deleted file mode 100644 index 903e0f39b..000000000 --- a/cpython-unix/patch-posixmodule-copy-file-range-weak.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -12488,4 +12488,9 @@ - #ifdef HAVE_COPY_FILE_RANGE -+/* Weak references. */ -+__attribute__((weak)) -+ssize_t copy_file_range(int src, off_t *offset_src, int dst, -+ off_t *offset_dst, size_t count, unsigned int flags); -+ - /*[clinic input] - - os.copy_file_range -@@ -18147,5 +18152,17 @@ posixmodule_exec(PyObject *m) - #endif - -+#ifdef HAVE_COPY_FILE_RANGE -+ if (copy_file_range == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_PopString(dict, "copy_file_range", NULL) < 0) { -+ return -1; -+ } -+ } -+#endif -+ - /* Initialize environ dictionary */ - if (PyModule_Add(m, "environ", convertenviron()) != 0) { - return -1; diff --git a/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch b/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch deleted file mode 100644 index 7e115b101..000000000 --- a/cpython-unix/patch-posixmodule-memfd-create-weak-3.13.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -528,6 +528,12 @@ extern char *ctermid_r(char *); - # include - #endif - -+#ifdef HAVE_MEMFD_CREATE -+/* Weak references. */ -+__attribute__((weak)) -+int memfd_create(const char *name, unsigned int flags); -+#endif -+ - /* eventfd() */ - #ifdef HAVE_SYS_EVENTFD_H - # include -@@ -15718,6 +15724,18 @@ posixmodule_exec(PyObject *m) - { - _posixstate *state = get_posix_state(m); - -+#ifdef HAVE_MEMFD_CREATE -+ if (memfd_create == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_DelItemString(dict, "memfd_create") < 0) { -+ return -1; -+ } -+ } -+#endif -+ - #if defined(HAVE_PWRITEV) - if (HAVE_PWRITEV_RUNTIME) {} else { - PyObject* dct = PyModule_GetDict(m); diff --git a/cpython-unix/patch-posixmodule-memfd-create-weak.patch b/cpython-unix/patch-posixmodule-memfd-create-weak.patch deleted file mode 100644 index 229f14ea9..000000000 --- a/cpython-unix/patch-posixmodule-memfd-create-weak.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -192,6 +192,12 @@ - # include // memfd_create(), MFD_CLOEXEC - #endif - -+#ifdef HAVE_MEMFD_CREATE -+/* Weak references. */ -+__attribute__((weak)) -+int memfd_create(const char *name, unsigned int flags); -+#endif -+ - #ifdef HAVE_SYS_EVENTFD_H - # include // eventfd() - #endif -@@ -18128,6 +18134,18 @@ posixmodule_exec(PyObject *m) - { - _posixstate *state = get_posix_state(m); - -+#ifdef HAVE_MEMFD_CREATE -+ if (memfd_create == NULL) { -+ PyObject *dict = PyModule_GetDict(m); -+ if (dict == NULL) { -+ return -1; -+ } -+ if (PyDict_PopString(dict, "memfd_create", NULL) < 0) { -+ return -1; -+ } -+ } -+#endif -+ - #if defined(HAVE_PWRITEV) - if (HAVE_PWRITEV_RUNTIME) {} else { - PyObject* dct = PyModule_GetDict(m); diff --git a/cpython-unix/patch-posixmodule-syscall-fallback.patch b/cpython-unix/patch-posixmodule-syscall-fallback.patch new file mode 100644 index 000000000..9174ac617 --- /dev/null +++ b/cpython-unix/patch-posixmodule-syscall-fallback.patch @@ -0,0 +1,108 @@ +diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h +--- a/Modules/clinic/posixmodule.c.h ++++ b/Modules/clinic/posixmodule.c.h +@@ -8766,5 +8766,5 @@ exit: + #endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */ + +-#if defined(HAVE_COPY_FILE_RANGE) ++#if defined(_Py_HAVE_COPY_FILE_RANGE) + + PyDoc_STRVAR(os_copy_file_range__doc__, +@@ -8882,5 +8882,5 @@ exit: + } + +-#endif /* defined(HAVE_COPY_FILE_RANGE) */ ++#endif /* defined(_Py_HAVE_COPY_FILE_RANGE) */ + + #if ((defined(HAVE_SPLICE) && !defined(_AIX))) +@@ -11460,5 +11460,5 @@ exit: + } + +-#if defined(HAVE_MEMFD_CREATE) ++#if defined(_Py_HAVE_MEMFD_CREATE) + + PyDoc_STRVAR(os_memfd_create__doc__, +@@ -11546,5 +11546,5 @@ exit: + } + +-#endif /* defined(HAVE_MEMFD_CREATE) */ ++#endif /* defined(_Py_HAVE_MEMFD_CREATE) */ + + #if (defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC)) +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +--- a/Modules/posixmodule.c ++++ b/Modules/posixmodule.c +@@ -13019,5 +13019,10 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + #endif /* HAVE_PWRITEV */ + +-#ifdef HAVE_COPY_FILE_RANGE ++#if defined(HAVE_COPY_FILE_RANGE) || \ ++ (defined(__linux__) && defined(__NR_copy_file_range)) ++# define _Py_HAVE_COPY_FILE_RANGE ++#endif ++ ++#ifdef _Py_HAVE_COPY_FILE_RANGE + /*[clinic input] + +@@ -13071,5 +13076,11 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + do { + Py_BEGIN_ALLOW_THREADS ++#ifdef HAVE_COPY_FILE_RANGE + ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags); ++#else ++ /* Largefile support makes off_t 64-bit, as the kernel expects. */ ++ ret = syscall(__NR_copy_file_range, src, p_offset_src, dst, p_offset_dst, ++ count, flags); ++#endif + Py_END_ALLOW_THREADS + } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); +@@ -13081,5 +13092,5 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + return PyLong_FromSsize_t(ret); + } +-#endif /* HAVE_COPY_FILE_RANGE*/ ++#endif /* _Py_HAVE_COPY_FILE_RANGE */ + + #if (defined(HAVE_SPLICE) && !defined(_AIX)) +@@ -15783,5 +15794,10 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) + } + +-#ifdef HAVE_MEMFD_CREATE ++#if defined(HAVE_MEMFD_CREATE) || \ ++ (defined(__linux__) && defined(__NR_memfd_create) && defined(MFD_CLOEXEC)) ++# define _Py_HAVE_MEMFD_CREATE ++#endif ++ ++#ifdef _Py_HAVE_MEMFD_CREATE + /*[clinic input] + os.memfd_create +@@ -15799,5 +15815,9 @@ os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) + const char *bytes = PyBytes_AS_STRING(name); + Py_BEGIN_ALLOW_THREADS ++#ifdef HAVE_MEMFD_CREATE + fd = memfd_create(bytes, flags); ++#else ++ fd = syscall(__NR_memfd_create, bytes, flags); ++#endif + Py_END_ALLOW_THREADS + if (fd == -1) { +@@ -18408,5 +18428,5 @@ all_ins(PyObject *m) + if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1; + #endif +-#ifdef HAVE_MEMFD_CREATE ++#ifdef _Py_HAVE_MEMFD_CREATE + if (PyModule_AddIntMacro(m, MFD_CLOEXEC)) return -1; + if (PyModule_AddIntMacro(m, MFD_ALLOW_SEALING)) return -1; +@@ -18456,5 +18476,5 @@ all_ins(PyObject *m) + if (PyModule_AddIntMacro(m, MFD_HUGE_16GB)) return -1; + #endif +-#endif /* HAVE_MEMFD_CREATE */ ++#endif /* _Py_HAVE_MEMFD_CREATE */ + + #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) +@@ -18712,5 +18732,5 @@ static const struct have_function { + #endif + +-#ifdef HAVE_MEMFD_CREATE ++#ifdef _Py_HAVE_MEMFD_CREATE + { "HAVE_MEMFD_CREATE", NULL }, + #endif diff --git a/pythonbuild/disttests/__init__.py b/pythonbuild/disttests/__init__.py index 98c8c6540..e64c87615 100644 --- a/pythonbuild/disttests/__init__.py +++ b/pythonbuild/disttests/__init__.py @@ -379,18 +379,12 @@ def test_os_getrandom(self): @unittest.skipUnless( "-linux-gnu" in os.environ["TARGET_TRIPLE"], - "memfd_create weak linking is enabled for Linux GNU targets", + "the memfd_create syscall fallback is enabled for Linux GNU targets", ) def test_os_memfd_create(self): - import ctypes import errno - libc = ctypes.CDLL(None) - libc_has_memfd_create = hasattr(libc, "memfd_create") - self.assertEqual(hasattr(os, "memfd_create"), libc_has_memfd_create) - - if not libc_has_memfd_create: - return + self.assertTrue(hasattr(os, "memfd_create")) try: fd = os.memfd_create("python-build-standalone-test") @@ -408,18 +402,12 @@ def test_os_memfd_create(self): @unittest.skipUnless( "-linux-gnu" in os.environ["TARGET_TRIPLE"], - "copy_file_range weak linking is enabled for Linux GNU targets", + "the copy_file_range syscall fallback is enabled for Linux GNU targets", ) def test_os_copy_file_range(self): - import ctypes import errno - libc = ctypes.CDLL(None) - libc_has_copy_file_range = hasattr(libc, "copy_file_range") - self.assertEqual(hasattr(os, "copy_file_range"), libc_has_copy_file_range) - - if not libc_has_copy_file_range: - return + self.assertTrue(hasattr(os, "copy_file_range")) data = b"copy_file_range" with tempfile.TemporaryFile() as src, tempfile.TemporaryFile() as dst: