From 566bb9f76049759f577b9ad60a6d5831ea1026c1 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 9 Jul 2026 09:52:38 +0200 Subject: [PATCH] gh-153400: Resolve newer libc/syscall wrappers at runtime on Linux os functions like copy_file_range() are gated on a configure-time AC_CHECK_FUNCS probe and compiled out when the build libc lacks the symbol. A redistributable built against an old glibc (the python-build-standalone builds target glibc 2.17) therefore never exposes them, even when run on a newer glibc or a capable kernel. Add Modules/posixshims.h, which on Linux always exposes _Py_(), resolves the libc symbol once at load time via dlsym(RTLD_DEFAULT) from a constructor, and falls back to the raw syscall when the running libc lacks the wrapper. Resolving in a constructor keeps dlsym(), which is not async-signal-safe, out of signal handlers and the fork()/exec() window. Off Linux the classic build-time HAVE_* direct calls are kept. Limit the shims to wrappers newer than the glibc 2.17 baseline, since anything at or below it is always present in the build libc: copy_file_range (glibc 2.27) memfd_create (glibc 2.27) pidfd_open (glibc 2.36) pidfd_getfd (glibc 2.36) pidfd_open() and pidfd_getfd() previously issued raw syscalls unconditionally; they now prefer the glibc wrapper when present. --- ...-07-09-10-19-10.gh-issue-153400.ELIvWE.rst | 4 + Modules/clinic/posixmodule.c.h | 18 +-- Modules/posixmodule.c | 26 ++-- Modules/posixshims.h | 141 ++++++++++++++++++ Tools/c-analyzer/cpython/ignored.tsv | 4 + 5 files changed, 171 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-09-10-19-10.gh-issue-153400.ELIvWE.rst create mode 100644 Modules/posixshims.h diff --git a/Misc/NEWS.d/next/Library/2026-07-09-10-19-10.gh-issue-153400.ELIvWE.rst b/Misc/NEWS.d/next/Library/2026-07-09-10-19-10.gh-issue-153400.ELIvWE.rst new file mode 100644 index 000000000000000..6f88863e13c64cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-10-19-10.gh-issue-153400.ELIvWE.rst @@ -0,0 +1,4 @@ +On Linux, :func:`os.copy_file_range`, :func:`os.memfd_create`, +:func:`os.pidfd_open` and :func:`os.pidfd_getfd` are now resolved at runtime, so +they stay available on interpreters built against an older glibc when the +running system provides them. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 00a4f6009186af9..53479a23b4dc1e6 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6330,7 +6330,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) +#if defined(_Py_HAVE_PIDFD_OPEN) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -6405,9 +6405,9 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#endif /* defined(_Py_HAVE_PIDFD_OPEN) */ -#if (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) +#if defined(_Py_HAVE_PIDFD_GETFD) PyDoc_STRVAR(os_pidfd_getfd__doc__, "pidfd_getfd($module, /, pidfd, targetfd, *, flags=0)\n" @@ -6492,7 +6492,7 @@ os_pidfd_getfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#endif /* defined(_Py_HAVE_PIDFD_GETFD) */ #if defined(HAVE_SETNS) @@ -8852,7 +8852,7 @@ os_pwritev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */ -#if defined(HAVE_COPY_FILE_RANGE) +#if defined(_Py_HAVE_COPY_FILE_RANGE) PyDoc_STRVAR(os_copy_file_range__doc__, "copy_file_range($module, /, src, dst, count, offset_src=None,\n" @@ -8968,7 +8968,7 @@ os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } -#endif /* defined(HAVE_COPY_FILE_RANGE) */ +#endif /* defined(_Py_HAVE_COPY_FILE_RANGE) */ #if ((defined(HAVE_SPLICE) && !defined(_AIX))) @@ -11546,7 +11546,7 @@ os_urandom(PyObject *module, PyObject *arg) return return_value; } -#if defined(HAVE_MEMFD_CREATE) +#if defined(_Py_HAVE_MEMFD_CREATE) PyDoc_STRVAR(os_memfd_create__doc__, "memfd_create($module, /, name, flags=MFD_CLOEXEC)\n" @@ -11632,7 +11632,7 @@ os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -#endif /* defined(HAVE_MEMFD_CREATE) */ +#endif /* defined(_Py_HAVE_MEMFD_CREATE) */ #if (defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC)) @@ -13733,4 +13733,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=250ea2e34fdd133f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7e8f950dc6f69067 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 57db175336702e2..b6218feffaa49cc 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -46,6 +46,8 @@ #include // ctermid() #include // system() +#include "posixshims.h" // _Py_copy_file_range() + #ifdef HAVE_UNISTD_H # include // symlink() #endif @@ -10773,8 +10775,7 @@ os_wait_impl(PyObject *module) // This system call always crashes on older Android versions. -#if defined(__linux__) && defined(__NR_pidfd_open) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 31) +#ifdef _Py_HAVE_PIDFD_OPEN /*[clinic input] os.pidfd_open pid: pid_t @@ -10790,7 +10791,7 @@ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) /*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { - int fd = syscall(__NR_pidfd_open, pid, flags); + int fd = _Py_pidfd_open(pid, flags); if (fd < 0) { return posix_error(); } @@ -10799,8 +10800,7 @@ os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) #endif -#if defined(__linux__) && defined(__NR_pidfd_getfd) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 31) +#ifdef _Py_HAVE_PIDFD_GETFD /*[clinic input] os.pidfd_getfd pidfd: int @@ -10819,7 +10819,7 @@ os_pidfd_getfd_impl(PyObject *module, int pidfd, int targetfd, unsigned int flags) /*[clinic end generated code: output=e1a1415a13c7137f input=ef6417fb10deb1cc]*/ { - int fd = syscall(__NR_pidfd_getfd, pidfd, targetfd, flags); + int fd = _Py_pidfd_getfd(pidfd, targetfd, flags); if (fd < 0) { return posix_error(); } @@ -13018,7 +13018,7 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } #endif /* HAVE_PWRITEV */ -#ifdef HAVE_COPY_FILE_RANGE +#ifdef _Py_HAVE_COPY_FILE_RANGE /*[clinic input] os.copy_file_range @@ -13070,7 +13070,7 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, do { Py_BEGIN_ALLOW_THREADS - ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags); + ret = _Py_copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags); Py_END_ALLOW_THREADS } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); @@ -13080,7 +13080,7 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, return PyLong_FromSsize_t(ret); } -#endif /* HAVE_COPY_FILE_RANGE*/ +#endif /* _Py_HAVE_COPY_FILE_RANGE */ #if (defined(HAVE_SPLICE) && !defined(_AIX)) /*[clinic input] @@ -15774,7 +15774,7 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) return PyBytesWriter_Finish(writer); } -#ifdef HAVE_MEMFD_CREATE +#ifdef _Py_HAVE_MEMFD_CREATE /*[clinic input] os.memfd_create @@ -15790,7 +15790,7 @@ os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) int fd; const char *bytes = PyBytes_AS_STRING(name); Py_BEGIN_ALLOW_THREADS - fd = memfd_create(bytes, flags); + fd = _Py_memfd_create(bytes, flags); Py_END_ALLOW_THREADS if (fd == -1) { return PyErr_SetFromErrno(PyExc_OSError); @@ -18393,7 +18393,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, GRND_RANDOM)) return -1; if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1; #endif -#ifdef HAVE_MEMFD_CREATE +#ifdef _Py_HAVE_MEMFD_CREATE if (PyModule_AddIntMacro(m, MFD_CLOEXEC)) return -1; if (PyModule_AddIntMacro(m, MFD_ALLOW_SEALING)) return -1; #ifdef MFD_HUGETLB @@ -18441,7 +18441,7 @@ all_ins(PyObject *m) #ifdef MFD_HUGE_16GB if (PyModule_AddIntMacro(m, MFD_HUGE_16GB)) return -1; #endif -#endif /* HAVE_MEMFD_CREATE */ +#endif /* _Py_HAVE_MEMFD_CREATE */ #if defined(HAVE_EVENTFD) && defined(EFD_CLOEXEC) if (PyModule_AddIntMacro(m, EFD_CLOEXEC)) return -1; diff --git a/Modules/posixshims.h b/Modules/posixshims.h new file mode 100644 index 000000000000000..b9369217eacec14 --- /dev/null +++ b/Modules/posixshims.h @@ -0,0 +1,141 @@ +/* posixshims.h: resolve newer libc/syscall wrappers at runtime, not build time. + + CPython gates functions like copy_file_range() on a configure-time + AC_CHECK_FUNCS probe, compiling them out when the build libc lacks the symbol. + That permanently disables them in a redistributable built against an old glibc + even when it later runs on a newer one. Instead, on Linux we always expose + _Py_(), resolve the libc symbol once at load time via dlsym(RTLD_DEFAULT) + from a constructor, and fall back to the raw syscall. A constructor (vs. lazy + resolution) keeps dlsym(), which is not async-signal-safe, out of signal + handlers and the fork()/exec() window, and lets the cache skip synchronization. + + Each _Py_() is declared explicitly; only the shared body is a macro. + Off Linux, each function keeps its classic build-time HAVE_* direct call. */ + +#ifndef Py_POSIXSHIMS_H +#define Py_POSIXSHIMS_H + +/* Needs ELF constructors, dlsym() and Linux syscall numbers. */ +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) \ + && defined(HAVE_DLFCN_H) && defined(HAVE_SYS_SYSCALL_H) +# define _Py_HAVE_POSIX_SHIMS +#endif + +#ifdef _Py_HAVE_POSIX_SHIMS + +#include // dlsym(), RTLD_DEFAULT +#include // __NR_* syscall numbers +#include // off_t, ssize_t +#include // syscall() + +/* The raw-syscall fallback needs a syscall number. If a stale build header + predates one, default it to an invalid number so the shim still builds and the + dlsym'd libc symbol stays usable; only the syscall fallback is lost, returning + -1/ENOSYS at runtime. above is the sole source of these + numbers in this translation unit, so nothing redefines them afterwards. */ +#ifndef __NR_copy_file_range +# define __NR_copy_file_range (-1) +#endif +#ifndef __NR_memfd_create +# define __NR_memfd_create (-1) +#endif +#ifndef __NR_pidfd_open +# define __NR_pidfd_open (-1) +#endif +#ifndef __NR_pidfd_getfd +# define __NR_pidfd_getfd (-1) +#endif + +/* A per-function cache plus a constructor that resolves the libc symbol once at + load time. The __asm__ barrier after dlsym() is load-bearing under LTO: + without it the compiler may tail-call dlsym(), breaking glibc's + return-address caller lookup and crashing at startup (glibc BZ #34156). */ +#define _Py_SHIM_RESOLVER(func) \ + static void *_Py_shim_##func; \ + __attribute__((constructor)) \ + static void _Py_shim_##func##_resolve(void) { \ + void *p = dlsym(RTLD_DEFAULT, #func); \ + __asm__ volatile("" ::: "memory"); \ + _Py_shim_##func = p; \ + } + +/* Body of an explicitly declared _Py_() wrapper: call the resolved libc + symbol if present, else fall back to syscall(__NR_, ...). On a kernel + too old for the syscall that returns -1/ENOSYS, which posixmodule.c reports as + an ordinary OSError. The cast is __typeof__ of _Py_ (the wrapper + itself), not of , which may be undeclared at build time. */ +#define _Py_SHIM_SYSCALL(func, ...) \ + if (_Py_shim_##func != NULL) { \ + return ((__typeof__(&_Py_##func)) _Py_shim_##func)(__VA_ARGS__); \ + } \ + return syscall(__NR_##func, __VA_ARGS__) + +#endif /* _Py_HAVE_POSIX_SHIMS */ + + +/* ---- copy_file_range() (glibc 2.27) ------------------------------------- */ + +#if defined(_Py_HAVE_POSIX_SHIMS) +# define _Py_HAVE_COPY_FILE_RANGE +_Py_SHIM_RESOLVER(copy_file_range) +/* off_t is 64-bit (_FILE_OFFSET_BITS=64), matching the kernel's loff_t. */ +static inline ssize_t +_Py_copy_file_range(int fd_in, off_t *off_in, int fd_out, off_t *off_out, + size_t len, unsigned int flags) +{ + _Py_SHIM_SYSCALL(copy_file_range, + fd_in, off_in, fd_out, off_out, len, flags); +} + +#elif defined(HAVE_COPY_FILE_RANGE) +# define _Py_HAVE_COPY_FILE_RANGE +# define _Py_copy_file_range copy_file_range +#endif + + +/* ---- memfd_create() (glibc 2.27) ---------------------------------------- */ + +#if defined(_Py_HAVE_POSIX_SHIMS) +# define _Py_HAVE_MEMFD_CREATE +_Py_SHIM_RESOLVER(memfd_create) +static inline int +_Py_memfd_create(const char *name, unsigned int flags) +{ + _Py_SHIM_SYSCALL(memfd_create, name, flags); +} + +#elif defined(HAVE_MEMFD_CREATE) +# define _Py_HAVE_MEMFD_CREATE +# define _Py_memfd_create memfd_create +#endif + + +/* ---- pidfd_open(), pidfd_getfd() (glibc 2.36) --------------------------- * + + Previously unconditional raw syscalls; the shim now prefers glibc's wrapper. + Linux-only from the start (no non-Linux branch); the Android API floor from + the original guards is preserved. */ + +#if defined(_Py_HAVE_POSIX_SHIMS) \ + && !(defined(__ANDROID__) && __ANDROID_API__ < 31) +# define _Py_HAVE_PIDFD_OPEN +_Py_SHIM_RESOLVER(pidfd_open) +static inline int +_Py_pidfd_open(pid_t pid, unsigned int flags) +{ + _Py_SHIM_SYSCALL(pidfd_open, pid, flags); +} +#endif + +#if defined(_Py_HAVE_POSIX_SHIMS) \ + && !(defined(__ANDROID__) && __ANDROID_API__ < 31) +# define _Py_HAVE_PIDFD_GETFD +_Py_SHIM_RESOLVER(pidfd_getfd) +static inline int +_Py_pidfd_getfd(int pidfd, int targetfd, unsigned int flags) +{ + _Py_SHIM_SYSCALL(pidfd_getfd, pidfd, targetfd, flags); +} +#endif + +#endif /* !Py_POSIXSHIMS_H */ diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 6e18593ad698570..79e8ce5f48c4721 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -789,3 +789,7 @@ Objects/dictobject.c - PyFrozenDict_Type - ## False positives Python/specialize.c - _Py_InitCleanup - Python/pystate.c - _no_tstate_sentinel - +Modules/posixshims.h - _Py_shim_copy_file_range - +Modules/posixshims.h - _Py_shim_memfd_create - +Modules/posixshims.h - _Py_shim_pidfd_open - +Modules/posixshims.h - _Py_shim_pidfd_getfd -