From 710db85b20e20c8b21c4b3f6777a2efe05efccd9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 20:15:17 +0200 Subject: [PATCH 1/7] use argon2 from openssl >= 3.2, drop argon2-cffi, fixes #7963 - src/borg/crypto/low_level.pyx: implement `argon2_hash` using OpenSSL's `EVP_KDF` API for ARGON2 (requires OpenSSL >= 3.2.0). - src/borg/crypto/key.py: switch to the native `argon2_hash` implementation, removing `argon2-cffi` dependency. - setup.py: require OpenSSL >= 3.2.0 for the crypto extension to ensure ARGON2 KDF support is available. - pyproject.toml: drop `argon2-cffi` dependency. - docs: update installation requirements and security documentation to reflect the transition to OpenSSL for Argon2. --- .github/workflows/ci.yml | 5 +-- docs/installation.rst | 4 +- docs/internals/security.rst | 2 +- pyproject.toml | 1 - scripts/msys2-install-deps | 2 +- setup.py | 4 +- src/borg/crypto/key.py | 13 +----- src/borg/crypto/low_level.pyi | 3 ++ src/borg/crypto/low_level.pyx | 78 +++++++++++++++++++++++++++++++++++ 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a0e784067..d57bf2038d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -568,8 +568,7 @@ jobs: pkgman install -y openssl3 pkgman install -y rust_bin pkgman install -y python3.11 - pkgman install -y cffi - pkgman install -y lz4_devel openssl3_devel libffi_devel + pkgman install -y lz4_devel openssl3_devel # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade @@ -655,7 +654,7 @@ jobs: - name: Build python venv run: | - # building cffi / argon2-cffi in the venv fails, so we try to use the system packages + # building native extensions in the venv fails, so we try to use the system packages python -m venv --system-site-packages env . env/bin/activate # python -m pip install --upgrade pip diff --git a/docs/installation.rst b/docs/installation.rst index 62c326098f..8b9de4d1fc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -162,10 +162,10 @@ following dependencies first. For the libraries you will also need their development header files (sometimes in a separate `-dev` or `-devel` package). * `Python 3`_ >= 3.11.0 -* OpenSSL_ >= 1.1.1 (LibreSSL will not work) +* OpenSSL_ >= 3.2.0 (LibreSSL will not work) * libacl_ (which depends on libattr_) * liblz4_ >= 1.7.0 (r129) -* libffi (required for argon2-cffi-bindings) + * pkg-config (cli tool) - Borg uses this to discover header and library locations automatically. Alternatively, you can also point to them via some environment variables, see setup.py. diff --git a/docs/internals/security.rst b/docs/internals/security.rst index 1534988aea..727376ce34 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -268,7 +268,7 @@ on widely used libraries providing them: - BLAKE3 is used via the blake3_ package (new repos, ``--id-hash blake3``). - BLAKE2b from Python's hashlib_ is only used to read legacy (borg < 2.0) repos. - HMAC and a constant-time comparison from Python's hmac_ standard library module are used. -- argon2 is used via argon2-cffi. +- argon2 is used from OpenSSL (>= 3.2). .. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle .. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack diff --git a/pyproject.toml b/pyproject.toml index f5965623a8..a3943ac88f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ dependencies = [ "packaging", "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0. "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently. - "argon2-cffi", "shtab>=1.8.0", "backports-zstd; python_version < '3.14'", # for python < 3.14. "jsonargparse>=4.47.0", diff --git a/scripts/msys2-install-deps b/scripts/msys2-install-deps index ae4dc4bd85..0cbf864242 100644 --- a/scripts/msys2-install-deps +++ b/scripts/msys2-install-deps @@ -1,6 +1,6 @@ #!/bin/bash -pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin} +pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,openssl,rclone,python-msgpack,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin} if [ "$1" = "development" ]; then pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist} diff --git a/setup.py b/setup.py index 1ad76e8048..04d34a00bd 100644 --- a/setup.py +++ b/setup.py @@ -144,7 +144,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s ) if is_win32: - crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="") + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=3.2.0", lib_subdir="") elif is_openbsd: # Use OpenSSL (not LibreSSL) because we need AES-OCB via the EVP API. Link # it statically to avoid conflicting with shared libcrypto from the base @@ -156,7 +156,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")], ) else: - crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1") + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0") crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 60d66d168d..8c342e3251 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -12,7 +12,6 @@ logger = create_logger() from blake3 import blake3 -import argon2.low_level from ..constants import * # NOQA from ..helpers import StableDict @@ -508,17 +507,9 @@ def argon2( parallelism = 1 # 8 is the smallest value that avoids the "Memory cost is too small" exception memory_cost = 8 - type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID} - key = argon2.low_level.hash_secret_raw( - secret=passphrase.encode("utf-8"), - hash_len=output_len_in_bytes, - salt=salt, - time_cost=time_cost, - memory_cost=memory_cost, - parallelism=parallelism, - type=type_map[type], + return low_level.argon2_hash( + passphrase.encode("utf-8"), salt, time_cost, memory_cost, parallelism, output_len_in_bytes, type ) - return key def decrypt_key_file_argon2(self, encrypted_key, passphrase): key = self.argon2( diff --git a/src/borg/crypto/low_level.pyi b/src/borg/crypto/low_level.pyi index 9d5e4c0566..07922a228f 100644 --- a/src/borg/crypto/low_level.pyi +++ b/src/borg/crypto/low_level.pyi @@ -12,6 +12,9 @@ def long_to_bytes(x: int) -> bytes: ... def hmac_sha256(key: bytes, data: bytes) -> bytes: ... def blake2b_256(key: bytes, data: bytes) -> bytes: ... def blake2b_128(data: bytes) -> bytes: ... +def argon2_hash( + secret: bytes, salt: bytes, time_cost: int, memory_cost: int, parallelism: int, hash_len: int, type: str +) -> bytes: ... # Exception classes class CryptoError(Exception): diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 149ea1d738..ca824a9482 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -88,6 +88,34 @@ cdef extern from "openssl/evp.h": int EVP_CTRL_AEAD_SET_TAG int EVP_CTRL_AEAD_SET_IVLEN +cdef extern from "openssl/kdf.h": + ctypedef struct EVP_KDF: + pass + ctypedef struct EVP_KDF_CTX: + pass + EVP_KDF *EVP_KDF_fetch(void *ctx, const char *algorithm, const char *properties) + void EVP_KDF_free(EVP_KDF *kdf) + EVP_KDF_CTX *EVP_KDF_CTX_new(EVP_KDF *kdf) + void EVP_KDF_CTX_free(EVP_KDF_CTX *ctx) + +cdef extern from "openssl/params.h": + ctypedef struct OSSL_PARAM: + pass + OSSL_PARAM OSSL_PARAM_construct_uint32(const char *key, uint32_t *buf) + OSSL_PARAM OSSL_PARAM_construct_octet_string(const char *key, void *buf, size_t bsize) + OSSL_PARAM OSSL_PARAM_construct_end() + +cdef extern from "openssl/core_names.h": + const char *OSSL_KDF_PARAM_THREADS + const char *OSSL_KDF_PARAM_ARGON2_LANES + const char *OSSL_KDF_PARAM_ITER + const char *OSSL_KDF_PARAM_ARGON2_MEMCOST + const char *OSSL_KDF_PARAM_SALT + const char *OSSL_KDF_PARAM_PASSWORD + +cdef extern from "openssl/kdf.h": + int EVP_KDF_derive(EVP_KDF_CTX *ctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[]) + import struct @@ -825,3 +853,53 @@ cdef class CSPRNG: # Swap items[i] and items[j] items[i], items[j] = items[j], items[i] + + +def argon2_hash(bytes secret, bytes salt, uint32_t time_cost, uint32_t memory_cost, + uint32_t parallelism, uint32_t hash_len, type): + cdef EVP_KDF *kdf = NULL + cdef EVP_KDF_CTX *kctx = NULL + cdef OSSL_PARAM params[8] + cdef OSSL_PARAM *p = params + cdef uint32_t threads = 1 + cdef bytes result + cdef const char *alg_name + cdef const unsigned char *secret_c = secret + cdef const unsigned char *salt_c = salt + + if type == "i" or type == b"i": + alg_name = b"ARGON2I" + elif type == "d" or type == b"d": + alg_name = b"ARGON2D" + elif type == "id" or type == b"id": + alg_name = b"ARGON2ID" + else: + raise ValueError("Invalid argon2 type") + + kdf = EVP_KDF_fetch(NULL, alg_name, NULL) + if kdf == NULL: + raise CryptoError("Argon2 KDF not found in OpenSSL (requires >= 3.2)") + + kctx = EVP_KDF_CTX_new(kdf) + if kctx == NULL: + EVP_KDF_free(kdf) + raise MemoryError("Failed to create KDF context") + + p[0] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_THREADS, &threads) + p[1] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_LANES, ¶llelism) + p[2] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ITER, &time_cost) + p[3] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_MEMCOST, &memory_cost) + p[4] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, salt_c, len(salt)) + p[5] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_PASSWORD, secret_c, len(secret)) + p[6] = OSSL_PARAM_construct_end() + + result = PyBytes_FromStringAndSize(NULL, hash_len) + + if EVP_KDF_derive(kctx, result, hash_len, params) <= 0: + EVP_KDF_CTX_free(kctx) + EVP_KDF_free(kdf) + raise CryptoError("EVP_KDF_derive failed") + + EVP_KDF_CTX_free(kctx) + EVP_KDF_free(kdf) + return result From e9ed2eaaf342497c3b5a043dae103e4ac24f5c03 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 20:21:09 +0200 Subject: [PATCH 2/7] CI: use FreeBSD 15.0 (has OpenSSL 3.5.x) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d57bf2038d..084419b008 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -396,11 +396,11 @@ jobs: matrix: include: - os: freebsd - version: '14.3' + version: '15.0' display_name: FreeBSD # Controls binary build and provenance attestation on tags do_binaries: true - artifact_prefix: borg-freebsd-14-x86_64-gh + artifact_prefix: borg-freebsd-15-x86_64-gh - os: netbsd version: '10.1' From dd7be9ff527c666907e85db726e896cbdeaa6e57 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 23:20:23 +0200 Subject: [PATCH 3/7] CI: NetBSD: build a fresh OpenSSL from source, we need >= 3.2 --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ setup.py | 9 +++++++++ 2 files changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084419b008..ac775471a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -524,6 +524,30 @@ jobs: touch ${TMPDIR}/testfile lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***" + # NetBSD 10 has a too old OpenSSL, build a fresher one. + VERSION="3.5.6" + echo "--- Building OpenSSL ${VERSION} ---" + PREFIX="/usr/local" + JOBS=$(sysctl -n hw.ncpu) + + pushd /tmp + ftp -o "openssl-${VERSION}.tar.gz" "https://www.openssl.org/source/openssl-${VERSION}.tar.gz" + tar xzf "openssl-${VERSION}.tar.gz" + pushd "openssl-${VERSION}" + + ./Configure --prefix="${PREFIX}" --openssldir="${PREFIX}/etc/ssl" --libdir=lib \ + -Wl,-rpath,${PREFIX}/lib shared threads no-tests no-docs + export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + make -j"${JOBS}" + sudo -E make install + popd + rm -rf "/tmp/openssl-${VERSION}" "/tmp/openssl-${VERSION}.tar.gz" + popd + + "${PREFIX}/bin/openssl" version + export BORG_OPENSSL_PREFIX="${PREFIX}" + export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" + tox3 -e py311-none ;; diff --git a/setup.py b/setup.py index 04d34a00bd..f586ec83ce 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ is_win32 = sys.platform.startswith("win32") is_openbsd = sys.platform.startswith("openbsd") +is_netbsd = sys.platform.startswith("netbsd") # Number of threads to use for cythonize, not used on Windows cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None @@ -155,6 +156,14 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s include_dirs=[os.path.join(openssl_prefix, "include", openssl_name)], extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")], ) + elif is_netbsd and os.environ.get("BORG_OPENSSL_PREFIX"): + # Similarly for NetBSD, if we built a custom OpenSSL, link it statically + # to avoid dynamic linker conflicts with the system OpenSSL loaded by Python. + openssl_prefix = os.environ.get("BORG_OPENSSL_PREFIX") + crypto_ext_lib = dict( + include_dirs=[os.path.join(openssl_prefix, "include")], + extra_objects=[os.path.join(openssl_prefix, "lib", "libcrypto.a")], + ) else: crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0") From f8d1ea54b658883587c24a6528dd968e4e086d06 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 01:03:09 +0200 Subject: [PATCH 4/7] CI: use ubuntu-26.04 runners, update binary glibc tag to 243 Bump all Ubuntu-based GitHub workflows from ubuntu-24.04 to ubuntu-26.04 (and ubuntu-24.04-arm to ubuntu-26.04-arm). Ubuntu 26.04 ships glibc 2.43, so rename the built binaries from glibc239 to glibc243. Use Python 3.14, there are no other Pythons yet on the runner. Co-Authored-By: Claude Opus 4.8 CI: use Python 3.14 on Ubuntu Currently, ubuntu-26.04 only has py314. --- .github/workflows/backport.yml | 2 +- .github/workflows/black.yaml | 2 +- .github/workflows/canary.yml | 8 +----- .github/workflows/ci.yml | 35 +++++++++++++-------------- .github/workflows/codeql-analysis.yml | 4 +-- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index deab27e150..820a5b7286 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -13,7 +13,7 @@ permissions: jobs: backport: name: Backport pull request - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 # Only run when pull request is merged diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 33ef1f60f1..b3f42014f4 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -21,7 +21,7 @@ concurrency: jobs: lint: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 12b7b8e218..d2c620ed06 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -18,13 +18,7 @@ jobs: matrix: include: # A representative subset of environments - - os: ubuntu-24.04 - python-version: '3.11' - toxenv: py311-llfuse - - os: ubuntu-24.04 - python-version: '3.12' - toxenv: py312-pyfuse3 - - os: ubuntu-24.04 + - os: ubuntu-26.04 python-version: '3.14' toxenv: py314-mfusepy - os: macos-15 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac775471a2..76c7f53d89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ permissions: jobs: lint: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: @@ -40,7 +40,7 @@ jobs: security: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: @@ -48,7 +48,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.11' + python-version: '3.14' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -59,7 +59,7 @@ jobs: asan_ubsan: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 25 needs: [lint] @@ -73,7 +73,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: '3.14' - name: Install system packages run: | @@ -141,19 +141,18 @@ jobs: ${{ fromJSON( github.event_name == 'pull_request' && '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "mypy"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "docs"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3", "store_cache": "1"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "mypy"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "docs"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "store_cache": "1"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} ] }' || '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-mfusepy", "store_cache": "1"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-x86_64-gh"}, - {"os": "ubuntu-24.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-arm64-gh"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy", "store_cache": "1"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-x86_64-gh"}, + {"os": "ubuntu-26.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-arm64-gh"}, {"os": "macos-15", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-arm64-gh"}, {"os": "macos-15-intel", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-x86_64-gh"} ] @@ -386,7 +385,7 @@ jobs: contents: read id-token: write attestations: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 180 needs: [lint] continue-on-error: true @@ -712,7 +711,7 @@ jobs: uses: codecov/codecov-action@v7 env: OS: ${{ runner.os }} - python: '3.11' + python: '3.14' with: token: ${{ secrets.CODECOV_TOKEN }} report_type: test_results @@ -724,7 +723,7 @@ jobs: uses: codecov/codecov-action@v7 env: OS: ${{ runner.os }} - python: '3.11' + python: '3.14' with: token: ${{ secrets.CODECOV_TOKEN }} report_type: coverage diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6a42a6dbc5..bb3716173b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ concurrency: jobs: analyze: name: Analyze - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 20 permissions: actions: read @@ -53,7 +53,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: 3.11 + python-version: 3.14 - name: Cache pip uses: actions/cache@v6 with: From d78c0f7ee1a665988266d7d0f3f40f3ff0d6d183 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 14:50:52 +0200 Subject: [PATCH 5/7] fix daemonizing: don't lose an early notify signal from the background process The foreground process installs SIGTERM/SIGHUP/SIGINT handlers (via archiver.run) before it reaches the point where it actually waits for the background (grandchild) process to notify it (via os.kill). If the background process started up and signalled fast enough, the signal was delivered to the foreground while it was still between os.fork() and the waiting code, so the globally installed handler raised at an unexpected, uncaught place. The signal then escaped daemonizing(), bubbled up through repository teardown (NotLocked) and made "borg mount" exit with rc 74. This was observed flaky in CI with coverage's sys.monitoring backend on Python 3.14 (its first-branch lazy source parse widens the window) and the pyfuse3 backend (faster grandchild startup). Fix the race by blocking the notify signals before the fork in _daemonize() and waiting for them atomically in the foreground. An early signal then stays pending and is reliably picked up by the wait. The background process restores the original signal mask so it keeps normal signal handling. Use signal.sigwait() plus a SIGALRM timer (signal.setitimer) for the wait, rather than signal.sigtimedwait(): the latter does not exist on macOS, where it would raise AttributeError in the foreground and let it die before the background migrated the lock (breaking test_migrate_lock_alive). Co-Authored-By: Claude Opus 4.8 --- src/borg/helpers/process.py | 97 ++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py index b19aabcf18..91872b9c0d 100644 --- a/src/borg/helpers/process.py +++ b/src/borg/helpers/process.py @@ -18,13 +18,31 @@ from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error +# Signals the background (grandchild) process uses to notify the foreground process that +# it has started up (or failed to start up), see daemonizing(). These are blocked across +# the fork in _daemonize() so that an early notification can not be delivered to the +# foreground before it is ready to wait for it - otherwise the globally installed signal +# handlers (see archiver.run) would raise at an unexpected, uncaught place. +DAEMONIZE_NOTIFY_SIGNALS = [ + sig for sig in (getattr(signal, name, None) for name in ("SIGTERM", "SIGHUP", "SIGINT")) if sig is not None +] +# The foreground process waits for the notify signals with signal.sigwait() (which, unlike +# signal.sigtimedwait(), is also available on macOS). To still honor the timeout, we arm a +# SIGALRM timer and wait for it as well, so it must be blocked and waited for, too. +DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + [signal.SIGALRM] + + @contextlib.contextmanager def _daemonize(): from ..platform import get_process_id old_id = get_process_id() + # Block the notify (and timeout) signals before forking, see DAEMONIZE_WAIT_SIGNALS. + old_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, DAEMONIZE_WAIT_SIGNALS) pid = os.fork() if pid: + # The foreground process keeps the notify signals blocked - daemonizing() waits + # for them explicitly via signal.sigwait(), so it can not miss an early one. exit_code = EXIT_SUCCESS try: yield old_id, None @@ -33,6 +51,8 @@ def _daemonize(): finally: logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id) os._exit(exit_code) + # Background process: restore the original signal mask, we want normal handling here. + signal.pthread_sigmask(signal.SIG_SETMASK, old_sigmask) os.setsid() pid = os.fork() if pid: @@ -81,47 +101,46 @@ def daemonizing(*, timeout=5, show_rc=False): # The original / parent process, waiting for a signal to die. logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id) exit_code = EXIT_SUCCESS - # Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case... - with ( - signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), - signal_handler("SIGHUP", raising_signal_handler(SigHup)), - signal_handler("SIGTERM", raising_signal_handler(SigTerm)), - ): + # The notify signals are blocked since before the fork in _daemonize(), so we can + # wait for them atomically here and will not miss an early signal from the background + # process (which could otherwise be delivered before we are ready to wait for it and + # then escape uncaught). See DAEMONIZE_WAIT_SIGNALS. + sighup = getattr(signal, "SIGHUP", None) + if timeout > 0: + # Arm a timer so we don't wait forever if the background never signals us. + # SIGALRM is blocked, too, so it does not run a handler but ends the sigwait(). + signal.setitimer(signal.ITIMER_REAL, timeout) try: - if timeout > 0: - time.sleep(timeout) - except SigTerm: - # Normal termination; expected from grandchild, see 'os.kill()' below - pass - except SigHup: - # Background wants to indicate a problem; see 'os.kill()' below, - # log message will come from grandchild. - exit_code = EXIT_WARNING - except KeyboardInterrupt: - # Manual termination. - logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id) - exit_code = EXIT_SIGNAL_BASE + 2 - except BaseException as e: - # Just in case... - logger.warning( - "Daemonizing: Foreground process received an exception while waiting:\n" - + "".join(traceback.format_exception(e.__class__, e, e.__traceback__)) - ) - exit_code = EXIT_WARNING - else: - logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?") - exit_code = EXIT_WARNING + sig = signal.sigwait(DAEMONIZE_WAIT_SIGNALS) finally: - # Before terminating the foreground process, honor --show-rc by logging the rc here as well. - # This is mostly a consistency fix and not very useful considering that the main action - # happens in the daemon process. - if show_rc: - from ..helpers import do_show_rc - - do_show_rc(exit_code) - # Don't call with-body, but die immediately! - # return would be sufficient, but we want to pass the exit code. - raise _ExitCodeException(exit_code) + signal.setitimer(signal.ITIMER_REAL, 0) # disarm + else: + sig = signal.SIGALRM # no waiting: behave as if the timeout had elapsed + if sig == signal.SIGALRM: + # Timeout: the background process did not signal us in time. + logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?") + exit_code = EXIT_WARNING + elif sig == signal.SIGTERM: + # Normal termination; expected from grandchild, see 'os.kill()' below. + pass + elif sighup is not None and sig == sighup: + # Background wants to indicate a problem; see 'os.kill()' below, + # log message will come from grandchild. + exit_code = EXIT_WARNING + elif sig == signal.SIGINT: + # Manual termination. + logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id) + exit_code = EXIT_SIGNAL_BASE + 2 + # Before terminating the foreground process, honor --show-rc by logging the rc here as well. + # This is mostly a consistency fix and not very useful considering that the main action + # happens in the daemon process. + if show_rc: + from ..helpers import do_show_rc + + do_show_rc(exit_code) + # Don't call with-body, but die immediately! + # return would be sufficient, but we want to pass the exit code. + raise _ExitCodeException(exit_code) # The background / grandchild process. sig_to_foreground = signal.SIGTERM From 55d78e11eb40610c6e436f769598f2db6d56e511 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 17:03:04 +0200 Subject: [PATCH 6/7] process.py: guard signal.SIGALRM for Windows signal.SIGALRM does not exist on Windows, so referencing it at module import time raised AttributeError, breaking the import chain (and the Windows PyInstaller build). Guard it with hasattr, matching the defensive getattr pattern already used for the notify signals. Daemonizing is not supported on Windows anyway (no os.fork), so the empty SIGALRM list has no functional effect there. Co-Authored-By: Claude Opus 4.8 --- src/borg/helpers/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py index 91872b9c0d..a66e7afbf1 100644 --- a/src/borg/helpers/process.py +++ b/src/borg/helpers/process.py @@ -29,7 +29,8 @@ # The foreground process waits for the notify signals with signal.sigwait() (which, unlike # signal.sigtimedwait(), is also available on macOS). To still honor the timeout, we arm a # SIGALRM timer and wait for it as well, so it must be blocked and waited for, too. -DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + [signal.SIGALRM] +# SIGALRM is not available on Windows (where daemonizing is not supported anyway). +DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + ([signal.SIGALRM] if hasattr(signal, "SIGALRM") else []) @contextlib.contextmanager From 00697337ce314e0b6fc1526fb07fb9227002c78a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 18:10:41 +0200 Subject: [PATCH 7/7] conftest: fix rmtree cleanup crash on Linux (os.lchflags does not exist) The archiver fixture's rmtree onerror handler called os.lchflags(path, 0) when has_lchflags was True. But has_lchflags is also True on Linux (flags are cleared via ioctl there), where os.lchflags does not exist, raising an uncaught AttributeError and turning teardown into an ERROR. Use borg's cross-platform platform.set_flags instead. Co-Authored-By: Claude Opus 4.8 --- src/borg/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/borg/conftest.py b/src/borg/conftest.py index c76cfcf626..981d288049 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -15,6 +15,7 @@ setup_logging() from borg.archiver import Archiver # noqa: E402 +from borg.platform import set_flags # noqa: E402 from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3, has_mfusepy # noqa: E402 from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported # noqa: E402 from borg.testsuite.archiver import BORG_EXES @@ -196,8 +197,10 @@ def archiver(tmp_path, set_env_variables): def maybe_clear_flags_and_retry(func, path, _exc_info): if has_lchflags: # Clear any BSD flags (e.g. UF_APPEND) that may have prevented removal, then retry once. + # Note: use borg's cross-platform set_flags, not os.lchflags - the latter does not exist + # on Linux even though has_lchflags is True there (Linux clears flags via ioctl). try: - os.lchflags(path, 0) + set_flags(path, 0) func(path) except OSError: pass