From c6886e6a2c4a6e8ff0cbc87dead625fbb1ef9d97 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 20:15:17 +0200 Subject: [PATCH 1/8] use argon2 from openssl >= 3.2, drop argon2-cffi, fixes #7963 - src/borg/crypto/low_level.pyx: implement `argon2_hash` using OpenSSL's `EVP_KDF` API for ARGON2 (requires OpenSSL >= 3.2.0). - src/borg/crypto/key.py: switch to the native `argon2_hash` implementation, removing `argon2-cffi` dependency. - setup.py: require OpenSSL >= 3.2.0 for the crypto extension to ensure ARGON2 KDF support is available. - pyproject.toml: drop `argon2-cffi` dependency. - docs: update installation requirements and security documentation to reflect the transition to OpenSSL for Argon2. --- .github/workflows/ci.yml | 5 +-- docs/installation.rst | 4 +- docs/internals/security.rst | 2 +- pyproject.toml | 1 - scripts/msys2-install-deps | 2 +- setup.py | 4 +- src/borg/crypto/key.py | 13 +----- src/borg/crypto/low_level.pyi | 3 ++ src/borg/crypto/low_level.pyx | 78 +++++++++++++++++++++++++++++++++++ 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c2a5fde95..290243d7cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -564,8 +564,7 @@ jobs: pkgman install -y openssl3 pkgman install -y rust_bin pkgman install -y python3.11 - pkgman install -y cffi - pkgman install -y lz4_devel openssl3_devel libffi_devel + pkgman install -y lz4_devel openssl3_devel # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade @@ -651,7 +650,7 @@ jobs: - name: Build python venv run: | - # building cffi / argon2-cffi in the venv fails, so we try to use the system packages + # building native extensions in the venv fails, so we try to use the system packages python -m venv --system-site-packages env . env/bin/activate # python -m pip install --upgrade pip diff --git a/docs/installation.rst b/docs/installation.rst index 62c326098f..8b9de4d1fc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -162,10 +162,10 @@ following dependencies first. For the libraries you will also need their development header files (sometimes in a separate `-dev` or `-devel` package). * `Python 3`_ >= 3.11.0 -* OpenSSL_ >= 1.1.1 (LibreSSL will not work) +* OpenSSL_ >= 3.2.0 (LibreSSL will not work) * libacl_ (which depends on libattr_) * liblz4_ >= 1.7.0 (r129) -* libffi (required for argon2-cffi-bindings) + * pkg-config (cli tool) - Borg uses this to discover header and library locations automatically. Alternatively, you can also point to them via some environment variables, see setup.py. diff --git a/docs/internals/security.rst b/docs/internals/security.rst index f5bb5ef08f..61a4cdd80d 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -241,7 +241,7 @@ on widely used libraries providing them: primitives implemented in libcrypto. - SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used. - HMAC and a constant-time comparison from Python's hmac_ standard library module are used. -- argon2 is used via argon2-cffi. +- argon2 is used from OpenSSL (>= 3.2). .. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle .. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack diff --git a/pyproject.toml b/pyproject.toml index c9e0ec8cc2..5ce1d5c76f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ dependencies = [ "packaging", "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0. "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently. - "argon2-cffi", "shtab>=1.8.0", "backports-zstd; python_version < '3.14'", # for python < 3.14. "jsonargparse>=4.47.0", diff --git a/scripts/msys2-install-deps b/scripts/msys2-install-deps index ae4dc4bd85..0cbf864242 100644 --- a/scripts/msys2-install-deps +++ b/scripts/msys2-install-deps @@ -1,6 +1,6 @@ #!/bin/bash -pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin} +pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,openssl,rclone,python-msgpack,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin} if [ "$1" = "development" ]; then pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist} diff --git a/setup.py b/setup.py index 19f1c92789..5e9d547bf1 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s ) if is_win32: - crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="") + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=3.2.0", lib_subdir="") elif is_openbsd: # Use OpenSSL (not LibreSSL) because we need AES-OCB via the EVP API. Link # it statically to avoid conflicting with shared libcrypto from the base @@ -154,7 +154,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")], ) else: - crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1") + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0") crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index d0750bdc0e..c6e9034002 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -12,7 +12,6 @@ logger = create_logger() from blake3 import blake3 -import argon2.low_level from ..constants import * # NOQA from ..helpers import StableDict @@ -483,17 +482,9 @@ def argon2( parallelism = 1 # 8 is the smallest value that avoids the "Memory cost is too small" exception memory_cost = 8 - type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID} - key = argon2.low_level.hash_secret_raw( - secret=passphrase.encode("utf-8"), - hash_len=output_len_in_bytes, - salt=salt, - time_cost=time_cost, - memory_cost=memory_cost, - parallelism=parallelism, - type=type_map[type], + return low_level.argon2_hash( + passphrase.encode("utf-8"), salt, time_cost, memory_cost, parallelism, output_len_in_bytes, type ) - return key def decrypt_key_file_argon2(self, encrypted_key, passphrase): key = self.argon2( diff --git a/src/borg/crypto/low_level.pyi b/src/borg/crypto/low_level.pyi index 9d5e4c0566..07922a228f 100644 --- a/src/borg/crypto/low_level.pyi +++ b/src/borg/crypto/low_level.pyi @@ -12,6 +12,9 @@ def long_to_bytes(x: int) -> bytes: ... def hmac_sha256(key: bytes, data: bytes) -> bytes: ... def blake2b_256(key: bytes, data: bytes) -> bytes: ... def blake2b_128(data: bytes) -> bytes: ... +def argon2_hash( + secret: bytes, salt: bytes, time_cost: int, memory_cost: int, parallelism: int, hash_len: int, type: str +) -> bytes: ... # Exception classes class CryptoError(Exception): diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 149ea1d738..ca824a9482 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -88,6 +88,34 @@ cdef extern from "openssl/evp.h": int EVP_CTRL_AEAD_SET_TAG int EVP_CTRL_AEAD_SET_IVLEN +cdef extern from "openssl/kdf.h": + ctypedef struct EVP_KDF: + pass + ctypedef struct EVP_KDF_CTX: + pass + EVP_KDF *EVP_KDF_fetch(void *ctx, const char *algorithm, const char *properties) + void EVP_KDF_free(EVP_KDF *kdf) + EVP_KDF_CTX *EVP_KDF_CTX_new(EVP_KDF *kdf) + void EVP_KDF_CTX_free(EVP_KDF_CTX *ctx) + +cdef extern from "openssl/params.h": + ctypedef struct OSSL_PARAM: + pass + OSSL_PARAM OSSL_PARAM_construct_uint32(const char *key, uint32_t *buf) + OSSL_PARAM OSSL_PARAM_construct_octet_string(const char *key, void *buf, size_t bsize) + OSSL_PARAM OSSL_PARAM_construct_end() + +cdef extern from "openssl/core_names.h": + const char *OSSL_KDF_PARAM_THREADS + const char *OSSL_KDF_PARAM_ARGON2_LANES + const char *OSSL_KDF_PARAM_ITER + const char *OSSL_KDF_PARAM_ARGON2_MEMCOST + const char *OSSL_KDF_PARAM_SALT + const char *OSSL_KDF_PARAM_PASSWORD + +cdef extern from "openssl/kdf.h": + int EVP_KDF_derive(EVP_KDF_CTX *ctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[]) + import struct @@ -825,3 +853,53 @@ cdef class CSPRNG: # Swap items[i] and items[j] items[i], items[j] = items[j], items[i] + + +def argon2_hash(bytes secret, bytes salt, uint32_t time_cost, uint32_t memory_cost, + uint32_t parallelism, uint32_t hash_len, type): + cdef EVP_KDF *kdf = NULL + cdef EVP_KDF_CTX *kctx = NULL + cdef OSSL_PARAM params[8] + cdef OSSL_PARAM *p = params + cdef uint32_t threads = 1 + cdef bytes result + cdef const char *alg_name + cdef const unsigned char *secret_c = secret + cdef const unsigned char *salt_c = salt + + if type == "i" or type == b"i": + alg_name = b"ARGON2I" + elif type == "d" or type == b"d": + alg_name = b"ARGON2D" + elif type == "id" or type == b"id": + alg_name = b"ARGON2ID" + else: + raise ValueError("Invalid argon2 type") + + kdf = EVP_KDF_fetch(NULL, alg_name, NULL) + if kdf == NULL: + raise CryptoError("Argon2 KDF not found in OpenSSL (requires >= 3.2)") + + kctx = EVP_KDF_CTX_new(kdf) + if kctx == NULL: + EVP_KDF_free(kdf) + raise MemoryError("Failed to create KDF context") + + p[0] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_THREADS, &threads) + p[1] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_LANES, ¶llelism) + p[2] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ITER, &time_cost) + p[3] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_MEMCOST, &memory_cost) + p[4] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, salt_c, len(salt)) + p[5] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_PASSWORD, secret_c, len(secret)) + p[6] = OSSL_PARAM_construct_end() + + result = PyBytes_FromStringAndSize(NULL, hash_len) + + if EVP_KDF_derive(kctx, result, hash_len, params) <= 0: + EVP_KDF_CTX_free(kctx) + EVP_KDF_free(kdf) + raise CryptoError("EVP_KDF_derive failed") + + EVP_KDF_CTX_free(kctx) + EVP_KDF_free(kdf) + return result From 297344cee6749ce05876f7fe8238a3d18fef22ed Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 20:21:09 +0200 Subject: [PATCH 2/8] CI: use FreeBSD 15.0 (has OpenSSL 3.5.x) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 290243d7cb..334de2bf8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -392,11 +392,11 @@ jobs: matrix: include: - os: freebsd - version: '14.3' + version: '15.0' display_name: FreeBSD # Controls binary build and provenance attestation on tags do_binaries: true - artifact_prefix: borg-freebsd-14-x86_64-gh + artifact_prefix: borg-freebsd-15-x86_64-gh - os: netbsd version: '10.1' From 10311eeff95696f33e02d37faf755362feb47e7d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 May 2026 23:20:23 +0200 Subject: [PATCH 3/8] CI: NetBSD: build a fresh OpenSSL from source, we need >= 3.2 --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ setup.py | 9 +++++++++ 2 files changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 334de2bf8e..9b8029b108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -520,6 +520,30 @@ jobs: touch ${TMPDIR}/testfile lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***" + # NetBSD 10 has a too old OpenSSL, build a fresher one. + VERSION="3.5.6" + echo "--- Building OpenSSL ${VERSION} ---" + PREFIX="/usr/local" + JOBS=$(sysctl -n hw.ncpu) + + pushd /tmp + ftp -o "openssl-${VERSION}.tar.gz" "https://www.openssl.org/source/openssl-${VERSION}.tar.gz" + tar xzf "openssl-${VERSION}.tar.gz" + pushd "openssl-${VERSION}" + + ./Configure --prefix="${PREFIX}" --openssldir="${PREFIX}/etc/ssl" --libdir=lib \ + -Wl,-rpath,${PREFIX}/lib shared threads no-tests no-docs + export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + make -j"${JOBS}" + sudo -E make install + popd + rm -rf "/tmp/openssl-${VERSION}" "/tmp/openssl-${VERSION}.tar.gz" + popd + + "${PREFIX}/bin/openssl" version + export BORG_OPENSSL_PREFIX="${PREFIX}" + export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" + tox3 -e py311-none ;; diff --git a/setup.py b/setup.py index 5e9d547bf1..3a9e63dc6c 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ is_win32 = sys.platform.startswith("win32") is_openbsd = sys.platform.startswith("openbsd") +is_netbsd = sys.platform.startswith("netbsd") # Number of threads to use for cythonize, not used on Windows cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None @@ -153,6 +154,14 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s include_dirs=[os.path.join(openssl_prefix, "include", openssl_name)], extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")], ) + elif is_netbsd and os.environ.get("BORG_OPENSSL_PREFIX"): + # Similarly for NetBSD, if we built a custom OpenSSL, link it statically + # to avoid dynamic linker conflicts with the system OpenSSL loaded by Python. + openssl_prefix = os.environ.get("BORG_OPENSSL_PREFIX") + crypto_ext_lib = dict( + include_dirs=[os.path.join(openssl_prefix, "include")], + extra_objects=[os.path.join(openssl_prefix, "lib", "libcrypto.a")], + ) else: crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0") From 8763ec03cf6834e099452f27032f8c3b510538b0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 01:03:09 +0200 Subject: [PATCH 4/8] CI: use ubuntu-26.04 runners, update binary glibc tag to 243 Bump all Ubuntu-based GitHub workflows from ubuntu-24.04 to ubuntu-26.04 (and ubuntu-24.04-arm to ubuntu-26.04-arm). Ubuntu 26.04 ships glibc 2.43, so rename the built binaries from glibc239 to glibc243. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/backport.yml | 2 +- .github/workflows/black.yaml | 2 +- .github/workflows/canary.yml | 6 +++--- .github/workflows/ci.yml | 28 +++++++++++++-------------- .github/workflows/codeql-analysis.yml | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 8174f121d3..1aa6c8e074 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -13,7 +13,7 @@ permissions: jobs: backport: name: Backport pull request - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 # Only run when pull request is merged diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 926e5ecf16..e7e9c86a28 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -21,7 +21,7 @@ concurrency: jobs: lint: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 268ea0b031..d02175ab46 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -18,13 +18,13 @@ jobs: matrix: include: # A representative subset of environments - - os: ubuntu-24.04 + - os: ubuntu-26.04 python-version: '3.11' toxenv: py311-llfuse - - os: ubuntu-24.04 + - os: ubuntu-26.04 python-version: '3.12' toxenv: py312-pyfuse3 - - os: ubuntu-24.04 + - os: ubuntu-26.04 python-version: '3.14' toxenv: py314-mfusepy - os: macos-15 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b8029b108..352a5b3f8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ permissions: jobs: lint: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: @@ -40,7 +40,7 @@ jobs: security: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 steps: @@ -59,7 +59,7 @@ jobs: asan_ubsan: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 25 needs: [lint] @@ -141,19 +141,19 @@ jobs: ${{ fromJSON( github.event_name == 'pull_request' && '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "mypy"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "docs"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} + {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "mypy"}, + {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "docs"}, + {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} ] }' || '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-mfusepy"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-x86_64-gh"}, - {"os": "ubuntu-24.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-arm64-gh"}, + {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, + {"os": "ubuntu-26.04", "python-version": "3.13", "toxenv": "py313-mfusepy"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-x86_64-gh"}, + {"os": "ubuntu-26.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-arm64-gh"}, {"os": "macos-15", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-arm64-gh"}, {"os": "macos-15-intel", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-x86_64-gh"} ] @@ -382,7 +382,7 @@ jobs: contents: read id-token: write attestations: write - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 180 needs: [lint] continue-on-error: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e02fc6add8..7823ac5446 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ concurrency: jobs: analyze: name: Analyze - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 20 permissions: actions: read From 97daf2cf790dab638f797e619d95c23e388e704a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 01:13:13 +0200 Subject: [PATCH 5/8] CI: use Python 3.14 on Ubuntu Currently, ubuntu-26.04 only has py314. --- .github/workflows/canary.yml | 6 ------ .github/workflows/ci.yml | 21 ++++++++++----------- .github/workflows/codeql-analysis.yml | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index d02175ab46..32aee7b537 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -18,12 +18,6 @@ jobs: matrix: include: # A representative subset of environments - - os: ubuntu-26.04 - python-version: '3.11' - toxenv: py311-llfuse - - os: ubuntu-26.04 - python-version: '3.12' - toxenv: py312-pyfuse3 - os: ubuntu-26.04 python-version: '3.14' toxenv: py314-mfusepy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 352a5b3f8f..cb782e441e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.11' + python-version: '3.14' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -73,7 +73,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: '3.14' - name: Install system packages run: | @@ -141,17 +141,16 @@ jobs: ${{ fromJSON( github.event_name == 'pull_request' && '{ "include": [ - {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "mypy"}, - {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "docs"}, - {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-26.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "mypy"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "docs"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-pyfuse3"}, {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} ] }' || '{ "include": [ - {"os": "ubuntu-26.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-26.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-26.04", "python-version": "3.13", "toxenv": "py313-mfusepy"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-llfuse"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy"}, {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-x86_64-gh"}, {"os": "ubuntu-26.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc243-arm64-gh"}, {"os": "macos-15", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-arm64-gh"}, @@ -708,7 +707,7 @@ jobs: uses: codecov/codecov-action@v7 env: OS: ${{ runner.os }} - python: '3.11' + python: '3.14' with: token: ${{ secrets.CODECOV_TOKEN }} report_type: test_results @@ -720,7 +719,7 @@ jobs: uses: codecov/codecov-action@v7 env: OS: ${{ runner.os }} - python: '3.11' + python: '3.14' with: token: ${{ secrets.CODECOV_TOKEN }} report_type: coverage diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7823ac5446..08ad5537df 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -53,7 +53,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: 3.11 + python-version: 3.14 - name: Cache pip uses: actions/cache@v5 with: From 63dbd2ac9e07051493f730010c183c8215aa2dd5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 14:50:52 +0200 Subject: [PATCH 6/8] fix daemonizing: don't lose an early notify signal from the background process The foreground process installs SIGTERM/SIGHUP/SIGINT handlers (via archiver.run) before it reaches the point where it actually waits for the background (grandchild) process to notify it (via os.kill). If the background process started up and signalled fast enough, the signal was delivered to the foreground while it was still between os.fork() and the waiting code, so the globally installed handler raised at an unexpected, uncaught place. The signal then escaped daemonizing(), bubbled up through repository teardown (NotLocked) and made "borg mount" exit with rc 74. This was observed flaky in CI with coverage's sys.monitoring backend on Python 3.14 (its first-branch lazy source parse widens the window) and the pyfuse3 backend (faster grandchild startup). Fix the race by blocking the notify signals before the fork in _daemonize() and waiting for them atomically in the foreground. An early signal then stays pending and is reliably picked up by the wait. The background process restores the original signal mask so it keeps normal signal handling. Use signal.sigwait() plus a SIGALRM timer (signal.setitimer) for the wait, rather than signal.sigtimedwait(): the latter does not exist on macOS, where it would raise AttributeError in the foreground and let it die before the background migrated the lock (breaking test_migrate_lock_alive). Co-Authored-By: Claude Opus 4.8 --- src/borg/helpers/process.py | 97 ++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py index b19aabcf18..91872b9c0d 100644 --- a/src/borg/helpers/process.py +++ b/src/borg/helpers/process.py @@ -18,13 +18,31 @@ from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error +# Signals the background (grandchild) process uses to notify the foreground process that +# it has started up (or failed to start up), see daemonizing(). These are blocked across +# the fork in _daemonize() so that an early notification can not be delivered to the +# foreground before it is ready to wait for it - otherwise the globally installed signal +# handlers (see archiver.run) would raise at an unexpected, uncaught place. +DAEMONIZE_NOTIFY_SIGNALS = [ + sig for sig in (getattr(signal, name, None) for name in ("SIGTERM", "SIGHUP", "SIGINT")) if sig is not None +] +# The foreground process waits for the notify signals with signal.sigwait() (which, unlike +# signal.sigtimedwait(), is also available on macOS). To still honor the timeout, we arm a +# SIGALRM timer and wait for it as well, so it must be blocked and waited for, too. +DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + [signal.SIGALRM] + + @contextlib.contextmanager def _daemonize(): from ..platform import get_process_id old_id = get_process_id() + # Block the notify (and timeout) signals before forking, see DAEMONIZE_WAIT_SIGNALS. + old_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, DAEMONIZE_WAIT_SIGNALS) pid = os.fork() if pid: + # The foreground process keeps the notify signals blocked - daemonizing() waits + # for them explicitly via signal.sigwait(), so it can not miss an early one. exit_code = EXIT_SUCCESS try: yield old_id, None @@ -33,6 +51,8 @@ def _daemonize(): finally: logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id) os._exit(exit_code) + # Background process: restore the original signal mask, we want normal handling here. + signal.pthread_sigmask(signal.SIG_SETMASK, old_sigmask) os.setsid() pid = os.fork() if pid: @@ -81,47 +101,46 @@ def daemonizing(*, timeout=5, show_rc=False): # The original / parent process, waiting for a signal to die. logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id) exit_code = EXIT_SUCCESS - # Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case... - with ( - signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), - signal_handler("SIGHUP", raising_signal_handler(SigHup)), - signal_handler("SIGTERM", raising_signal_handler(SigTerm)), - ): + # The notify signals are blocked since before the fork in _daemonize(), so we can + # wait for them atomically here and will not miss an early signal from the background + # process (which could otherwise be delivered before we are ready to wait for it and + # then escape uncaught). See DAEMONIZE_WAIT_SIGNALS. + sighup = getattr(signal, "SIGHUP", None) + if timeout > 0: + # Arm a timer so we don't wait forever if the background never signals us. + # SIGALRM is blocked, too, so it does not run a handler but ends the sigwait(). + signal.setitimer(signal.ITIMER_REAL, timeout) try: - if timeout > 0: - time.sleep(timeout) - except SigTerm: - # Normal termination; expected from grandchild, see 'os.kill()' below - pass - except SigHup: - # Background wants to indicate a problem; see 'os.kill()' below, - # log message will come from grandchild. - exit_code = EXIT_WARNING - except KeyboardInterrupt: - # Manual termination. - logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id) - exit_code = EXIT_SIGNAL_BASE + 2 - except BaseException as e: - # Just in case... - logger.warning( - "Daemonizing: Foreground process received an exception while waiting:\n" - + "".join(traceback.format_exception(e.__class__, e, e.__traceback__)) - ) - exit_code = EXIT_WARNING - else: - logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?") - exit_code = EXIT_WARNING + sig = signal.sigwait(DAEMONIZE_WAIT_SIGNALS) finally: - # Before terminating the foreground process, honor --show-rc by logging the rc here as well. - # This is mostly a consistency fix and not very useful considering that the main action - # happens in the daemon process. - if show_rc: - from ..helpers import do_show_rc - - do_show_rc(exit_code) - # Don't call with-body, but die immediately! - # return would be sufficient, but we want to pass the exit code. - raise _ExitCodeException(exit_code) + signal.setitimer(signal.ITIMER_REAL, 0) # disarm + else: + sig = signal.SIGALRM # no waiting: behave as if the timeout had elapsed + if sig == signal.SIGALRM: + # Timeout: the background process did not signal us in time. + logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?") + exit_code = EXIT_WARNING + elif sig == signal.SIGTERM: + # Normal termination; expected from grandchild, see 'os.kill()' below. + pass + elif sighup is not None and sig == sighup: + # Background wants to indicate a problem; see 'os.kill()' below, + # log message will come from grandchild. + exit_code = EXIT_WARNING + elif sig == signal.SIGINT: + # Manual termination. + logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id) + exit_code = EXIT_SIGNAL_BASE + 2 + # Before terminating the foreground process, honor --show-rc by logging the rc here as well. + # This is mostly a consistency fix and not very useful considering that the main action + # happens in the daemon process. + if show_rc: + from ..helpers import do_show_rc + + do_show_rc(exit_code) + # Don't call with-body, but die immediately! + # return would be sufficient, but we want to pass the exit code. + raise _ExitCodeException(exit_code) # The background / grandchild process. sig_to_foreground = signal.SIGTERM From 7878629e465d7b16c787f23e33fa0bf79b85aa8e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 17:03:04 +0200 Subject: [PATCH 7/8] process.py: guard signal.SIGALRM for Windows signal.SIGALRM does not exist on Windows, so referencing it at module import time raised AttributeError, breaking the import chain (and the Windows PyInstaller build). Guard it with hasattr, matching the defensive getattr pattern already used for the notify signals. Daemonizing is not supported on Windows anyway (no os.fork), so the empty SIGALRM list has no functional effect there. Co-Authored-By: Claude Opus 4.8 --- src/borg/helpers/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py index 91872b9c0d..a66e7afbf1 100644 --- a/src/borg/helpers/process.py +++ b/src/borg/helpers/process.py @@ -29,7 +29,8 @@ # The foreground process waits for the notify signals with signal.sigwait() (which, unlike # signal.sigtimedwait(), is also available on macOS). To still honor the timeout, we arm a # SIGALRM timer and wait for it as well, so it must be blocked and waited for, too. -DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + [signal.SIGALRM] +# SIGALRM is not available on Windows (where daemonizing is not supported anyway). +DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + ([signal.SIGALRM] if hasattr(signal, "SIGALRM") else []) @contextlib.contextmanager From 7f16bc37da5c7c750d1e60278fc375f4487281c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 14 Jun 2026 18:10:41 +0200 Subject: [PATCH 8/8] conftest: fix rmtree cleanup crash on Linux (os.lchflags does not exist) The archiver fixture's rmtree onerror handler called os.lchflags(path, 0) when has_lchflags was True. But has_lchflags is also True on Linux (flags are cleared via ioctl there), where os.lchflags does not exist, raising an uncaught AttributeError and turning teardown into an ERROR. Use borg's cross-platform platform.set_flags instead. Co-Authored-By: Claude Opus 4.8 --- src/borg/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/borg/conftest.py b/src/borg/conftest.py index fb80b5bff2..82594fe46d 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -15,6 +15,7 @@ setup_logging() from borg.archiver import Archiver # noqa: E402 +from borg.platform import set_flags # noqa: E402 from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3, has_mfusepy # noqa: E402 from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported # noqa: E402 from borg.testsuite.archiver import BORG_EXES @@ -197,8 +198,10 @@ def archiver(tmp_path, set_env_variables): def maybe_clear_flags_and_retry(func, path, _exc_info): if has_lchflags: # Clear any BSD flags (e.g. UF_APPEND) that may have prevented removal, then retry once. + # Note: use borg's cross-platform set_flags, not os.lchflags - the latter does not exist + # on Linux even though has_lchflags is True there (Linux clears flags via ioctl). try: - os.lchflags(path, 0) + set_flags(path, 0) func(path) except OSError: pass