From 8ed5cc2fc2f56ec8e89c364775e70ddeefe9d614 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 5 Aug 2026 14:48:09 +0200 Subject: [PATCH 01/13] chunkers: add AVX-512 / VAES scan kernels fastcdc, buzhash64: AVX-512 variants of the 8-lane candidate test (one 512-bit vector, vptestnmq fusing the AND and the ==0 test into a mask register), runtime-detected on x86-64 above the AVX2 kernels. BORG_FASTCDC_NO_AVX512 / BORG_BUZHASH64_NO_AVX512 cap dispatch at AVX2 for benchmarking the kernels against each other. rabin-aes/goldilocks-aes/toeplitz-aes: VAES/AVX-512 variant of the x86-64 hardware path, kind "vaes": groups of 32 positions encrypted as 8 zmm vectors of 4 AES blocks each - 4x fewer AES instructions, register-resident round keys (no per-round reloads), 8 independent chains to hide the vaesenc latency, vpexpandq digest placement and a masked vptestnmq cut test without extracts. BORG_PHTE_NO_VAES caps the AES chunkers at the 128-bit AES-NI path. The VAES path needs GCC >= 11 / clang >= 14 for __builtin_cpu_supports("vaes"); older compilers keep the AES-NI path. All kernels return bit-identical cut points; the existing kernel-identity tests cover the new paths where the CPU has them. Co-Authored-By: Claude Fable 5 --- docs/changes.rst | 6 +- src/borg/chunkers/buzhash64.pyx | 2 +- src/borg/chunkers/buzhash64_impl.c | 64 +++++++- src/borg/chunkers/buzhash64_impl.h | 2 +- src/borg/chunkers/fastcdc.pyx | 10 +- src/borg/chunkers/fastcdc_impl.c | 71 ++++++++- src/borg/chunkers/fastcdc_impl.h | 2 +- src/borg/chunkers/goldilocks_aes_impl.h | 2 +- src/borg/chunkers/phte_core.h | 35 +++++ src/borg/chunkers/phte_scan.h | 137 +++++++++++++++++- src/borg/chunkers/rabin_aes_impl.h | 2 +- src/borg/chunkers/toeplitz_aes_impl.h | 2 +- src/borg/testsuite/chunkers/buzhash64_test.py | 4 +- src/borg/testsuite/chunkers/fastcdc_test.py | 6 +- .../testsuite/chunkers/phte_chunkers_test.py | 2 +- 15 files changed, 316 insertions(+), 31 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index a76c3ccb4b..7c6a90d785 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -186,13 +186,15 @@ New features: - Chunkers: - fastcdc is the new and faster default chunker, #9957 - - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034: + - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043: - NEON on aarch64 (e.g. Apple Silicon) - - AVX2 on x86-64 (Intel / AMD) + - AVX-512 / AVX2 on x86-64 (Intel / AMD) - blocked scalar elsewhere - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987 + - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64 + (4 AES blocks per instruction), #10043 - zero-copy fill and lazy buffer compaction optimizations - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice replacement for `borg mount` in some use cases, #9942 diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx index 610e80f83f..4e7b75a40a 100644 --- a/src/borg/chunkers/buzhash64.pyx +++ b/src/borg/chunkers/buzhash64.pyx @@ -155,7 +155,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'.""" + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'.""" return (bz64_kernel_name(self.force_scalar)).decode("ascii") cdef object process(self): diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index ca43d60a91..32936e8c3a 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -22,14 +22,15 @@ * also removes one rotate per byte from the sequential path. * * Kernel dispatch mirrors fastcdc_impl.c: NEON on aarch64 (baseline there), - * AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere, sequential - * when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit + * AVX-512 or AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere, + * sequential when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit * mask): sequential 1150, blocked scalar 2340, NEON 2260 MB/s - NEON and * blocked are a tie here (throughput and energy, within noise), because the * XOR/AND/compare test also runs well on the scalar ALUs; the symmetric * dispatch is kept for consistency across the SIMD chunker kernels. * All kernels return bit-identical results. */ +#include #include #include "buzhash64_impl.h" @@ -209,8 +210,56 @@ bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, return j; } +/* --- AVX-512 (x86-64, runtime detected) ---------------------------------- */ + +#define BZ_KIND_512 "avx512" + +/* The same 8-lane test as the AVX2 kernel in one 512-bit vector: vptestnmq + * fuses the AND and the ==0 test per lane into a mask register. Only the + * test narrows; the scalar prefix work (bz64_block_prefix) is shared. */ +__attribute__((target("avx512f"))) static size_t +bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot, + const uint8_t *pr, const uint8_t *pa, + size_t n, uint64_t *sum_io, uint64_t mask) +{ + uint64_t sum = *sum_io; + uint64_t s[8]; + size_t j = 0; + /* _mm512_set_epi64 takes arguments high lane first: lane k = ROTL(mask, 7-k) */ + __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1), + (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3), + (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), + (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); + + while (j + 8 <= n && (sum & mask) != 0) { + bz64_block_prefix(T, Trot, pr + j, pa + j, s); + uint64_t c = BZ_ROTL(sum, 8); + __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c), + _mm512_loadu_si512((const void *)s)); + if (_mm512_testn_epi64_mask(H, M)) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ s[7]; + j += 8; + } + if (j < n) + j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); + *sum_io = sum; + return j; +} + static int bz64_simd_available(void) { + /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2 (for benchmarking the + * kernels against each other); like the detection itself it is read + * once per process, so it must be set before the first chunker use. */ + if (__builtin_cpu_supports("avx512f")) { + const char *e = getenv("BORG_BUZHASH64_NO_AVX512"); + if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0) + return 2; + } return __builtin_cpu_supports("avx2"); } @@ -233,7 +282,8 @@ static int bz64_simd_available(void) /* --- dispatch ----------------------------------------------------------- */ -/* resolved once; a racy double-init writes the same value, so it is benign */ +/* resolved once (0 = blocked, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined); + * a racy double-init writes the same value, so it is benign */ static int bz64_use_simd = -1; size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, @@ -244,6 +294,10 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask); if (bz64_use_simd < 0) bz64_use_simd = bz64_simd_available(); +#ifdef BZ_KIND_512 + if (bz64_use_simd == 2) + return bz64_scan_simd512(table, table_rot, p_rem, p_add, n, sum, mask); +#endif if (bz64_use_simd) return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask); return bz64_scan_blocked(table, table_rot, p_rem, p_add, n, sum, mask); @@ -255,5 +309,9 @@ const char *bz64_kernel_name(int force_scalar) return "scalar"; if (bz64_use_simd < 0) bz64_use_simd = bz64_simd_available(); +#ifdef BZ_KIND_512 + if (bz64_use_simd == 2) + return BZ_KIND_512; +#endif return bz64_use_simd ? BZ_KIND : "blocked"; } diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h index cd805341c7..2a9220cc73 100644 --- a/src/borg/chunkers/buzhash64_impl.h +++ b/src/borg/chunkers/buzhash64_impl.h @@ -27,7 +27,7 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, size_t n, uint64_t *sum, uint64_t mask, int force_scalar); -/* Name of the kernel bz64_scan would use: "neon", "avx2", "blocked" or "scalar". */ +/* Name of the kernel bz64_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */ const char *bz64_kernel_name(int force_scalar); #endif diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx index 7172fe94fc..535f189896 100644 --- a/src/borg/chunkers/fastcdc.pyx +++ b/src/borg/chunkers/fastcdc.pyx @@ -33,9 +33,11 @@ cdef extern from "fastcdc_impl.h": # class only provides the keyed Gear table and the _scan() hook calling the C kernel. # # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on -# aarch64, AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to the plain -# sequential Gear loop: every byte position is tested, cuts and hash state are exactly the -# same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop (for tests). +# aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to +# the plain sequential Gear loop: every byte position is tested, cuts and hash state are +# exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop +# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 (for benchmarking, must be +# set before the first chunker use). @cython.boundscheck(False) @@ -85,7 +87,7 @@ cdef class ChunkerFastCDC(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'.""" + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'.""" return (fc_kernel_name(self.force_scalar)).decode("ascii") cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept: diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index f7c184caa3..4ddfbb1d82 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -21,9 +21,11 @@ * which an exact sequential recheck of the block resolves. The per-lane * masks (mask << 7 .. mask << 0) are constants of the scan. * - * All kernels (sequential, blocked scalar, NEON, AVX2) return bit-identical - * cut positions and fp values; the chunker's golden tests depend on this. */ + * All kernels (sequential, blocked scalar, NEON, AVX2, AVX-512) return + * bit-identical cut positions and fp values; the chunker's golden tests + * depend on this. */ +#include #include #include "fastcdc_impl.h" @@ -214,8 +216,62 @@ fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, return -1; } +/* --- AVX-512 (x86-64, runtime detected) --------------------------------- */ + +#define FC_KIND_512 "avx512" + +/* The same 8-lane candidate test as the AVX2 kernel, in one 512-bit vector: + * vptestnmq fuses the AND and the ==0 test per lane into a mask register, + * so the whole test is broadcast + load + add + testn + branch. Only the + * test narrows; the scalar prefix work (fc_block_prefix) is shared. */ +__attribute__((target("avx512f"))) static int64_t +fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) +{ + uint64_t fp = *fp_io; + uint64_t s[8]; + /* _mm512_set_epi64 takes arguments high lane first: lane j = mask << (7-j) */ + __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1), + (long long)(mask << 2), (long long)(mask << 3), + (long long)(mask << 4), (long long)(mask << 5), + (long long)(mask << 6), (long long)(mask << 7)); + size_t i = 0; + + for (; i + 8 <= n; i += 8) { + fc_block_prefix(gear, p + i, s); + uint64_t c = fp << 8; + __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c), + _mm512_loadu_si512((const void *)s)); + if (_mm512_testn_epi64_mask(H, M)) { + int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } else { + fp = c + s[7]; + } + } + if (i < n) { + int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask); + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } + *fp_io = fp; + return -1; +} + static int fc_simd_available(void) { + /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2 (for benchmarking the + * kernels against each other); like the detection itself it is read + * once per process, so it must be set before the first chunker use. */ + if (__builtin_cpu_supports("avx512f")) { + const char *e = getenv("BORG_FASTCDC_NO_AVX512"); + if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0) + return 2; + } return __builtin_cpu_supports("avx2"); } @@ -236,7 +292,8 @@ static int fc_simd_available(void) /* --- dispatch ----------------------------------------------------------- */ -/* resolved once; a racy double-init writes the same value, so it is benign */ +/* resolved once (0 = blocked, 1 = FC_KIND, 2 = FC_KIND_512 where defined); + * a racy double-init writes the same value, so it is benign */ static int fc_use_simd = -1; int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) @@ -245,6 +302,10 @@ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, return fc_scan_seq(gear, p, n, fp, mask); if (fc_use_simd < 0) fc_use_simd = fc_simd_available(); +#ifdef FC_KIND_512 + if (fc_use_simd == 2) + return fc_scan_simd512(gear, p, n, fp, mask); +#endif if (fc_use_simd) return fc_scan_simd(gear, p, n, fp, mask); return fc_scan_blocked(gear, p, n, fp, mask); @@ -256,5 +317,9 @@ const char *fc_kernel_name(int force_scalar) return "scalar"; if (fc_use_simd < 0) fc_use_simd = fc_simd_available(); +#ifdef FC_KIND_512 + if (fc_use_simd == 2) + return FC_KIND_512; +#endif return fc_use_simd ? FC_KIND : "blocked"; } diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h index 10887ec3d8..a66606c707 100644 --- a/src/borg/chunkers/fastcdc_impl.h +++ b/src/borg/chunkers/fastcdc_impl.h @@ -20,7 +20,7 @@ * All kernels return bit-identical results. */ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar); -/* Name of the kernel fc_scan would use: "neon", "avx2", "blocked" or "scalar". */ +/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */ const char *fc_kernel_name(int force_scalar); #endif diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h index 0b0dd148ab..e758cae44a 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.h +++ b/src/borg/chunkers/goldilocks_aes_impl.h @@ -43,7 +43,7 @@ GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t a void gl_free(GL_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *gl_kind(const GL_CTX *ctx); /* Full (non-rolling) polynomial hash of the 64 bytes at q (Horner): the diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h index fe9856e6b1..ec8ad6a2e4 100644 --- a/src/borg/chunkers/phte_core.h +++ b/src/borg/chunkers/phte_core.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -25,6 +26,7 @@ typedef struct { uint8_t rk[11][16]; /* AES-128 round keys, for the hardware paths */ EVP_CIPHER_CTX *evp; /* portable path */ int use_hw; + int use_hw512; /* VAES/AVX-512 variant of the hardware path (x86-64) */ } PHTE_BASE; /* --- endianness-explicit helpers (byte loops, so cut points do not depend @@ -130,10 +132,34 @@ phte_aes1_ni(const uint8_t rk[11][16], __m128i b) return _mm_aesenclast_si128(b, _mm_loadu_si128((const __m128i *)rk[10])); } +/* VAES/AVX-512 variant of the hardware path (4 AES blocks per instruction). + * __builtin_cpu_supports("vaes") needs GCC >= 11 / clang >= 14; older + * compilers simply keep the 128-bit AES-NI path. */ +#if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || (defined(__clang__) && __clang_major__ >= 14) +#define PHTE_HAVE_HW512 1 +#define PHTE_KIND_HW512 "vaes" + +static int phte_hw512_available(void) +{ + /* BORG_PHTE_NO_VAES caps the AES chunkers at the 128-bit AES-NI path + * (for benchmarking the paths against each other); it is read at + * context creation, ""/"0" mean off. */ + const char *e; + if (!__builtin_cpu_supports("avx512f") || !__builtin_cpu_supports("vaes")) + return 0; + e = getenv("BORG_PHTE_NO_VAES"); + return e == NULL || e[0] == 0 || strcmp(e, "0") == 0; +} +#endif + #else #define PHTE_HAVE_HW 0 #endif +#ifndef PHTE_HAVE_HW512 +#define PHTE_HAVE_HW512 0 +#endif + /* --- context base management ------------------------------------------- */ /* Expand the AES key, select the scan path and set up the OpenSSL context. @@ -146,6 +172,11 @@ static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int force_sw) #else (void)force_sw; b->use_hw = 0; +#endif +#if PHTE_HAVE_HW512 + b->use_hw512 = b->use_hw && phte_hw512_available(); +#else + b->use_hw512 = 0; #endif b->evp = EVP_CIPHER_CTX_new(); if (b->evp == NULL) @@ -169,6 +200,10 @@ static void phte_base_free(PHTE_BASE *b) static const char *phte_base_kind(const PHTE_BASE *b) { +#if PHTE_HAVE_HW512 + if (b->use_hw512) + return PHTE_KIND_HW512; +#endif #if PHTE_HAVE_HW if (b->use_hw) return PHTE_KIND_HW; diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 4695cae788..776b2360da 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -19,9 +19,11 @@ * - a portable path batching digests through OpenSSL EVP AES-128-ECB, * - a hardware path using AES instructions directly (arm64 crypto extension, * x86-64 AES-NI), which interleaves the serial rolling-hash chain with - * pipelined AES so the AES work is (mostly) hidden behind it. + * pipelined AES so the AES work is (mostly) hidden behind it, + * - a VAES/AVX-512 variant of the x86-64 hardware path (runtime-detected) + * encrypting 4 AES blocks per instruction. * - * Both paths produce bit-identical cut points. + * All paths produce bit-identical cut points. */ #define PH_CAT2(a, b) a##b @@ -105,11 +107,12 @@ static int64_t PH_FN(scan_evp)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t * /* --- hardware paths ---------------------------------------------------- * - * Both hardware paths process groups of 8 positions: the two rolling lanes - * advance 4 stride-2 steps each (two independent dependency chains), then the - * 8 digests are encrypted with interleaved AES instructions, which execute on - * different ports than the rolling work and thus overlap with the next - * group's. + * The 128-bit hardware paths process groups of 8 positions (the VAES/AVX-512 + * path below uses the same structure with groups of 32): the two rolling + * lanes advance 4 stride-2 steps each (two independent dependency chains), + * then the 8 digests are encrypted with interleaved AES instructions, which + * execute on different ports than the rolling work and thus overlap with the + * next group's. * * Loop invariant: entering a group at position i, db = digest at i and * da = digest at i+1 (both already computed); the group emits digests for @@ -334,10 +337,130 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t #endif /* PHTE_HAVE_HW */ +#if PHTE_HAVE_HW512 + +/* VAES/AVX-512 variant of the x86-64 hardware path: the same two rolling + * lanes, but groups of 32 positions whose digests are encrypted as 8 zmm + * vectors of 4 AES blocks each - 4x fewer AES instructions than the 128-bit + * path, the round keys stay register-resident (32 zmm registers instead of + * 16 xmm, so no per-round key reloads), and the 8 independent chains hide + * the vaesenc latency (2 chains would be latency-bound). vpexpandq places a + * vector's 4 digests into its block-low qwords in one instruction, and a + * masked vptestnmq tests the 4 ciphertext-low qwords against the mask + * without extracting. Every digest is exact and every position is tested, + * so cut points stay bit-identical to the other paths. */ +__attribute__((target("aes,sse2,vaes,avx512f"))) static int64_t +PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) +{ + __m512i k[11]; + __m512i M = _mm512_set1_epi64((long long)mask); + uint64_t d = *digest, da, db; + const uint8_t *qo = p - 64; + size_t i = 0; + int lanes_live = 0; + + /* broadcast each 128-bit round key to all four block lanes */ + for (int j = 0; j < 11; j++) + k[j] = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c->base.rk[j])); + + if (n >= 34) { + db = PH_ROLL(c, d, qo[0], p[0]); /* d_0 */ + da = PH_ROLL2(c, d, qo[0], qo[1], p[0], p[1]); /* d_1 */ + lanes_live = 1; + } + + while (lanes_live && i + 34 <= n) { + uint64_t dg[32]; + __m512i b0, b1, b2, b3, b4, b5, b6, b7; + __mmask8 h0, h1, h2, h3, h4, h5, h6, h7; + + dg[0] = db; + dg[1] = da; + for (int j = 2; j < 32; j += 2) { + db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]); + dg[j] = db; + da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]); + dg[j + 1] = da; + } + /* prepare the next group's invariant (digests at i+32, i+33) */ + db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]); + da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]); + + /* qword lanes 0,2,4,6 take 4 digests, lanes 1,3,5,7 stay zero + * (= AES input block bytes 8..15, as in the other paths) */ + b0 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg), k[0]); + b1 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 4), k[0]); + b2 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 8), k[0]); + b3 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 12), k[0]); + b4 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 16), k[0]); + b5 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 20), k[0]); + b6 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 24), k[0]); + b7 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 28), k[0]); + for (int r = 1; r < 10; r++) { + b0 = _mm512_aesenc_epi128(b0, k[r]); + b1 = _mm512_aesenc_epi128(b1, k[r]); + b2 = _mm512_aesenc_epi128(b2, k[r]); + b3 = _mm512_aesenc_epi128(b3, k[r]); + b4 = _mm512_aesenc_epi128(b4, k[r]); + b5 = _mm512_aesenc_epi128(b5, k[r]); + b6 = _mm512_aesenc_epi128(b6, k[r]); + b7 = _mm512_aesenc_epi128(b7, k[r]); + } + h0 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b0, k[10]), M); + h1 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b1, k[10]), M); + h2 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b2, k[10]), M); + h3 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b3, k[10]), M); + h4 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b4, k[10]), M); + h5 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b5, k[10]), M); + h6 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b6, k[10]), M); + h7 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b7, k[10]), M); + + if (h0 | h1 | h2 | h3 | h4 | h5 | h6 | h7) { + /* block j of vector g is qword lane 2j and position i + 4g + j */ + const uint8_t hs[8] = {h0, h1, h2, h3, h4, h5, h6, h7}; + for (int j = 0; j < 32; j++) { + if ((hs[j >> 2] >> ((j & 3) * 2)) & 1) { + *digest = dg[j]; + return (int64_t)(i + j); + } + } + } + i += 32; + } + /* tail: single positions, exactly as in the 128-bit path */ + if (lanes_live && i < n) { + __m128i b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)db)); + if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) { + *digest = db; + return (int64_t)i; + } + d = db; + i++; + } + while (i < n) { + __m128i b; + d = PH_ROLL(c, d, qo[i], p[i]); + b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)d)); + if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) { + *digest = d; + return (int64_t)i; + } + i++; + } + *digest = d; + return -1; +} + +#endif /* PHTE_HAVE_HW512 */ + /* --- dispatch (exported) ------------------------------------------------ */ int64_t PH_FN(scan)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) { +#if PHTE_HAVE_HW512 + if (c->base.use_hw512) + return PH_FN(scan_hw512)(c, p, n, digest, mask); +#endif #if PHTE_HAVE_HW if (c->base.use_hw) return PH_FN(scan_hw)(c, p, n, digest, mask); diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h index 655c8b206b..22fe17ad46 100644 --- a/src/borg/chunkers/rabin_aes_impl.h +++ b/src/borg/chunkers/rabin_aes_impl.h @@ -40,7 +40,7 @@ RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); void ra_free(RA_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *ra_kind(const RA_CTX *ctx); /* Full (non-rolling) Rabin digest of the 64 bytes at q: the window warm-up at diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h index bc005dcf5b..dec05d32e7 100644 --- a/src/borg/chunkers/toeplitz_aes_impl.h +++ b/src/borg/chunkers/toeplitz_aes_impl.h @@ -44,7 +44,7 @@ TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); void tp_free(TP_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *tp_kind(const TP_CTX *ctx); /* Full (non-rolling) digest of the 64 bytes at q: the window warm-up at the diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py index 452a5b0583..ebb9849475 100644 --- a/src/borg/testsuite/chunkers/buzhash64_test.py +++ b/src/borg/testsuite/chunkers/buzhash64_test.py @@ -143,7 +143,7 @@ def rnd_key(): def test_buzhash64_kernels_identical(): - # the blocked/AVX2 scan kernel (auto-selected) and the plain sequential + # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the plain sequential # loop must produce identical cut points. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") @@ -162,4 +162,4 @@ def sizes(chunker): del os.environ["BORG_BUZHASH64_FORCE_SCALAR"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx2", "blocked", "scalar") + assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar") diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py index 68812f64ca..d2ce69f3e0 100644 --- a/src/borg/testsuite/chunkers/fastcdc_test.py +++ b/src/borg/testsuite/chunkers/fastcdc_test.py @@ -179,8 +179,8 @@ def rnd_key(): def test_fastcdc_kernels_identical(): - # the SIMD scan kernel (neon/avx2/blocked, auto-selected) and the plain - # sequential Gear loop must produce identical cut points. + # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the + # plain sequential Gear loop must produce identical cut points. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") @@ -198,4 +198,4 @@ def sizes(chunker): del os.environ["BORG_FASTCDC_FORCE_SCALAR"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx2", "blocked", "scalar") + assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar") diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py index 2ef3f0c219..2dc2cac3e6 100644 --- a/src/borg/testsuite/chunkers/phte_chunkers_test.py +++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py @@ -61,7 +61,7 @@ def sizes(chunker): del os.environ[env_var] assert sizes_default == sizes_evp # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("aes-arm64", "aes-ni", "evp") + assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp") def test_chunksize_distribution(chunker_spec): From 17ded35f186b389d4f34409882b0cfb778a01176 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 5 Aug 2026 14:52:53 +0200 Subject: [PATCH 02/13] chunkers: add BORG_FASTCDC_NO_AVX2 / BORG_BUZHASH64_NO_AVX2 Cap dispatch at the blocked scalar kernel, completing the bench ladder on x86-64: default (avx512) -> NO_AVX512 (avx2) -> NO_AVX2 (blocked) -> FORCE_SCALAR (sequential). Read once per process, like the CPU detection itself. Co-Authored-By: Claude Fable 5 --- src/borg/chunkers/buzhash64_impl.c | 22 ++++++++++++++-------- src/borg/chunkers/fastcdc.pyx | 4 ++-- src/borg/chunkers/fastcdc_impl.c | 22 ++++++++++++++-------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index 32936e8c3a..5e8a3568ee 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -250,16 +250,22 @@ bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot, return j; } +static int bz64_env_set(const char *name) +{ + const char *e = getenv(name); + return e != NULL && e[0] != 0 && strcmp(e, "0") != 0; +} + static int bz64_simd_available(void) { - /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2 (for benchmarking the - * kernels against each other); like the detection itself it is read - * once per process, so it must be set before the first chunker use. */ - if (__builtin_cpu_supports("avx512f")) { - const char *e = getenv("BORG_BUZHASH64_NO_AVX512"); - if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0) - return 2; - } + /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2 + * at the blocked scalar kernel (for benchmarking the kernels against + * each other); like the detection itself they are read once per process, + * so they must be set before the first chunker use. */ + if (bz64_env_set("BORG_BUZHASH64_NO_AVX2")) + return 0; + if (__builtin_cpu_supports("avx512f") && !bz64_env_set("BORG_BUZHASH64_NO_AVX512")) + return 2; return __builtin_cpu_supports("avx2"); } diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx index 535f189896..8d915d59c1 100644 --- a/src/borg/chunkers/fastcdc.pyx +++ b/src/borg/chunkers/fastcdc.pyx @@ -36,8 +36,8 @@ cdef extern from "fastcdc_impl.h": # aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to # the plain sequential Gear loop: every byte position is tested, cuts and hash state are # exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop -# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 (for benchmarking, must be -# set before the first chunker use). +# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1 +# at the blocked scalar kernel (for benchmarking, must be set before the first chunker use). @cython.boundscheck(False) diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index 4ddfbb1d82..46dfd141c6 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -262,16 +262,22 @@ fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_i return -1; } +static int fc_env_set(const char *name) +{ + const char *e = getenv(name); + return e != NULL && e[0] != 0 && strcmp(e, "0") != 0; +} + static int fc_simd_available(void) { - /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2 (for benchmarking the - * kernels against each other); like the detection itself it is read - * once per process, so it must be set before the first chunker use. */ - if (__builtin_cpu_supports("avx512f")) { - const char *e = getenv("BORG_FASTCDC_NO_AVX512"); - if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0) - return 2; - } + /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at + * the blocked scalar kernel (for benchmarking the kernels against each + * other); like the detection itself they are read once per process, so + * they must be set before the first chunker use. */ + if (fc_env_set("BORG_FASTCDC_NO_AVX2")) + return 0; + if (__builtin_cpu_supports("avx512f") && !fc_env_set("BORG_FASTCDC_NO_AVX512")) + return 2; return __builtin_cpu_supports("avx2"); } From ddd54be9f24daa45e9fb7ce4e283d2038cccb3f0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 01:13:59 +0200 Subject: [PATCH 03/13] chunkers: build the aligned domain in the vector kernels The AVX2 and AVX-512 scan kernels of fastcdc and buzhash64 only vectorized the 8-lane candidate test: the per-block prefix work stayed scalar and was handed over through the stack. Eight 8-byte stores immediately followed by one 32/64-byte vector load cannot use store-to-load forwarding, so every block paid a forwarding stall - enough that the fastcdc AVX2 kernel was slower than the blocked scalar fallback it is supposed to beat. Both vector kernels now: - do the table lookups for the next block while the current one is tested (double buffer), which puts a full loop body between the stores and the load, so the stores have retired by then and the stall disappears, and - build the aligned domain in the vector domain: vpsllvq / vprolvq for the per-lane shifts resp. rotations, then the prefix sum / prefix XOR over the lanes - three valignq steps on AVX-512, two vpermq steps per 4-lane half plus a carry on AVX2. Only the table lookups stay scalar. The serial chain across blocks stays what it was (one scalar add + shift resp. rotate); s[7] is read out of the vector with vpermq off that chain. A gather-based kernel avoiding the round-trip entirely was tried and dropped: vpgatherqq is too slow on Zen 4 (1102 vs 2778 MB/s). Pipelining two blocks ahead instead of one also regressed. All kernels still return bit-identical cut points; the kernel-identity and fuzz tests cover this, and chunk boundaries were verified equal across the avx512, avx2 and blocked tiers for all chunkers. Measured on an AMD Ryzen 5 8500GE (Zen 4/4c, AVX-512 on a 256-bit datapath), borg benchmark cpu, 1 GB, pinned: fastcdc 0.818s -> 0.381s (2.15x) buzhash64 0.906s -> 0.667s (1.36x) Per kernel tier (MB/s, isolated chunker benchmark): fastcdc avx512 1173 -> 2689 avx2 800 -> 2583 buzhash64 avx512 1052 -> 1509 avx2 1026 -> 1478 AVX-512 stays close to AVX2 here because Zen 4 double-pumps 512-bit ops; the win is the removed stall and the vectorized prefix, not the width. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/buzhash64_impl.c | 167 +++++++++++++++++++++-------- src/borg/chunkers/fastcdc_impl.c | 163 ++++++++++++++++++++-------- 2 files changed, 239 insertions(+), 91 deletions(-) diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index 5e8a3568ee..e54f45374b 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -172,39 +172,99 @@ static int bz64_simd_available(void) #include +/* Load the block's 8 per-byte deltas D_t = Trot[out_t] ^ T[in_t] (via two + * 8-byte data loads), without the rotations and the prefix XOR: the vector + * kernels do those in the vector domain. Endianness-independent. */ +static inline void bz64_block_delta(const uint64_t *T, const uint64_t *Trot, + const uint8_t *pr, const uint8_t *pa, uint64_t d[8]) +{ + uint64_t wr, wa; + memcpy(&wr, pr, 8); + memcpy(&wa, pa, 8); +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wr = __builtin_bswap64(wr); + wa = __builtin_bswap64(wa); +#endif + d[0] = Trot[(uint8_t)wr] ^ T[(uint8_t)wa]; + d[1] = Trot[(uint8_t)(wr >> 8)] ^ T[(uint8_t)(wa >> 8)]; + d[2] = Trot[(uint8_t)(wr >> 16)] ^ T[(uint8_t)(wa >> 16)]; + d[3] = Trot[(uint8_t)(wr >> 24)] ^ T[(uint8_t)(wa >> 24)]; + d[4] = Trot[(uint8_t)(wr >> 32)] ^ T[(uint8_t)(wa >> 32)]; + d[5] = Trot[(uint8_t)(wr >> 40)] ^ T[(uint8_t)(wa >> 40)]; + d[6] = Trot[(uint8_t)(wr >> 48)] ^ T[(uint8_t)(wa >> 48)]; + d[7] = Trot[(uint8_t)(wr >> 56)] ^ T[(uint8_t)(wa >> 56)]; +} + +/* AVX2 has no variable rotate, so ROTL(v, k) is built from a variable left + * and right shift. The k = 0 lane needs a 64-bit right shift, which + * vpsrlvq defines as zero - exactly what (v << 0) | 0 = v needs. */ +__attribute__((target("avx2"))) static inline __m256i bz64_rotl4(__m256i v, __m256i k, __m256i kc) +{ + return _mm256_or_si256(_mm256_sllv_epi64(v, k), _mm256_srlv_epi64(v, kc)); +} + +/* The 8-lane test in two 256-bit vectors, with the aligned domain built in + * the vector domain: per-lane rotations, then a prefix XOR inside each 4-lane + * half (two vpermq+vpxor steps) plus a broadcast of the low half's total into + * the high half. + * + * Only the 2x8 table lookups stay scalar, and they are done one block ahead + * into a double buffer: a 32-byte vector load placed right after the eight + * 8-byte scalar stores that filled it cannot use store-to-load forwarding and + * stalls, twice per block. Loading block j+1 while block j is tested puts a + * full loop body between the stores and the load, so the stall disappears. */ __attribute__((target("avx2"))) static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { uint64_t sum = *sum_io; - uint64_t s[8]; + uint64_t d[16]; /* double buffer: block being tested + block being loaded */ size_t j = 0; + int cur = 0; /* _mm256_set_epi64x takes arguments high lane first */ - __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), - (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); - __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1), - (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3)); - __m256i zero = _mm256_setzero_si256(); - - while (j + 8 <= n && (sum & mask) != 0) { - bz64_block_prefix(T, Trot, pr + j, pa + j, s); - uint64_t c = BZ_ROTL(sum, 8); - __m256i C = _mm256_set1_epi64x((long long)c); - __m256i H1 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[0])); - __m256i H2 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[4])); - __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); - __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); - __m256i any = _mm256_or_si256(z1, z2); - if (!_mm256_testz_si256(any, any)) { - size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ - *sum_io = sum; - return j + r; + const __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), + (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); + const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1), + (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3)); + const __m256i KL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 rotate by 7..4 */ + const __m256i KH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 rotate by 3..0 */ + const __m256i KLC = _mm256_set_epi64x(60, 59, 58, 57); /* 64 - KL */ + const __m256i KHC = _mm256_set_epi64x(64, 63, 62, 61); /* 64 - KH */ + const __m256i zero = _mm256_setzero_si256(); + + if (n >= 16 && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr, pa, d); + while (j + 16 <= n && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */ + __m256i lo = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur)), KL, KLC); + __m256i hi = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur + 4)), KH, KHC); + /* prefix XOR inside each half: rotate the lanes up by 1 resp. 2 + * (vpermq), zero the lanes rotated in, XOR */ + lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03)); + lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F)); + hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03)); + hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F)); + hi = _mm256_xor_si256(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */ + uint64_t c = BZ_ROTL(sum, 8); + __m256i C = _mm256_set1_epi64x((long long)c); + __m256i H1 = _mm256_xor_si256(C, lo); + __m256i H2 = _mm256_xor_si256(C, hi); + __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); + __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); + __m256i any = _mm256_or_si256(z1, z2); + if (!_mm256_testz_si256(any, any)) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128( + _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */ + j += 8; + cur ^= 8; } - sum = c ^ s[7]; - j += 8; } - if (j < n) + if (j < n) /* up to 15 bytes here (one block more than the scalar kernels) */ j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); *sum_io = sum; return j; @@ -214,37 +274,56 @@ bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, #define BZ_KIND_512 "avx512" -/* The same 8-lane test as the AVX2 kernel in one 512-bit vector: vptestnmq - * fuses the AND and the ==0 test per lane into a mask register. Only the - * test narrows; the scalar prefix work (bz64_block_prefix) is shared. */ +/* The same 8-lane test as the AVX2 kernel, but the aligned domain fits in one + * 512-bit vector: vprolvq applies the per-lane rotations (lane k by 7-k) in + * one instruction, three valignq+vpxorq steps turn the rotated deltas into + * the prefix XORs s[0..7] in one pass instead of two halves plus a carry, and + * vptestnmq fuses the AND and the ==0 test into a mask register. + * + * The table lookups are pipelined one block ahead exactly as in the AVX2 + * kernel above, for the same store-to-load forwarding reason. */ __attribute__((target("avx512f"))) static size_t bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { uint64_t sum = *sum_io; - uint64_t s[8]; + uint64_t d[16]; /* double buffer: block being tested + block being loaded */ size_t j = 0; + int cur = 0; /* _mm512_set_epi64 takes arguments high lane first: lane k = ROTL(mask, 7-k) */ - __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1), - (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3), - (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), - (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); - - while (j + 8 <= n && (sum & mask) != 0) { - bz64_block_prefix(T, Trot, pr + j, pa + j, s); - uint64_t c = BZ_ROTL(sum, 8); - __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c), - _mm512_loadu_si512((const void *)s)); - if (_mm512_testn_epi64_mask(H, M)) { - size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ - *sum_io = sum; - return j + r; + const __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1), + (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3), + (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), + (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); + const __m512i RT = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane k: ROTL by 7-k */ + const __m512i Z = _mm512_setzero_si512(); + + if (n >= 16 && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr, pa, d); + while (j + 16 <= n && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */ + /* u = rotated deltas, then the prefix XOR over the 8 lanes + * (shift lanes up by 1, 2, 4 and XOR; _mm512_alignr_epi64(u, Z, + * 8-k) shifts up by k, zero-filling) */ + __m512i u = _mm512_rolv_epi64(_mm512_loadu_si512((const void *)(d + cur)), RT); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 7)); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 6)); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 4)); + uint64_t c = BZ_ROTL(sum, 8); + __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c), u); + if (_mm512_testn_epi64_mask(H, M)) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128( + _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */ + j += 8; + cur ^= 8; } - sum = c ^ s[7]; - j += 8; } - if (j < n) + if (j < n) /* up to 15 bytes here (one block more than the other kernels) */ j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); *sum_io = sum; return j; diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index 46dfd141c6..5924ae4304 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -173,39 +173,87 @@ static int fc_simd_available(void) #include +/* Load the block's 8 gear values (via one 8-byte data load), without the + * shifts and the prefix sum: the vector kernels do those in the vector + * domain. Endianness-independent: bytes are extracted by shifting. */ +static inline void fc_block_gear(const uint64_t *gear, const uint8_t *p, uint64_t g[8]) +{ + uint64_t w; + memcpy(&w, p, 8); +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + w = __builtin_bswap64(w); +#endif + g[0] = gear[(uint8_t)w]; + g[1] = gear[(uint8_t)(w >> 8)]; + g[2] = gear[(uint8_t)(w >> 16)]; + g[3] = gear[(uint8_t)(w >> 24)]; + g[4] = gear[(uint8_t)(w >> 32)]; + g[5] = gear[(uint8_t)(w >> 40)]; + g[6] = gear[(uint8_t)(w >> 48)]; + g[7] = gear[(uint8_t)(w >> 56)]; +} + +/* The 8-lane candidate test in two 256-bit vectors, with the aligned domain + * built in the vector domain: vpsllvq applies the per-lane shifts, then an + * inclusive prefix sum inside each 4-lane half (two vpermq+vpaddq steps) plus + * a broadcast of the low half's total into the high half. + * + * Only the 8 gear table lookups stay scalar, and they are done one block + * ahead into a double buffer: a 32-byte vector load placed right after the + * eight 8-byte scalar stores that filled it cannot use store-to-load + * forwarding and stalls, twice per block. Loading block i+1 while block i is + * tested puts a full loop body between the stores and the load, so the stores + * have retired by then and the stall disappears. Before this, the stalls made + * the AVX2 kernel slower than the blocked scalar one. */ __attribute__((target("avx2"))) static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; - uint64_t s[8]; + uint64_t g[16]; /* double buffer: block being tested + block being loaded */ /* _mm256_set_epi64x takes arguments high lane first */ - __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5), - (long long)(mask << 6), (long long)(mask << 7)); - __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1), - (long long)(mask << 2), (long long)(mask << 3)); - __m256i zero = _mm256_setzero_si256(); + const __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5), + (long long)(mask << 6), (long long)(mask << 7)); + const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1), + (long long)(mask << 2), (long long)(mask << 3)); + const __m256i SL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 <<= 7..4 */ + const __m256i SH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 <<= 3..0 */ + const __m256i zero = _mm256_setzero_si256(); size_t i = 0; - - for (; i + 8 <= n; i += 8) { - fc_block_prefix(gear, p + i, s); - uint64_t c = fp << 8; - __m256i C = _mm256_set1_epi64x((long long)c); - __m256i H1 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[0])); - __m256i H2 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[4])); - __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); - __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); - __m256i any = _mm256_or_si256(z1, z2); - if (!_mm256_testz_si256(any, any)) { - int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ - if (r >= 0) { - *fp_io = fp; - return (int64_t)i + r; + int cur = 0; + + if (n >= 16) { + fc_block_gear(gear, p, g); + for (; i + 16 <= n; i += 8, cur ^= 8) { + fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */ + __m256i lo = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur)), SL); + __m256i hi = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur + 4)), SH); + /* inclusive prefix sum inside each half: rotate the lanes up by 1 + * resp. 2 (vpermq), zero the lanes rotated in, add */ + lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03)); + lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F)); + hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03)); + hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F)); + hi = _mm256_add_epi64(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */ + uint64_t c = fp << 8; + __m256i C = _mm256_set1_epi64x((long long)c); + __m256i H1 = _mm256_add_epi64(C, lo); + __m256i H2 = _mm256_add_epi64(C, hi); + __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); + __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); + __m256i any = _mm256_or_si256(z1, z2); + if (!_mm256_testz_si256(any, any)) { + int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } else { + fp = c + (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128( + _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */ } - } else { - fp = c + s[7]; } } - if (i < n) { + if (i < n) { /* up to 15 bytes here (one block more than the scalar kernels) */ int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask); if (r >= 0) { *fp_io = fp; @@ -220,38 +268,59 @@ fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, #define FC_KIND_512 "avx512" -/* The same 8-lane candidate test as the AVX2 kernel, in one 512-bit vector: - * vptestnmq fuses the AND and the ==0 test per lane into a mask register, - * so the whole test is broadcast + load + add + testn + branch. Only the - * test narrows; the scalar prefix work (fc_block_prefix) is shared. */ +/* The same 8-lane candidate test as the AVX2 kernel, but the aligned domain + * fits in one 512-bit vector: vpsllvq applies the per-lane shifts (lane j << + * (7-j)), three valignq+vpaddq steps turn the shifted gear values into the + * inclusive prefix sums s[0..7] in one pass instead of two halves plus a + * carry, and vptestnmq fuses the AND and the ==0 test into a mask register. + * + * The gear lookups are pipelined one block ahead exactly as in the AVX2 + * kernel above, for the same store-to-load forwarding reason. + * + * The serial chain stays short: only fp = c + s[7] (scalar add + shift per + * 8 bytes) is carried across blocks, and s[7] is read out of the vector with + * vpermq off that chain. */ __attribute__((target("avx512f"))) static int64_t fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; - uint64_t s[8]; + uint64_t g[16]; /* double buffer: block being tested + block being loaded */ /* _mm512_set_epi64 takes arguments high lane first: lane j = mask << (7-j) */ - __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1), - (long long)(mask << 2), (long long)(mask << 3), - (long long)(mask << 4), (long long)(mask << 5), - (long long)(mask << 6), (long long)(mask << 7)); + const __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1), + (long long)(mask << 2), (long long)(mask << 3), + (long long)(mask << 4), (long long)(mask << 5), + (long long)(mask << 6), (long long)(mask << 7)); + const __m512i SH = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane j <<= 7-j */ + const __m512i Z = _mm512_setzero_si512(); size_t i = 0; - - for (; i + 8 <= n; i += 8) { - fc_block_prefix(gear, p + i, s); - uint64_t c = fp << 8; - __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c), - _mm512_loadu_si512((const void *)s)); - if (_mm512_testn_epi64_mask(H, M)) { - int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ - if (r >= 0) { - *fp_io = fp; - return (int64_t)i + r; + int cur = 0; + + if (n >= 16) { + fc_block_gear(gear, p, g); + for (; i + 16 <= n; i += 8, cur ^= 8) { + fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */ + /* u = shifted gear values, then the inclusive prefix sum over the + * 8 lanes (Hillis-Steele: shift lanes up by 1, 2, 4 and add; + * _mm512_alignr_epi64(u, Z, 8-k) shifts up by k, zero-filling) */ + __m512i u = _mm512_sllv_epi64(_mm512_loadu_si512((const void *)(g + cur)), SH); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 7)); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 6)); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 4)); + uint64_t c = fp << 8; + __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c), u); + if (_mm512_testn_epi64_mask(H, M)) { + int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } else { + fp = c + (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128( + _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */ } - } else { - fp = c + s[7]; } } - if (i < n) { + if (i < n) { /* up to 15 bytes here (one block more than the other kernels) */ int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask); if (r >= 0) { *fp_io = fp; From aa21d6f037f669de4515eeb4b2ceeed2e1f621f5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 08:44:00 +0200 Subject: [PATCH 04/13] goldilocks-aes: keep the field reductions out of the branch predictor gl_add and gl_mul reduce with plain if() statements. The conditions are data-dependent and essentially random, and GCC turns them into real conditional jumps - 55 of them in the VAES scan kernel, missing about 27% of the time. The mispredicts, not the arithmetic, were what made this chunker slow: a single gl_mul cost 15.1 cycles, against 1.8 for the raw 64x64 multiply it is built on and 3.1 for the same code with the final canonicalisation removed. Writing the reductions as mask arithmetic or as a ternary does not help: GCC recognises both and converts them straight back to branches (measured identical miss counts). The __builtin_*_overflow forms do, because the carry/borrow stays in the flags and the compiler settles on sbb/adc plus a mask. gl_mul drops to 4.4 cycles and the kernel keeps 5 conditional jumps instead of 55. Everything stays canonical, which matters because the state is fed to AES verbatim - a different representation of the same field element would change cut decisions. Cut points are bit-identical: the chunker tests and the fuzz tests pass, and the chunk-boundary digest is unchanged across the vaes, aes-ni and evp paths. Measured on an AMD Ryzen 5 8500GE, borg benchmark cpu, 1 GB, both states built and measured back to back, pinned to one core, best of two passes: goldilocks-aes 7.121s -> 2.164s (3.29x) Isolated kernel, same box: 94.0 -> 336.9 MB/s. This is the rolling hash only, so it applies to all three scan paths, not just the VAES one. rabin-aes and toeplitz-aes are unaffected: their rolling hashes are GF(2)[x] and have no data-dependent reduction. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/goldilocks_aes_impl.c | 34 +++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c index 5b045e89fc..423f76669f 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.c +++ b/src/borg/chunkers/goldilocks_aes_impl.c @@ -35,14 +35,19 @@ struct GL_CTX { * AES verbatim, so a non-canonical representation of the same field element * would change cut decisions. */ +/* The reductions are data-dependent and unpredictable (~27% branch-miss rate + * measured), so they must not become branches. Writing them as if() or as a + * ternary lets GCC emit real conditional jumps - the mispredicts alone cost + * more than the arithmetic (gl_mul: 15.1 vs 4.4 cycles per call). The + * __builtin_*_overflow forms keep the carry/borrow in the flags, so the + * compiler settles on sbb/adc + mask instead. Results are unchanged: the + * value is still fully canonical, which matters because it is fed to AES. */ static inline uint64_t gl_add(uint64_t a, uint64_t b) { - uint64_t s = a + b; - if (s < a) /* carry out: 2^64 mod p = GL_EPS; cannot re-carry */ - s += GL_EPS; - if (s >= GL_P) - s -= GL_P; - return s; + uint64_t s, u; + unsigned char carry = __builtin_add_overflow(a, b, &s); + s += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* 2^64 mod p; cannot re-carry */ + return __builtin_sub_overflow(s, GL_P, &u) ? s : u; } /* a * b mod p via the standard 2^64 = 2^32 - 1 folding (risc0/plonky2 style): @@ -53,16 +58,13 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b) __uint128_t t = (__uint128_t)a * b; uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64); uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS; - uint64_t t0 = lo - hi_hi; - if (lo < hi_hi) /* borrow: -2^64 = -eps (mod p) */ - t0 -= GL_EPS; - uint64_t t1 = hi_lo * GL_EPS; /* < 2^64 - 2^33 + 2, no overflow */ - uint64_t r = t0 + t1; - if (r < t1) /* carry out; cannot re-carry (t1 <= 2^64 - 2^33 + 1) */ - r += GL_EPS; - if (r >= GL_P) - r -= GL_P; - return r; + uint64_t t0, r, u; + unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0); + t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */ + /* t1 = hi_lo * eps < 2^64 - 2^33 + 2, no overflow */ + unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r); + r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */ + return __builtin_sub_overflow(r, GL_P, &u) ? r : u; } /* Advance the state by one byte. The state is the window's polynomial hash From 202088852156fa759a9762d7aa01ce05082faadf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 09:39:07 +0200 Subject: [PATCH 05/13] chunkers: store the AES chunkers' digests pre-spread An AES input block wants the digest in its low qword and zero in its high one. The VAES kernel stored the 32 digests of a group packed and spread them at use with vpexpandq, eight of those per group. Storing them pre-spread costs nothing extra: the odd slots hold the zeros, they are written once when the scan starts and never touched again, and the chain writes the same 32 values it wrote before - just at 2j instead of j. Each vector's AES input then becomes a plain 512-bit load. Measured on an AMD Ryzen 5 8500GE, isolated scan kernel, pinned, best of 5: toeplitz-aes 652.9 -> 712.8 MB/s (+9.2%) rabin-aes 650.0 -> 690.8 MB/s (+6.3%) goldilocks-aes 337.3 -> 347.2 MB/s (+2.9%) borg benchmark cpu, 1 GB, both states built and measured back to back, pinned, best of three passes: toeplitz-aes 1.250s -> 1.078s (1.16x) rabin-aes 1.176s -> 1.107s (1.06x) goldilocks-aes 2.160s -> 2.192s (unchanged; the passes overlap) goldilocks-aes is dominated by its field arithmetic, so it barely notices either way. Cut points are bit-identical: the chunker and fuzz tests pass and the chunk-boundary digests are unchanged on all three chunkers across the vaes, aes-ni and evp paths. This is the only one of several attempts that paid off. For the record, on this CPU the following were measured and rejected: widening the rolling hash to four lanes at stride 4 (toeplitz 653 -> 402, rabin 649 -> 458 MB/s, because stride-4 doubles the table footprint against a 32 KiB L1d while the lookups per position stay the same); hoisting the chain-independent table lookups into their own pass (653 -> 579 and 650 -> 495, the delta buffer's traffic costs more than the scheduling gain, so the hardware was already hoisting them); and rolling the digest buffer one group ahead (+2.5% and +1%, not worth the complexity). A microbenchmark of the rolling step alone puts the ceiling for extra lanes at 16% for rabin and nothing for toeplitz, which is why the four-lane attempt could not have won. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/phte_scan.h | 52 +++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 776b2360da..2492b602a6 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -344,16 +344,25 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t * vectors of 4 AES blocks each - 4x fewer AES instructions than the 128-bit * path, the round keys stay register-resident (32 zmm registers instead of * 16 xmm, so no per-round key reloads), and the 8 independent chains hide - * the vaesenc latency (2 chains would be latency-bound). vpexpandq places a - * vector's 4 digests into its block-low qwords in one instruction, and a - * masked vptestnmq tests the 4 ciphertext-low qwords against the mask - * without extracting. Every digest is exact and every position is tested, - * so cut points stay bit-identical to the other paths. */ + * the vaesenc latency (2 chains would be latency-bound), and a masked + * vptestnmq tests the 4 ciphertext-low qwords against the mask without + * extracting. Every digest is exact and every position is tested, so cut + * points stay bit-identical to the other paths. + * + * The digests are stored pre-spread: an AES input block wants the digest in + * the low qword and zero in the high one, so dgs[] keeps a digest in every + * even slot and a zero in every odd one. The zeros are written once at entry + * and never touched again, which makes each vector's input a plain aligned + * 512-bit load. Packing the digests and spreading them with vpexpandq (eight + * of those per group) instead costs the same stores and measurably more + * time - about 9% of the whole toeplitz-aes scan, 6% of rabin-aes. */ __attribute__((target("aes,sse2,vaes,avx512f"))) static int64_t PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) { __m512i k[11]; __m512i M = _mm512_set1_epi64((long long)mask); + /* digest at dgs[2j], zero at dgs[2j+1] (= AES input block bytes 8..15) */ + uint64_t dgs[64] __attribute__((aligned(64))); uint64_t d = *digest, da, db; const uint8_t *qo = p - 64; size_t i = 0; @@ -362,6 +371,8 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6 /* broadcast each 128-bit round key to all four block lanes */ for (int j = 0; j < 11; j++) k[j] = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c->base.rk[j])); + for (int j = 1; j < 64; j += 2) + dgs[j] = 0; if (n >= 34) { db = PH_ROLL(c, d, qo[0], p[0]); /* d_0 */ @@ -370,32 +381,31 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6 } while (lanes_live && i + 34 <= n) { - uint64_t dg[32]; __m512i b0, b1, b2, b3, b4, b5, b6, b7; __mmask8 h0, h1, h2, h3, h4, h5, h6, h7; - dg[0] = db; - dg[1] = da; + dgs[0] = db; + dgs[2] = da; for (int j = 2; j < 32; j += 2) { db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]); - dg[j] = db; + dgs[2 * j] = db; da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]); - dg[j + 1] = da; + dgs[2 * j + 2] = da; } /* prepare the next group's invariant (digests at i+32, i+33) */ db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]); da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]); - /* qword lanes 0,2,4,6 take 4 digests, lanes 1,3,5,7 stay zero - * (= AES input block bytes 8..15, as in the other paths) */ - b0 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg), k[0]); - b1 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 4), k[0]); - b2 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 8), k[0]); - b3 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 12), k[0]); - b4 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 16), k[0]); - b5 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 20), k[0]); - b6 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 24), k[0]); - b7 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 28), k[0]); + /* each vector's 4 blocks are already laid out in dgs (digest in the + * low qword of every block, zero in the high one) */ + b0 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 0)), k[0]); + b1 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 8)), k[0]); + b2 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 16)), k[0]); + b3 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 24)), k[0]); + b4 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 32)), k[0]); + b5 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 40)), k[0]); + b6 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 48)), k[0]); + b7 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 56)), k[0]); for (int r = 1; r < 10; r++) { b0 = _mm512_aesenc_epi128(b0, k[r]); b1 = _mm512_aesenc_epi128(b1, k[r]); @@ -420,7 +430,7 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6 const uint8_t hs[8] = {h0, h1, h2, h3, h4, h5, h6, h7}; for (int j = 0; j < 32; j++) { if ((hs[j >> 2] >> ((j & 3) * 2)) & 1) { - *digest = dg[j]; + *digest = dgs[2 * j]; return (int64_t)(i + j); } } From a9e5d032a41ee8142d16dee473f60bf645069301 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 11:30:10 +0200 Subject: [PATCH 06/13] fastcdc: test against a limit instead of masking, on aarch64 Per lane the NEON kernel tested "(c + s_j) & M_j == 0". Because (x & M) == 0 means x's set bits are a subset of ~M's, which implies x <= ~M, the and plus compare-against-zero can be one unsigned compare against a limit computed once per scan. It never loses a cut for any mask, and for the contiguous high-bit masks fastcdc uses it is exactly equivalent - still the same superset test the block's sequential recheck resolves. Apple M3 Pro, 128 MiB of random data, chunkify end to end: 3088 -> 3431 MB/s (1.11x). Cut points unchanged (verified against the sequential kernel). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/fastcdc_impl.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index 5924ae4304..fe2c49a8d3 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -121,24 +121,35 @@ static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n, #include +/* Per lane, "(x & M) == 0" is replaced by "x <= ~M", turning the and plus + * compare-against-zero into one unsigned compare against a limit computed + * once per scan. + * + * (x & M) == 0 means x's set bits are a subset of ~M's, which implies + * x <= ~M for any M - so this never loses a cut, whatever the mask looks + * like. For the contiguous high-bit masks the chunker actually uses (mask + * has its one-bits at the top and shifting left only drops bits off the top, + * so ~M is 2^(64-k)-1) the two are exactly equivalent, and this stays the + * same superset test as the masked form: the block's exact sequential + * recheck resolves candidates either way. */ static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; uint64_t s[8]; - uint64x2_t M12 = {mask << 7, mask << 6}; - uint64x2_t M34 = {mask << 5, mask << 4}; - uint64x2_t M56 = {mask << 3, mask << 2}; - uint64x2_t M78 = {mask << 1, mask}; + uint64x2_t L12 = {~(mask << 7), ~(mask << 6)}; + uint64x2_t L34 = {~(mask << 5), ~(mask << 4)}; + uint64x2_t L56 = {~(mask << 3), ~(mask << 2)}; + uint64x2_t L78 = {~(mask << 1), ~mask}; size_t i = 0; for (; i + 8 <= n; i += 8) { fc_block_prefix(gear, p + i, s); uint64_t c = fp << 8; uint64x2_t C = vdupq_n_u64(c); - uint64x2_t z12 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[0], s[1]}), M12)); - uint64x2_t z34 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[2], s[3]}), M34)); - uint64x2_t z56 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[4], s[5]}), M56)); - uint64x2_t z78 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[6], s[7]}), M78)); + uint64x2_t z12 = vcgeq_u64(L12, vaddq_u64(C, (uint64x2_t){s[0], s[1]})); + uint64x2_t z34 = vcgeq_u64(L34, vaddq_u64(C, (uint64x2_t){s[2], s[3]})); + uint64x2_t z56 = vcgeq_u64(L56, vaddq_u64(C, (uint64x2_t){s[4], s[5]})); + uint64x2_t z78 = vcgeq_u64(L78, vaddq_u64(C, (uint64x2_t){s[6], s[7]})); uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78)); if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ From 193a26f4e9f3ade4708258f5c9768c194953c410 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 11:30:45 +0200 Subject: [PATCH 07/13] buzhash64: drop the NEON kernel, blocked scalar is faster there The NEON kernel lost to the blocked scalar one on Apple M3 Pro: 2323 vs 2519 MB/s end to end, and in an isolated kernel benchmark 2360 vs 2590 MB/s with the same ~9% gap at every mask size from 17 to 23 bits. So aarch64 now falls through to the blocked scalar kernel. Nothing in this test wants a vector register. clang keeps the block's eight prefix XORs in general registers, and xor/and/compare per lane retires at more than one lane per cycle on the wide scalar ALUs; the vector form first has to move those eight values to the SIMD side and then reduce the 8-lane result back to a condition flag (umaxv) before the loop branch can resolve. fastcdc keeps its NEON kernel because its per-lane work is larger (add plus per-lane shifted masks) - enough to pay for the trip, and there it wins by 2x. Cut points unchanged - this only changes which bit-identical kernel runs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/buzhash64_impl.c | 77 +++++++++--------------------- 1 file changed, 23 insertions(+), 54 deletions(-) diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index e54f45374b..b5c05c57c6 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -21,13 +21,9 @@ * Trot[b] = ROTL(T[b], window_size % 64) is precomputed by the caller, which * also removes one rotate per byte from the sequential path. * - * Kernel dispatch mirrors fastcdc_impl.c: NEON on aarch64 (baseline there), - * AVX-512 or AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere, - * sequential when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit - * mask): sequential 1150, blocked scalar 2340, NEON 2260 MB/s - NEON and - * blocked are a tie here (throughput and energy, within noise), because the - * XOR/AND/compare test also runs well on the scalar ALUs; the symmetric - * dispatch is kept for consistency across the SIMD chunker kernels. + * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blocked + * scalar everywhere else including aarch64 (which has no vector kernel, see + * the note where it would go), sequential when forced. * All kernels return bit-identical results. */ #include @@ -118,56 +114,29 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, return j; } -/* --- NEON (aarch64 baseline, always available) -------------------------- */ - -#if defined(__aarch64__) -#define BZ_KIND "neon" - -#include - -static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, - const uint8_t *pr, const uint8_t *pa, - size_t n, uint64_t *sum_io, uint64_t mask) -{ - uint64_t sum = *sum_io; - uint64_t s[8]; - size_t j = 0; - uint64x2_t M12 = {BZ_ROTL(mask, 7), BZ_ROTL(mask, 6)}; - uint64x2_t M34 = {BZ_ROTL(mask, 5), BZ_ROTL(mask, 4)}; - uint64x2_t M56 = {BZ_ROTL(mask, 3), BZ_ROTL(mask, 2)}; - uint64x2_t M78 = {BZ_ROTL(mask, 1), mask}; - - while (j + 8 <= n && (sum & mask) != 0) { - bz64_block_prefix(T, Trot, pr + j, pa + j, s); - uint64_t c = BZ_ROTL(sum, 8); - uint64x2_t Cv = vdupq_n_u64(c); - uint64x2_t z12 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[0], s[1]}), M12)); - uint64x2_t z34 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[2], s[3]}), M34)); - uint64x2_t z56 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[4], s[5]}), M56)); - uint64x2_t z78 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[6], s[7]}), M78)); - uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78)); - if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { - size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ - *sum_io = sum; - return j + r; - } - sum = c ^ s[7]; - j += 8; - } - if (j < n) - j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); - *sum_io = sum; - return j; -} - -static int bz64_simd_available(void) -{ - return 1; -} +/* --- aarch64: no vector kernel ------------------------------------------ + * + * There was a NEON kernel here doing the 8-lane test in vector registers. + * It is gone because it lost to the blocked scalar kernel: measured on an + * Apple M3 Pro (48 MiB, window 4095), blocked 2590 MB/s vs NEON 2360 MB/s, + * the same ~9% gap at every mask size from 17 to 23 bits. + * + * The reason is that nothing here needs a vector register in the first + * place. clang keeps the block's eight prefix XORs in general registers and + * the scalar test is xor/and/compare per lane, which the wide scalar ALUs + * retire at more than one lane per cycle; the vector form has to move those + * eight values across to the SIMD side and then reduce the 8-lane result + * back to one condition flag (umaxv, ~4 cycles) before the loop branch can + * resolve. The move-and-reduce round trip costs more than the parallel test + * saves. fastcdc keeps its NEON kernel because its test is add/and/compare + * with per-lane shifted masks - enough extra per-lane work to pay for the + * trip - and there it wins by 2x. + * + * So aarch64 falls through to the blocked scalar kernel below. */ /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */ -#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) #define BZ_KIND "avx2" #include From 063edc015964481dc97aa6dc7ca53c2d52e386a6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 11:31:00 +0200 Subject: [PATCH 08/13] chunkers: keep the AES chunkers' cut test in the vector domain on aarch64 The aarch64 scan moved all 8 ciphertexts to general registers (one fmov each) and tested them with 8 scalar and/compare pairs, even though the common path needs nothing but the single "did any lane hit" bit. uzp1 packs the low halves - the cut-decision uint64 - of two blocks into one vector, so the test is 4 and/compare-zero pairs, an or-reduce and one umaxv; the 8 values are only materialised on an actual hit. Apple M3 Pro, 128 MiB of random data, chunkify end to end: toeplitz-aes 866 -> 921 MB/s (1.06x) rabin-aes 854 -> 912 MB/s (1.07x) goldilocks-aes 461 -> 522 MB/s (1.13x) Cut points unchanged (verified against the EVP path). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/phte_scan.h | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 2492b602a6..1362658099 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -181,18 +181,28 @@ static int64_t PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *d b6 = veorq_u8(vaeseq_u8(b6, k[9]), k[10]); b7 = veorq_u8(vaeseq_u8(b7, k[9]), k[10]); + /* Test all 8 ciphertexts without leaving the vector domain: uzp1 + * packs the low halves (the cut-decision uint64) of two blocks into + * one vector, then and/cmeq-zero/or-reduce. Moving the 8 values to + * general registers instead - one fmov each, plus 8 scalar and/cmp + * pairs - costs noticeably more on the common no-hit path, where + * nothing but the single "any lane hit" bit is ever needed. */ { - uint64_t c0 = vgetq_lane_u64(vreinterpretq_u64_u8(b0), 0); - uint64_t c1 = vgetq_lane_u64(vreinterpretq_u64_u8(b1), 0); - uint64_t c2 = vgetq_lane_u64(vreinterpretq_u64_u8(b2), 0); - uint64_t c3 = vgetq_lane_u64(vreinterpretq_u64_u8(b3), 0); - uint64_t c4 = vgetq_lane_u64(vreinterpretq_u64_u8(b4), 0); - uint64_t c5 = vgetq_lane_u64(vreinterpretq_u64_u8(b5), 0); - uint64_t c6 = vgetq_lane_u64(vreinterpretq_u64_u8(b6), 0); - uint64_t c7 = vgetq_lane_u64(vreinterpretq_u64_u8(b7), 0); - if ((((c0 & mask) == 0) | ((c1 & mask) == 0) | ((c2 & mask) == 0) | ((c3 & mask) == 0) | - ((c4 & mask) == 0) | ((c5 & mask) == 0) | ((c6 & mask) == 0) | ((c7 & mask) == 0))) { - uint64_t cs[8] = {c0, c1, c2, c3, c4, c5, c6, c7}; + const uint64x2_t maskv = vdupq_n_u64(mask); + uint64x2_t h01 = vuzp1q_u64(vreinterpretq_u64_u8(b0), vreinterpretq_u64_u8(b1)); + uint64x2_t h23 = vuzp1q_u64(vreinterpretq_u64_u8(b2), vreinterpretq_u64_u8(b3)); + uint64x2_t h45 = vuzp1q_u64(vreinterpretq_u64_u8(b4), vreinterpretq_u64_u8(b5)); + uint64x2_t h67 = vuzp1q_u64(vreinterpretq_u64_u8(b6), vreinterpretq_u64_u8(b7)); + uint64x2_t any = vorrq_u64(vorrq_u64(vceqzq_u64(vandq_u64(h01, maskv)), + vceqzq_u64(vandq_u64(h23, maskv))), + vorrq_u64(vceqzq_u64(vandq_u64(h45, maskv)), + vceqzq_u64(vandq_u64(h67, maskv)))); + if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { + uint64_t cs[8]; + vst1q_u64(cs + 0, h01); + vst1q_u64(cs + 2, h23); + vst1q_u64(cs + 4, h45); + vst1q_u64(cs + 6, h67); for (int j = 0; j < 8; j++) { if ((cs[j] & mask) == 0) { *digest = dg[j]; From e310b896f3473dd087ab63b9045489b437d7d418 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 11:59:20 +0200 Subject: [PATCH 09/13] goldilocks-aes: drop the multiply's redundant canonicalization gl_mul ended with a conditional subtraction of p to return a canonical field element, but both callers feed the result straight into a gl_add that canonicalizes anyway. gl_mul_lazy skips it and returns a merely reduced representative (< 2^64 rather than < p). gl_add still returns canonical with a non-canonical first operand: for a < 2^64 and b < p a carry leaves a + b - 2^64 <= p - 2, so adding eps cannot carry a second time, and the single conditional subtraction of p then lands below p (r - p < 2^32). The digest handed to AES therefore stays canonical, which it must be - it is encrypted verbatim, so a different representative of the same element would move cut points. This kernel is bound by the multiply latency chain (~17 cycles per stride-2 roll, two lanes, which alone accounts for its throughput), so removing the trailing compare and select from that chain is worth more than the op count suggests. Apple M3 Pro, 128 MiB of random data, chunkify end to end: 517 -> 561 MB/s (1.08x). Cut points unchanged (identical chunk-boundary digests, and identical over 48 MiB at 17/19/21/23 mask bits against the canonical version). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/goldilocks_aes_impl.c | 36 +++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c index 423f76669f..f808a59ebe 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.c +++ b/src/borg/chunkers/goldilocks_aes_impl.c @@ -31,9 +31,11 @@ struct GL_CTX { /* --- Goldilocks field arithmetic --------------------------------------- * - * All values are kept canonical (< p) at every step: the state is fed to - * AES verbatim, so a non-canonical representation of the same field element - * would change cut decisions. */ + * Every digest the scan produces is canonical (< p): it is fed to AES + * verbatim, so a non-canonical representation of the same field element + * would change cut decisions. The rolls reach that via gl_add, which always + * canonicalizes; only the multiply inside them is allowed to hand on a + * merely reduced (< 2^64) representative, see gl_mul_lazy. */ /* The reductions are data-dependent and unpredictable (~27% branch-miss rate * measured), so they must not become branches. Writing them as if() or as a @@ -67,6 +69,30 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b) return __builtin_sub_overflow(r, GL_P, &u) ? r : u; } +/* gl_mul without the final canonicalization: the result is congruent to a*b + * mod p but only bounded by 2^64, not by p. + * + * That is enough for the rolls, because both feed it straight into a gl_add + * whose other operand is canonical, and that gl_add canonicalizes anyway. + * The bound still holds with a non-canonical first operand: for a < 2^64 and + * b < p, a carry leaves a + b - 2^64 <= p - 2, so adding eps cannot carry a + * second time, and the one conditional subtraction of p then lands below p + * (r - p < 2^32). So the digest handed to AES stays canonical - which it must + * be, since it is fed to AES verbatim - while the multiply chain that limits + * this kernel loses its trailing compare and select. */ +static inline uint64_t gl_mul_lazy(uint64_t a, uint64_t b) +{ + __uint128_t t = (__uint128_t)a * b; + uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64); + uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS; + uint64_t t0, r; + unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0); + t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */ + unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r); + r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */ + return r; +} + /* Advance the state by one byte. The state is the window's polynomial hash * s = sum b_j * K^(63-j) (oldest byte at K^63, newest at K^0), so * s' = s*K - byte_out*K^64 + byte_in. @@ -75,7 +101,7 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b) static inline uint64_t gl_roll(const GL_CTX *c, uint64_t s, uint8_t byte_out, uint8_t byte_in) { uint64_t delta = gl_add(c->nout64_tbl[byte_out], byte_in); - return gl_add(gl_mul(s, c->k1), delta); + return gl_add(gl_mul_lazy(s, c->k1), delta); } /* Advance the state by TWO bytes in one step (exact composition of two @@ -89,7 +115,7 @@ static inline uint64_t gl_roll2(const GL_CTX *c, uint64_t s, { uint64_t delta = gl_add(gl_add(c->nout65_tbl[o0], c->nout64_tbl[o1]), gl_add(c->in1_tbl[i0], i1)); - return gl_add(gl_mul(s, c->k2), delta); + return gl_add(gl_mul_lazy(s, c->k2), delta); } uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q) From 18744d69a1bc6711fc09b27e3ad31ae8e591f381 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 12:17:52 +0200 Subject: [PATCH 10/13] chunkers: rename the "blocked" scan kernel to "blockwise" "blocked" is a past participle and reads as "obstructed" or "disabled" rather than "processed in blocks". The neighbouring API makes that misreading easy to land on: after setting BORG_FASTCDC_NO_AVX2=1, which does block AVX2, the chunker reports kernel == "blocked", which parses as "the kernel was blocked" - a wrong reading that happens to be almost right, so nothing corrects it. The exported name also dropped the word that carried the meaning: the code says "blocked scalar" in prose but exported only "blocked". "blockwise" says block by block and cannot mean obstructed. The name has not been released (it was added after 2.0.0b22), so nothing depends on it. Also drops 'neon' from buzhash64's documented kernel values and from the changelog: buzhash64 has no vector kernel on aarch64 any more, so that value can no longer be returned. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- docs/changes.rst | 5 ++-- src/borg/chunkers/buzhash64.pyx | 2 +- src/borg/chunkers/buzhash64_impl.c | 24 +++++++++---------- src/borg/chunkers/buzhash64_impl.h | 5 ++-- src/borg/chunkers/fastcdc.pyx | 6 ++--- src/borg/chunkers/fastcdc_impl.c | 20 ++++++++-------- src/borg/chunkers/fastcdc_impl.h | 2 +- src/borg/testsuite/chunkers/buzhash64_test.py | 4 ++-- src/borg/testsuite/chunkers/fastcdc_test.py | 4 ++-- 9 files changed, 37 insertions(+), 35 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 7c6a90d785..28872ef66b 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -188,9 +188,10 @@ New features: - fastcdc is the new and faster default chunker, #9957 - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043: - - NEON on aarch64 (e.g. Apple Silicon) - AVX-512 / AVX2 on x86-64 (Intel / AMD) - - blocked scalar elsewhere + - fastcdc also NEON on aarch64 (e.g. Apple Silicon); buzhash64 is faster + blockwise there, so it has no vector kernel on aarch64 + - blockwise scalar elsewhere - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987 - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64 diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx index 4e7b75a40a..03874b92c9 100644 --- a/src/borg/chunkers/buzhash64.pyx +++ b/src/borg/chunkers/buzhash64.pyx @@ -155,7 +155,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'.""" + """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'.""" return (bz64_kernel_name(self.force_scalar)).decode("ascii") cdef object process(self): diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index b5c05c57c6..27fdbcd0ad 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -21,7 +21,7 @@ * Trot[b] = ROTL(T[b], window_size % 64) is precomputed by the caller, which * also removes one rotate per byte from the sequential path. * - * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blocked + * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blockwise * scalar everywhere else including aarch64 (which has no vector kernel, see * the note where it would go), sequential when forced. * All kernels return bit-identical results. */ @@ -82,9 +82,9 @@ static inline void bz64_block_prefix(const uint64_t *T, const uint64_t *Trot, s[7] = s8; } -/* --- blocked scalar (the default kernel) --------------------------------- */ +/* --- blockwise scalar (the default kernel) --------------------------------- */ -static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, +static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { @@ -117,8 +117,8 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, /* --- aarch64: no vector kernel ------------------------------------------ * * There was a NEON kernel here doing the 8-lane test in vector registers. - * It is gone because it lost to the blocked scalar kernel: measured on an - * Apple M3 Pro (48 MiB, window 4095), blocked 2590 MB/s vs NEON 2360 MB/s, + * It is gone because it lost to the blockwise scalar kernel: measured on an + * Apple M3 Pro (48 MiB, window 4095), blockwise 2590 MB/s vs NEON 2360 MB/s, * the same ~9% gap at every mask size from 17 to 23 bits. * * The reason is that nothing here needs a vector register in the first @@ -132,7 +132,7 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, * with per-lane shifted masks - enough extra per-lane work to pay for the * trip - and there it wins by 2x. * - * So aarch64 falls through to the blocked scalar kernel below. */ + * So aarch64 falls through to the blockwise scalar kernel below. */ /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */ @@ -307,7 +307,7 @@ static int bz64_env_set(const char *name) static int bz64_simd_available(void) { /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2 - * at the blocked scalar kernel (for benchmarking the kernels against + * at the blockwise scalar kernel (for benchmarking the kernels against * each other); like the detection itself they are read once per process, * so they must be set before the first chunker use. */ if (bz64_env_set("BORG_BUZHASH64_NO_AVX2")) @@ -318,13 +318,13 @@ static int bz64_simd_available(void) } #else -#define BZ_KIND "blocked" +#define BZ_KIND "blockwise" static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { - return bz64_scan_blocked(T, Trot, pr, pa, n, sum_io, mask); + return bz64_scan_blockwise(T, Trot, pr, pa, n, sum_io, mask); } static int bz64_simd_available(void) @@ -336,7 +336,7 @@ static int bz64_simd_available(void) /* --- dispatch ----------------------------------------------------------- */ -/* resolved once (0 = blocked, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined); +/* resolved once (0 = blockwise, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined); * a racy double-init writes the same value, so it is benign */ static int bz64_use_simd = -1; @@ -354,7 +354,7 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, #endif if (bz64_use_simd) return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask); - return bz64_scan_blocked(table, table_rot, p_rem, p_add, n, sum, mask); + return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask); } const char *bz64_kernel_name(int force_scalar) @@ -367,5 +367,5 @@ const char *bz64_kernel_name(int force_scalar) if (bz64_use_simd == 2) return BZ_KIND_512; #endif - return bz64_use_simd ? BZ_KIND : "blocked"; + return bz64_use_simd ? BZ_KIND : "blockwise"; } diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h index 2a9220cc73..032d5367c7 100644 --- a/src/borg/chunkers/buzhash64_impl.h +++ b/src/borg/chunkers/buzhash64_impl.h @@ -1,5 +1,5 @@ /* buzhash64 chunker scan kernel: the cyclic-polynomial rolling hash inner - * loop, with blocked/SIMD implementations that are bit-identical to the + * loop, with blockwise/SIMD implementations that are bit-identical to the * plain sequential loop (see buzhash64_impl.c for the algebra). * * The cut decision granularity stays at 1 byte: every position is tested @@ -27,7 +27,8 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, size_t n, uint64_t *sum, uint64_t mask, int force_scalar); -/* Name of the kernel bz64_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */ +/* Name of the kernel bz64_scan would use: "avx512", "avx2", "blockwise" or "scalar" + * (there is no vector kernel on aarch64, see buzhash64_impl.c). */ const char *bz64_kernel_name(int force_scalar); #endif diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx index 8d915d59c1..04f6414cb2 100644 --- a/src/borg/chunkers/fastcdc.pyx +++ b/src/borg/chunkers/fastcdc.pyx @@ -33,11 +33,11 @@ cdef extern from "fastcdc_impl.h": # class only provides the keyed Gear table and the _scan() hook calling the C kernel. # # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on -# aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to +# aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to # the plain sequential Gear loop: every byte position is tested, cuts and hash state are # exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop # (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1 -# at the blocked scalar kernel (for benchmarking, must be set before the first chunker use). +# at the blockwise scalar kernel (for benchmarking, must be set before the first chunker use). @cython.boundscheck(False) @@ -87,7 +87,7 @@ cdef class ChunkerFastCDC(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'.""" + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.""" return (fc_kernel_name(self.force_scalar)).decode("ascii") cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept: diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index fe2c49a8d3..59e97457dd 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -21,7 +21,7 @@ * which an exact sequential recheck of the block resolves. The per-lane * masks (mask << 7 .. mask << 0) are constants of the scan. * - * All kernels (sequential, blocked scalar, NEON, AVX2, AVX-512) return + * All kernels (sequential, blockwise scalar, NEON, AVX2, AVX-512) return * bit-identical cut positions and fp values; the chunker's golden tests * depend on this. */ @@ -76,9 +76,9 @@ static inline void fc_block_prefix(const uint64_t *gear, const uint8_t *p, uint6 s[7] = s8; } -/* --- blocked scalar (portable C, no intrinsics) ------------------------- */ +/* --- blockwise scalar (portable C, no intrinsics) ------------------------- */ -static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) +static int64_t fc_scan_blockwise(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; uint64_t M[8]; @@ -215,7 +215,7 @@ static inline void fc_block_gear(const uint64_t *gear, const uint8_t *p, uint64_ * forwarding and stalls, twice per block. Loading block i+1 while block i is * tested puts a full loop body between the stores and the load, so the stores * have retired by then and the stall disappears. Before this, the stalls made - * the AVX2 kernel slower than the blocked scalar one. */ + * the AVX2 kernel slower than the blockwise scalar one. */ __attribute__((target("avx2"))) static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { @@ -351,7 +351,7 @@ static int fc_env_set(const char *name) static int fc_simd_available(void) { /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at - * the blocked scalar kernel (for benchmarking the kernels against each + * the blockwise scalar kernel (for benchmarking the kernels against each * other); like the detection itself they are read once per process, so * they must be set before the first chunker use. */ if (fc_env_set("BORG_FASTCDC_NO_AVX2")) @@ -362,11 +362,11 @@ static int fc_simd_available(void) } #else -#define FC_KIND "blocked" +#define FC_KIND "blockwise" static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { - return fc_scan_blocked(gear, p, n, fp_io, mask); + return fc_scan_blockwise(gear, p, n, fp_io, mask); } static int fc_simd_available(void) @@ -378,7 +378,7 @@ static int fc_simd_available(void) /* --- dispatch ----------------------------------------------------------- */ -/* resolved once (0 = blocked, 1 = FC_KIND, 2 = FC_KIND_512 where defined); +/* resolved once (0 = blockwise, 1 = FC_KIND, 2 = FC_KIND_512 where defined); * a racy double-init writes the same value, so it is benign */ static int fc_use_simd = -1; @@ -394,7 +394,7 @@ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, #endif if (fc_use_simd) return fc_scan_simd(gear, p, n, fp, mask); - return fc_scan_blocked(gear, p, n, fp, mask); + return fc_scan_blockwise(gear, p, n, fp, mask); } const char *fc_kernel_name(int force_scalar) @@ -407,5 +407,5 @@ const char *fc_kernel_name(int force_scalar) if (fc_use_simd == 2) return FC_KIND_512; #endif - return fc_use_simd ? FC_KIND : "blocked"; + return fc_use_simd ? FC_KIND : "blockwise"; } diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h index a66606c707..eab87de6da 100644 --- a/src/borg/chunkers/fastcdc_impl.h +++ b/src/borg/chunkers/fastcdc_impl.h @@ -20,7 +20,7 @@ * All kernels return bit-identical results. */ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar); -/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */ +/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blockwise" or "scalar". */ const char *fc_kernel_name(int force_scalar); #endif diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py index ebb9849475..3a336f80db 100644 --- a/src/borg/testsuite/chunkers/buzhash64_test.py +++ b/src/borg/testsuite/chunkers/buzhash64_test.py @@ -143,7 +143,7 @@ def rnd_key(): def test_buzhash64_kernels_identical(): - # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the plain sequential + # the SIMD scan kernel (avx512/avx2/blockwise, auto-selected) and the plain sequential # loop must produce identical cut points. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") @@ -162,4 +162,4 @@ def sizes(chunker): del os.environ["BORG_BUZHASH64_FORCE_SCALAR"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar") + assert default.kernel in ("avx512", "avx2", "blockwise", "scalar") diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py index d2ce69f3e0..4962bf99eb 100644 --- a/src/borg/testsuite/chunkers/fastcdc_test.py +++ b/src/borg/testsuite/chunkers/fastcdc_test.py @@ -179,7 +179,7 @@ def rnd_key(): def test_fastcdc_kernels_identical(): - # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the + # the SIMD scan kernel (neon/avx512/avx2/blockwise, auto-selected) and the # plain sequential Gear loop must produce identical cut points. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") @@ -198,4 +198,4 @@ def sizes(chunker): del os.environ["BORG_FASTCDC_FORCE_SCALAR"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar") + assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar") From 350748c3e42c6614459d9d2cfb3b77920c65555b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 13:21:07 +0200 Subject: [PATCH 11/13] chunkers: one BORG_*_KERNEL env var per chunker family Replaces ten env vars with three: BORG_FASTCDC_NO_AVX512=1 -> BORG_FASTCDC_KERNEL=avx2 BORG_FASTCDC_NO_AVX2=1 -> BORG_FASTCDC_KERNEL=blockwise BORG_FASTCDC_FORCE_SCALAR=1 -> BORG_FASTCDC_KERNEL=scalar (same for BORG_BUZHASH64_*) BORG_PHTE_NO_VAES=1 -> BORG_AES_CHUNKER_KERNEL=aes-ni BORG__AES_FORCE_EVP=1 -> BORG_AES_CHUNKER_KERNEL=evp The NO_* names were negative flags that did not do what they said: they capped a dispatch ladder rather than disabling one kernel, so NO_AVX2=1 also disabled AVX-512. They could not express "use AVX-512" at all, they used a second grammar (FORCE_*) for the same job, and they left no way to select the blockwise kernel on aarch64, where the NO_AVX* vars are no-ops. The values are exactly the strings the .kernel property reports, so what you read back is what you set. "auto" (the default) takes the best kernel the CPU can run; any other value is a demand, and creating the chunker raises ValueError if it cannot be honoured. Silently falling back is the failure this is meant to prevent: it turns a benchmark, or a CI job that means to pin one kernel, into a measurement of a different one. The error separates the three reasons, which need different fixes - a typo, too old a compiler, or the wrong CPU - because on x86-64 "your CPU has VAES but your build does not" is a real and distinct case (it needs gcc >= 11 / clang >= 14). The three per-chunker *_AES_FORCE_EVP vars become one: all three AES chunkers share phte_scan.h, so which scan paths exist is a property of the build and the CPU, never of the individual chunker. Selection now happens per chunker instance, so the old caveat that the NO_* vars had to be set before the first chunker use is gone. Parsing and validation live in one place (kernel_env.py plus the C name resolvers) instead of being split between getenv() in C and os.environ in Cython. None of the old names were ever released (they postdate 2.0.0b22). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- src/borg/chunkers/buzhash64.pyx | 39 ++++-- src/borg/chunkers/buzhash64_impl.c | 112 ++++++++++++----- src/borg/chunkers/buzhash64_impl.h | 32 ++++- src/borg/chunkers/fastcdc.pyx | 42 +++++-- src/borg/chunkers/fastcdc_impl.c | 119 +++++++++++++----- src/borg/chunkers/fastcdc_impl.h | 36 +++++- src/borg/chunkers/goldilocks_aes.pyx | 35 +++++- src/borg/chunkers/goldilocks_aes_impl.c | 4 +- src/borg/chunkers/goldilocks_aes_impl.h | 6 +- src/borg/chunkers/kernel_env.py | 42 +++++++ src/borg/chunkers/phte_core.h | 71 +++++++++-- src/borg/chunkers/phte_kernel.h | 36 ++++++ src/borg/chunkers/rabin_aes.pyx | 31 ++++- src/borg/chunkers/rabin_aes_impl.c | 4 +- src/borg/chunkers/rabin_aes_impl.h | 6 +- src/borg/chunkers/toeplitz_aes.pyx | 33 ++++- src/borg/chunkers/toeplitz_aes_impl.c | 4 +- src/borg/chunkers/toeplitz_aes_impl.h | 6 +- src/borg/testsuite/chunkers/buzhash64_test.py | 4 +- src/borg/testsuite/chunkers/fastcdc_test.py | 33 ++++- .../testsuite/chunkers/phte_chunkers_test.py | 29 +++-- 21 files changed, 584 insertions(+), 140 deletions(-) create mode 100644 src/borg/chunkers/kernel_env.py create mode 100644 src/borg/chunkers/phte_kernel.h diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx index 03874b92c9..26c94de41e 100644 --- a/src/borg/chunkers/buzhash64.pyx +++ b/src/borg/chunkers/buzhash64.pyx @@ -17,8 +17,13 @@ from .base cimport ChunkerBase cdef extern from "buzhash64_impl.h": size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar) nogil - const char *bz64_kernel_name(int force_scalar) + size_t n, uint64_t *sum, uint64_t mask, int kernel) nogil + const char *bz64_kernel_name(int kernel) + int bz64_kernel_select(const char *name, int *out_id) + const char *bz64_kernel_names() + int BZ_K_AUTO + +from .kernel_env import kernel_error, requested_kernel # Cyclic polynomial / buzhash # @@ -101,6 +106,20 @@ cdef uint64_t _buzhash64_update(uint64_t sum, unsigned char remove, unsigned cha return BARREL_SHIFT64(sum, 1) ^ BARREL_SHIFT64(h[remove], lenmod) ^ h[add] +cdef int _select_kernel() except -1: + """Resolve BORG_BUZHASH64_KERNEL to a kernel id, raising if it cannot be honoured.""" + cdef int kid = BZ_K_AUTO + cdef int rc + want = requested_kernel("BORG_BUZHASH64_KERNEL") + if want is None: + return BZ_K_AUTO + rc = bz64_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_BUZHASH64_KERNEL", want, rc, + (bz64_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerBuzHash64(ChunkerBase): """ Content-Defined Chunker, variable chunk sizes. @@ -118,7 +137,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): """ cdef uint64_t* table cdef uint64_t* table_rot - cdef int force_scalar + cdef int kernel_id cdef size_t window_size def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int hash_window_size, int nc_level=0, size_t normal_size=0, bint sparse=False): @@ -139,7 +158,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): lenmod = hash_window_size & 0x3f for i_rot in range(256): self.table_rot[i_rot] = BARREL_SHIFT64(self.table[i_rot], lenmod) - self.force_scalar = 1 if os.environ.get("BORG_BUZHASH64_FORCE_SCALAR", "") not in ("", "0") else 0 + self.kernel_id = _select_kernel() # buzhash64 output is uniform, so contiguous low-bit masks are used (high_masks=False) self._setup_common("buzhash64", chunk_min_exp, chunk_max_exp, hash_mask_bits, nc_level, normal_size, False, sparse) @@ -155,8 +174,12 @@ cdef class ChunkerBuzHash64(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'.""" - return (bz64_kernel_name(self.force_scalar)).decode("ascii") + """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'. + + With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the + requested kernel - creating the chunker fails otherwise. + """ + return (bz64_kernel_name(self.kernel_id)).decode("ascii") cdef object process(self): """Process the chunker's buffer and return the next chunk.""" @@ -169,7 +192,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): cdef uint8_t* stop_at cdef uint8_t* nc_stop cdef size_t did_bytes, span - cdef int force_scalar = self.force_scalar + cdef int kernel_id = self.kernel_id if self.done: if self.bytes_read == self.bytes_yielded: @@ -229,7 +252,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): span = stop_at - p with nogil: did_bytes = bz64_scan(self.table, self.table_rot, p, p + window_size, - span, &sum, mask, force_scalar) + span, &sum, mask, kernel_id) self.position += did_bytes self.remaining -= did_bytes diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index 27fdbcd0ad..4284a6d6d3 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -298,22 +298,12 @@ bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot, return j; } -static int bz64_env_set(const char *name) -{ - const char *e = getenv(name); - return e != NULL && e[0] != 0 && strcmp(e, "0") != 0; -} - static int bz64_simd_available(void) { - /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2 - * at the blockwise scalar kernel (for benchmarking the kernels against - * each other); like the detection itself they are read once per process, - * so they must be set before the first chunker use. */ - if (bz64_env_set("BORG_BUZHASH64_NO_AVX2")) - return 0; - if (__builtin_cpu_supports("avx512f") && !bz64_env_set("BORG_BUZHASH64_NO_AVX512")) +#ifdef BZ_KIND_512 + if (__builtin_cpu_supports("avx512f")) return 2; +#endif return __builtin_cpu_supports("avx2"); } @@ -334,38 +324,102 @@ static int bz64_simd_available(void) #endif +/* --- kernel selection --------------------------------------------------- */ + +const char *bz64_kernel_names(void) +{ +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, avx512, avx2, blockwise, scalar"; +#else + return "auto, blockwise, scalar"; +#endif +} + +int bz64_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = BZ_K_AUTO; + return BZ_KSEL_OK; + } + if (strcmp(name, "scalar") == 0) { + *out_id = BZ_K_SCALAR; + return BZ_KSEL_OK; + } + if (strcmp(name, "blockwise") == 0) { + *out_id = BZ_K_BLOCKWISE; + return BZ_KSEL_OK; + } +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "avx2") == 0) { + if (!__builtin_cpu_supports("avx2")) + return BZ_KSEL_NOCPU; + *out_id = BZ_K_VECTOR; + return BZ_KSEL_OK; + } + if (strcmp(name, "avx512") == 0) { +#ifndef BZ_KIND_512 + return BZ_KSEL_NOTBUILT; /* compiler too old for the target attribute */ +#else + if (!__builtin_cpu_supports("avx512f")) + return BZ_KSEL_NOCPU; + *out_id = BZ_K_VECTOR512; + return BZ_KSEL_OK; +#endif + } +#endif + return BZ_KSEL_UNKNOWN; +} + /* --- dispatch ----------------------------------------------------------- */ -/* resolved once (0 = blockwise, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined); - * a racy double-init writes the same value, so it is benign */ -static int bz64_use_simd = -1; +/* the kernel BZ_K_AUTO resolves to, worked out once; a racy double-init + * writes the same value, so it is benign */ +static int bz64_auto = -1; + +static int bz64_auto_kernel(void) +{ + int a; + if (bz64_auto < 0) { + a = bz64_simd_available(); + bz64_auto = (a == 2) ? BZ_K_VECTOR512 : (a ? BZ_K_VECTOR : BZ_K_BLOCKWISE); + } + return bz64_auto; +} size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar) + size_t n, uint64_t *sum, uint64_t mask, int kernel) { - if (force_scalar) + if (kernel == BZ_K_AUTO) + kernel = bz64_auto_kernel(); + switch (kernel) { + case BZ_K_SCALAR: return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask); - if (bz64_use_simd < 0) - bz64_use_simd = bz64_simd_available(); + case BZ_K_BLOCKWISE: + return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask); #ifdef BZ_KIND_512 - if (bz64_use_simd == 2) + case BZ_K_VECTOR512: return bz64_scan_simd512(table, table_rot, p_rem, p_add, n, sum, mask); #endif - if (bz64_use_simd) + default: return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask); - return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask); + } } -const char *bz64_kernel_name(int force_scalar) +const char *bz64_kernel_name(int kernel) { - if (force_scalar) + if (kernel == BZ_K_AUTO) + kernel = bz64_auto_kernel(); + switch (kernel) { + case BZ_K_SCALAR: return "scalar"; - if (bz64_use_simd < 0) - bz64_use_simd = bz64_simd_available(); + case BZ_K_BLOCKWISE: + return "blockwise"; #ifdef BZ_KIND_512 - if (bz64_use_simd == 2) + case BZ_K_VECTOR512: return BZ_KIND_512; #endif - return bz64_use_simd ? BZ_KIND : "blockwise"; + default: + return BZ_KIND; + } } diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h index 032d5367c7..f6dbfdd900 100644 --- a/src/borg/chunkers/buzhash64_impl.h +++ b/src/borg/chunkers/buzhash64_impl.h @@ -21,14 +21,34 @@ * ROTL(table[b], window_size % 64) (precomputed by the caller). * p_rem points at the byte leaving the window, p_add at the byte entering * (p_add = p_rem + window_size); both must have n readable bytes. - * force_scalar != 0 selects the sequential reference loop (for tests); - * all kernels return bit-identical results. */ + * kernel is one of BZ_K_*; BZ_K_AUTO picks the best one this CPU can run. + * All kernels return bit-identical results. */ +/* Scan kernel ids, a tier ladder; see fastcdc_impl.h for the rationale. */ +#define BZ_K_AUTO 0 /* best kernel this CPU can run */ +#define BZ_K_SCALAR 1 /* sequential reference loop */ +#define BZ_K_BLOCKWISE 2 /* portable 8-lane C */ +#define BZ_K_VECTOR 3 /* avx2 (there is no aarch64 vector kernel) */ +#define BZ_K_VECTOR512 4 /* avx512 */ + +/* Results of bz64_kernel_select(). */ +#define BZ_KSEL_OK 0 +#define BZ_KSEL_UNKNOWN 1 /* not a kernel name on this platform */ +#define BZ_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define BZ_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a kernel name for this build; see fc_kernel_select(). */ +int bz64_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the kernel names this build accepts. */ +const char *bz64_kernel_names(void); + size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar); + size_t n, uint64_t *sum, uint64_t mask, int kernel); -/* Name of the kernel bz64_scan would use: "avx512", "avx2", "blockwise" or "scalar" - * (there is no vector kernel on aarch64, see buzhash64_impl.c). */ -const char *bz64_kernel_name(int force_scalar); +/* Name of the kernel selects: "avx512", "avx2", "blockwise" or + * "scalar" (there is no vector kernel on aarch64, see buzhash64_impl.c). + * For BZ_K_AUTO, the auto-selected one. */ +const char *bz64_kernel_name(int kernel); #endif diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx index 04f6414cb2..ca0d11216c 100644 --- a/src/borg/chunkers/fastcdc.pyx +++ b/src/borg/chunkers/fastcdc.pyx @@ -13,8 +13,13 @@ from ..crypto.low_level import CSPRNG from .base cimport ChunkerBase cdef extern from "fastcdc_impl.h": - int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) nogil - const char *fc_kernel_name(int force_scalar) + int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel) nogil + const char *fc_kernel_name(int kernel) + int fc_kernel_select(const char *name, int *out_id) + const char *fc_kernel_names() + int FC_K_AUTO + +from .kernel_env import kernel_error, requested_kernel # FastCDC content-defined chunker (Xia et al., USENIX ATC 2016). # @@ -35,9 +40,8 @@ cdef extern from "fastcdc_impl.h": # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on # aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to # the plain sequential Gear loop: every byte position is tested, cuts and hash state are -# exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop -# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1 -# at the blockwise scalar kernel (for benchmarking, must be set before the first chunker use). +# exactly the same, only faster. BORG_FASTCDC_KERNEL pins the kernel instead of letting it +# be auto-selected; see kernel_env.py and the .kernel property. @cython.boundscheck(False) @@ -60,6 +64,20 @@ cdef uint64_t* fastcdc_init_gear(bytes key) except NULL: return gear +cdef int _select_kernel() except -1: + """Resolve BORG_FASTCDC_KERNEL to a kernel id, raising if it cannot be honoured.""" + cdef int kid = FC_K_AUTO + cdef int rc + want = requested_kernel("BORG_FASTCDC_KERNEL") + if want is None: + return FC_K_AUTO + rc = fc_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_FASTCDC_KERNEL", want, rc, + (fc_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerFastCDC(ChunkerBase): """ FastCDC content-defined chunker, variable chunk sizes, keyed Gear hash. @@ -67,11 +85,11 @@ cdef class ChunkerFastCDC(ChunkerBase): Unlike the buzhash chunkers, Gear is window-less, so there is no hash_window_size parameter. """ cdef uint64_t* gear - cdef int force_scalar + cdef int kernel_id def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int nc_level=0, size_t normal_size=0, bint sparse=False): self.gear = NULL - self.force_scalar = 1 if os.environ.get("BORG_FASTCDC_FORCE_SCALAR", "") not in ("", "0") else 0 + self.kernel_id = _select_kernel() self.gear = fastcdc_init_gear(key) # Gear accumulates information in its high bits, so the cut-decision # masks must use the high bits of the hash (high_masks=True). The Gear @@ -87,13 +105,17 @@ cdef class ChunkerFastCDC(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.""" - return (fc_kernel_name(self.force_scalar)).decode("ascii") + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'. + + With BORG_FASTCDC_KERNEL set to anything but "auto", this is always the + requested kernel - creating the chunker fails otherwise. + """ + return (fc_kernel_name(self.kernel_id)).decode("ascii") cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept: cdef int64_t r with nogil: - r = fc_scan(self.gear, p, n, digest, mask, self.force_scalar) + r = fc_scan(self.gear, p, n, digest, mask, self.kernel_id) return r diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index 59e97457dd..ac58923727 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -342,22 +342,12 @@ fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_i return -1; } -static int fc_env_set(const char *name) -{ - const char *e = getenv(name); - return e != NULL && e[0] != 0 && strcmp(e, "0") != 0; -} - static int fc_simd_available(void) { - /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at - * the blockwise scalar kernel (for benchmarking the kernels against each - * other); like the detection itself they are read once per process, so - * they must be set before the first chunker use. */ - if (fc_env_set("BORG_FASTCDC_NO_AVX2")) - return 0; - if (__builtin_cpu_supports("avx512f") && !fc_env_set("BORG_FASTCDC_NO_AVX512")) +#ifdef FC_KIND_512 + if (__builtin_cpu_supports("avx512f")) return 2; +#endif return __builtin_cpu_supports("avx2"); } @@ -376,36 +366,107 @@ static int fc_simd_available(void) #endif +/* --- kernel selection --------------------------------------------------- */ + +const char *fc_kernel_names(void) +{ +#if defined(__aarch64__) + return "auto, neon, blockwise, scalar"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, avx512, avx2, blockwise, scalar"; +#else + return "auto, blockwise, scalar"; +#endif +} + +int fc_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = FC_K_AUTO; + return FC_KSEL_OK; + } + if (strcmp(name, "scalar") == 0) { + *out_id = FC_K_SCALAR; + return FC_KSEL_OK; + } + if (strcmp(name, "blockwise") == 0) { + *out_id = FC_K_BLOCKWISE; + return FC_KSEL_OK; + } +#if defined(__aarch64__) + if (strcmp(name, "neon") == 0) { + *out_id = FC_K_VECTOR; /* baseline on aarch64, always runnable */ + return FC_KSEL_OK; + } +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "avx2") == 0) { + if (!__builtin_cpu_supports("avx2")) + return FC_KSEL_NOCPU; + *out_id = FC_K_VECTOR; + return FC_KSEL_OK; + } + if (strcmp(name, "avx512") == 0) { +#ifndef FC_KIND_512 + return FC_KSEL_NOTBUILT; /* compiler too old for the target attribute */ +#else + if (!__builtin_cpu_supports("avx512f")) + return FC_KSEL_NOCPU; + *out_id = FC_K_VECTOR512; + return FC_KSEL_OK; +#endif + } +#endif + return FC_KSEL_UNKNOWN; +} + /* --- dispatch ----------------------------------------------------------- */ -/* resolved once (0 = blockwise, 1 = FC_KIND, 2 = FC_KIND_512 where defined); - * a racy double-init writes the same value, so it is benign */ -static int fc_use_simd = -1; +/* the kernel FC_K_AUTO resolves to, worked out once; a racy double-init + * writes the same value, so it is benign */ +static int fc_auto = -1; + +static int fc_auto_kernel(void) +{ + int a; + if (fc_auto < 0) { + a = fc_simd_available(); + fc_auto = (a == 2) ? FC_K_VECTOR512 : (a ? FC_K_VECTOR : FC_K_BLOCKWISE); + } + return fc_auto; +} -int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) +int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel) { - if (force_scalar) + if (kernel == FC_K_AUTO) + kernel = fc_auto_kernel(); + switch (kernel) { + case FC_K_SCALAR: return fc_scan_seq(gear, p, n, fp, mask); - if (fc_use_simd < 0) - fc_use_simd = fc_simd_available(); + case FC_K_BLOCKWISE: + return fc_scan_blockwise(gear, p, n, fp, mask); #ifdef FC_KIND_512 - if (fc_use_simd == 2) + case FC_K_VECTOR512: return fc_scan_simd512(gear, p, n, fp, mask); #endif - if (fc_use_simd) + default: return fc_scan_simd(gear, p, n, fp, mask); - return fc_scan_blockwise(gear, p, n, fp, mask); + } } -const char *fc_kernel_name(int force_scalar) +const char *fc_kernel_name(int kernel) { - if (force_scalar) + if (kernel == FC_K_AUTO) + kernel = fc_auto_kernel(); + switch (kernel) { + case FC_K_SCALAR: return "scalar"; - if (fc_use_simd < 0) - fc_use_simd = fc_simd_available(); + case FC_K_BLOCKWISE: + return "blockwise"; #ifdef FC_KIND_512 - if (fc_use_simd == 2) + case FC_K_VECTOR512: return FC_KIND_512; #endif - return fc_use_simd ? FC_KIND : "blockwise"; + default: + return FC_KIND; + } } diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h index eab87de6da..145127b485 100644 --- a/src/borg/chunkers/fastcdc_impl.h +++ b/src/borg/chunkers/fastcdc_impl.h @@ -11,16 +11,42 @@ #include #include +/* Scan kernel ids, a tier ladder. Which names map onto them depends on the + * build: "neon" exists only on aarch64, "avx2"/"avx512" only on x86-64. */ +#define FC_K_AUTO 0 /* best kernel this CPU can run */ +#define FC_K_SCALAR 1 /* sequential reference loop */ +#define FC_K_BLOCKWISE 2 /* portable 8-lane C */ +#define FC_K_VECTOR 3 /* the platform's vector kernel: neon or avx2 */ +#define FC_K_VECTOR512 4 /* avx512 */ + +/* Results of fc_kernel_select(). */ +#define FC_KSEL_OK 0 +#define FC_KSEL_UNKNOWN 1 /* not a kernel name on this platform */ +#define FC_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define FC_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a kernel name for this build. On FC_KSEL_OK the id is stored in + * *out_id, otherwise *out_id is left alone. The three failures are kept apart + * because they need different fixes: a typo, too old a compiler, or the wrong + * CPU. */ +int fc_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the kernel names this build accepts, for error + * messages. Names a CPU cannot run are still listed. */ +const char *fc_kernel_names(void); + /* Scan up to n positions: for i = 0..n-1 advance fp = (fp << 1) + gear[p[i]] * and test (fp & mask) == 0. * Returns the first i that matched (fp is left at position i), or -1 if none * matched (fp is left at position n-1). - * gear: the keyed 256-entry table. force_scalar != 0 selects the sequential - * reference loop (for tests); otherwise the best kernel for this CPU is used. + * gear: the keyed 256-entry table. kernel is one of FC_K_*; FC_K_AUTO picks + * the best one this CPU can run. Callers are expected to have validated any + * explicit choice with fc_kernel_select() - an unrunnable one falls back to + * FC_K_AUTO rather than crashing. * All kernels return bit-identical results. */ -int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar); +int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel); -/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blockwise" or "scalar". */ -const char *fc_kernel_name(int force_scalar); +/* Name of the kernel selects; for FC_K_AUTO, the auto-selected one. */ +const char *fc_kernel_name(int kernel); #endif diff --git a/src/borg/chunkers/goldilocks_aes.pyx b/src/borg/chunkers/goldilocks_aes.pyx index 50a6e62d28..329577a723 100644 --- a/src/borg/chunkers/goldilocks_aes.pyx +++ b/src/borg/chunkers/goldilocks_aes.pyx @@ -45,7 +45,11 @@ cdef extern from "goldilocks_aes_impl.h": ctypedef struct GL_CTX: pass int GL_TABLES - GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int force_sw) + GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void gl_free(GL_CTX *ctx) const char *gl_kind(const GL_CTX *ctx) uint64_t gl_digest64(const GL_CTX *ctx, const uint8_t *q) @@ -105,6 +109,27 @@ def _derive(bytes key): return aes_key, k, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerGoldilocksAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -128,8 +153,8 @@ cdef class ChunkerGoldilocksAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_GOLDILOCKS_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") self.kernel_str = (gl_kind(self.ctx)).decode("ascii") @@ -176,7 +201,7 @@ def goldilocks_aes_digest64(k, bytes window): for i in range(256): c_tables[t * 256 + i] = tables[t][i] memset(aes_key, 0, 16) - ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1) + ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, PHTE_K_EVP) if ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") d = gl_digest64(ctx, PyBytes_AsString(window)) @@ -205,7 +230,7 @@ def goldilocks_aes_scan_all(k, bytes aes_key, bytes data, mask, bint force_sw=Fa for t in range(len(tables)): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - ctx = gl_new(c_tables, k, (k * k) % _GL_P, PyBytes_AsString(aes_key), 1 if force_sw else 0) + ctx = gl_new(c_tables, k, (k * k) % _GL_P, PyBytes_AsString(aes_key), PHTE_K_EVP if force_sw else PHTE_K_AUTO) if ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") try: diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c index f808a59ebe..f6760ce16f 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.c +++ b/src/borg/chunkers/goldilocks_aes_impl.c @@ -135,7 +135,7 @@ uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q) #include "phte_scan.h" GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2, - const uint8_t aes_key[16], int force_sw) + const uint8_t aes_key[16], int kernel) { GL_CTX *c = calloc(1, sizeof(GL_CTX)); if (c == NULL) @@ -145,7 +145,7 @@ GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2, memcpy(c->in1_tbl, tables + 2 * 256, sizeof(c->in1_tbl)); c->k1 = k1; c->k2 = k2; - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h index e758cae44a..f1f04a3444 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.h +++ b/src/borg/chunkers/goldilocks_aes_impl.h @@ -21,6 +21,8 @@ #include #include +#include "phte_kernel.h" + typedef struct GL_CTX GL_CTX; /* Number of 256-entry rolling tables passed to gl_new, in this order: @@ -37,9 +39,9 @@ typedef struct GL_CTX GL_CTX; * k1: the secret evaluation point K (canonical, 0 <= K < p). * k2: K^2 mod p. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int force_sw); +GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int kernel); void gl_free(GL_CTX *ctx); diff --git a/src/borg/chunkers/kernel_env.py b/src/borg/chunkers/kernel_env.py new file mode 100644 index 0000000000..aacfcc76ae --- /dev/null +++ b/src/borg/chunkers/kernel_env.py @@ -0,0 +1,42 @@ +"""Shared parsing for the BORG_*_KERNEL scan-kernel selection env vars. + +Each SIMD-accelerated chunker picks its inner scan kernel automatically. The +env vars exist to pin it instead - for benchmarking one kernel against +another, and for CI that wants to prove a particular kernel was exercised +rather than hope it was. + +"auto" (the default) takes the best kernel the CPU can run. Any other value is +a demand, not a preference: if that kernel cannot run here, chunker creation +raises ValueError rather than quietly falling back, because a silent fallback +turns a benchmark or a test into a measurement of something else. + +The C side resolves names, since which names exist depends on the build and +the CPU; this module only turns its verdict into an exception. +""" + +import os + +# must match the FC_KSEL_* / BZ_KSEL_* / PHTE_KSEL_* codes in the C headers +KSEL_OK = 0 +KSEL_UNKNOWN = 1 +KSEL_NOTBUILT = 2 +KSEL_NOCPU = 3 + + +def kernel_error(envvar, want, status, valid): + """The ValueError for a kernel request that cannot be honoured.""" + if status == KSEL_UNKNOWN: + detail = "not a kernel of this build" + elif status == KSEL_NOTBUILT: + detail = "not compiled into this build (needs a newer compiler)" + elif status == KSEL_NOCPU: + detail = "this CPU does not support it" + else: + detail = "unavailable" + return ValueError(f"{envvar}={want!r}: {detail}. Valid values: {valid}") + + +def requested_kernel(envvar): + """The kernel name requested via , or None for auto/unset.""" + want = os.environ.get(envvar, "").strip().lower() + return None if want in ("", "auto") else want diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h index ec8ad6a2e4..7c7925e698 100644 --- a/src/borg/chunkers/phte_core.h +++ b/src/borg/chunkers/phte_core.h @@ -13,6 +13,8 @@ #ifndef BORG_PHTE_CORE_H #define BORG_PHTE_CORE_H +#include "phte_kernel.h" + #include #include #include @@ -141,14 +143,7 @@ phte_aes1_ni(const uint8_t rk[11][16], __m128i b) static int phte_hw512_available(void) { - /* BORG_PHTE_NO_VAES caps the AES chunkers at the 128-bit AES-NI path - * (for benchmarking the paths against each other); it is read at - * context creation, ""/"0" mean off. */ - const char *e; - if (!__builtin_cpu_supports("avx512f") || !__builtin_cpu_supports("vaes")) - return 0; - e = getenv("BORG_PHTE_NO_VAES"); - return e == NULL || e[0] == 0 || strcmp(e, "0") == 0; + return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("vaes"); } #endif @@ -160,24 +155,76 @@ static int phte_hw512_available(void) #define PHTE_HAVE_HW512 0 #endif +/* --- kernel selection --------------------------------------------------- + * + * Shared by all three AES chunkers: the scan path lives in phte_scan.h and + * only the rolling hash differs between them, so there is no machine on which + * one of them has a hardware path and another does not. Hence one selector + * (BORG_AES_CHUNKER_KERNEL) rather than one per chunker. */ + +const char *phte_kernel_names(void) +{ +#if defined(__aarch64__) + return "auto, aes-arm64, evp"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, vaes, aes-ni, evp"; +#else + return "auto, evp"; +#endif +} + +int phte_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = PHTE_K_AUTO; + return PHTE_KSEL_OK; + } + if (strcmp(name, "evp") == 0) { + *out_id = PHTE_K_EVP; + return PHTE_KSEL_OK; + } +#if PHTE_HAVE_HW + if (strcmp(name, PHTE_KIND_HW) == 0) { + if (!phte_hw_available()) + return PHTE_KSEL_NOCPU; + *out_id = PHTE_K_HW; + return PHTE_KSEL_OK; + } +#endif +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "vaes") == 0) { +#if !PHTE_HAVE_HW512 + return PHTE_KSEL_NOTBUILT; /* compiler too old for the VAES intrinsics */ +#else + if (!phte_hw512_available()) + return PHTE_KSEL_NOCPU; + *out_id = PHTE_K_HW512; + return PHTE_KSEL_OK; +#endif + } +#endif + return PHTE_KSEL_UNKNOWN; +} + /* --- context base management ------------------------------------------- */ /* Expand the AES key, select the scan path and set up the OpenSSL context. + * kernel is one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns 0 on failure (caller frees its context). */ -static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int force_sw) +static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int kernel) { phte_aes128_expand(aes_key, b->rk); #if PHTE_HAVE_HW - b->use_hw = !force_sw && phte_hw_available(); + b->use_hw = (kernel == PHTE_K_AUTO || kernel == PHTE_K_HW || kernel == PHTE_K_HW512) && phte_hw_available(); #else - (void)force_sw; b->use_hw = 0; #endif #if PHTE_HAVE_HW512 - b->use_hw512 = b->use_hw && phte_hw512_available(); + b->use_hw512 = b->use_hw && kernel != PHTE_K_HW && phte_hw512_available(); #else b->use_hw512 = 0; #endif + (void)kernel; b->evp = EVP_CIPHER_CTX_new(); if (b->evp == NULL) return 0; diff --git a/src/borg/chunkers/phte_kernel.h b/src/borg/chunkers/phte_kernel.h new file mode 100644 index 0000000000..1340c55d34 --- /dev/null +++ b/src/borg/chunkers/phte_kernel.h @@ -0,0 +1,36 @@ +/* Scan-path selection shared by the three UHF-then-PRF ("Chk-PHTE") chunkers + * (toeplitz-aes, rabin-aes, goldilocks-aes). + * + * They share phte_scan.h and differ only in their rolling hash, so the + * available scan paths are a property of the build and the CPU, never of the + * individual chunker: there is no machine where one of them has a hardware + * path and another does not. Hence one selector for all three. */ + +#ifndef BORG_PHTE_KERNEL_H +#define BORG_PHTE_KERNEL_H + +/* Scan path ids, a tier ladder. Which names map onto them depends on the + * build: "vaes" only on x86-64, and the 128-bit hardware path is spelled + * "aes-ni" on x86-64 and "aes-arm64" on aarch64. */ +#define PHTE_K_AUTO 0 /* best path this CPU can run */ +#define PHTE_K_EVP 1 /* portable OpenSSL EVP batch path */ +#define PHTE_K_HW 2 /* 128-bit AES instructions: aes-ni / aes-arm64 */ +#define PHTE_K_HW512 3 /* VAES/AVX-512, 4 AES blocks per instruction */ + +/* Results of phte_kernel_select(). */ +#define PHTE_KSEL_OK 0 +#define PHTE_KSEL_UNKNOWN 1 /* not a scan path name on this platform */ +#define PHTE_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define PHTE_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a scan path name for this build. On PHTE_KSEL_OK the id is stored + * in *out_id, otherwise *out_id is left alone. The three failures are kept + * apart because they need different fixes: a typo, too old a compiler, or the + * wrong CPU. */ +int phte_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the scan path names this build accepts, for error + * messages. Names a CPU cannot run are still listed. */ +const char *phte_kernel_names(void); + +#endif /* BORG_PHTE_KERNEL_H */ diff --git a/src/borg/chunkers/rabin_aes.pyx b/src/borg/chunkers/rabin_aes.pyx index 8198a8f17b..fa55ded145 100644 --- a/src/borg/chunkers/rabin_aes.pyx +++ b/src/borg/chunkers/rabin_aes.pyx @@ -57,7 +57,11 @@ cdef extern from "rabin_aes_impl.h": ctypedef struct RA_CTX: pass int RA_TABLES - RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw) + RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void ra_free(RA_CTX *ctx) const char *ra_kind(const RA_CTX *ctx) uint64_t ra_digest64(const RA_CTX *ctx, const uint8_t *q) @@ -175,6 +179,27 @@ def _derive(bytes key): return aes_key, p, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerRabinAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -198,8 +223,8 @@ cdef class ChunkerRabinAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_RABIN_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = ra_new(c_tables, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = ra_new(c_tables, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up rabin-aes kernel") self.kernel_str = (ra_kind(self.ctx)).decode("ascii") diff --git a/src/borg/chunkers/rabin_aes_impl.c b/src/borg/chunkers/rabin_aes_impl.c index fc21611f8f..af3ab6d1b1 100644 --- a/src/borg/chunkers/rabin_aes_impl.c +++ b/src/borg/chunkers/rabin_aes_impl.c @@ -66,7 +66,7 @@ uint64_t ra_digest64(const RA_CTX *c, const uint8_t *q) #define PH_DIGEST64 ra_digest64 #include "phte_scan.h" -RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int force_sw) +RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int kernel) { RA_CTX *c = calloc(1, sizeof(RA_CTX)); if (c == NULL) @@ -76,7 +76,7 @@ RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16] memcpy(c->w1_tbl, tables + 2 * 256, sizeof(c->w1_tbl)); memcpy(c->out8_tbl, tables + 3 * 256, sizeof(c->out8_tbl)); memcpy(c->out16_tbl, tables + 4 * 256, sizeof(c->out16_tbl)); - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h index 22fe17ad46..5a4fc07e40 100644 --- a/src/borg/chunkers/rabin_aes_impl.h +++ b/src/borg/chunkers/rabin_aes_impl.h @@ -19,6 +19,8 @@ #include #include +#include "phte_kernel.h" + typedef struct RA_CTX RA_CTX; /* Number of 256-entry rolling tables passed to ra_new, in this order: @@ -34,9 +36,9 @@ typedef struct RA_CTX RA_CTX; /* Create a kernel context. * tables: RA_TABLES * 256 uint64 entries, see above. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); +RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel); void ra_free(RA_CTX *ctx); diff --git a/src/borg/chunkers/toeplitz_aes.pyx b/src/borg/chunkers/toeplitz_aes.pyx index 482ed4d3db..8bc5ff423c 100644 --- a/src/borg/chunkers/toeplitz_aes.pyx +++ b/src/borg/chunkers/toeplitz_aes.pyx @@ -50,7 +50,11 @@ cdef extern from "toeplitz_aes_impl.h": ctypedef struct TP_CTX: pass int TP_TABLES - TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw) + TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void tp_free(TP_CTX *ctx) const char *tp_kind(const TP_CTX *ctx) uint64_t tp_digest64(const TP_CTX *ctx, const uint8_t *q) @@ -107,6 +111,27 @@ def _derive(bytes key): return aes_key, t, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerToeplitzAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -130,8 +155,8 @@ cdef class ChunkerToeplitzAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_TOEPLITZ_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = tp_new(c_tables, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = tp_new(c_tables, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up toeplitz-aes kernel") self.kernel_str = (tp_kind(self.ctx)).decode("ascii") @@ -177,7 +202,7 @@ def toeplitz_aes_digest64(bytes key, bytes window): for i in range(256): c_tables[t * 256 + i] = tables[t][i] memset(aes_key_c, 0, 16) - ctx = tp_new(c_tables, aes_key_c, 1) + ctx = tp_new(c_tables, aes_key_c, PHTE_K_EVP) if ctx == NULL: raise MemoryError("Failed to set up toeplitz-aes kernel") d = tp_digest64(ctx, PyBytes_AsString(window)) diff --git a/src/borg/chunkers/toeplitz_aes_impl.c b/src/borg/chunkers/toeplitz_aes_impl.c index 4e9239bddc..36797d60d3 100644 --- a/src/borg/chunkers/toeplitz_aes_impl.c +++ b/src/borg/chunkers/toeplitz_aes_impl.c @@ -84,7 +84,7 @@ uint64_t tp_digest64(const TP_CTX *c, const uint8_t *q) #define PH_DIGEST64 tp_digest64 #include "phte_scan.h" -TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int force_sw) +TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int kernel) { TP_CTX *c = calloc(1, sizeof(TP_CTX)); if (c == NULL) @@ -93,7 +93,7 @@ TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16] memcpy(c->in1_tbl, tables + 1 * 256, sizeof(c->in1_tbl)); memcpy(c->out64_tbl, tables + 2 * 256, sizeof(c->out64_tbl)); memcpy(c->out65_tbl, tables + 3 * 256, sizeof(c->out65_tbl)); - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h index dec05d32e7..5aa3f61f8b 100644 --- a/src/borg/chunkers/toeplitz_aes_impl.h +++ b/src/borg/chunkers/toeplitz_aes_impl.h @@ -24,6 +24,8 @@ #include #include +#include "phte_kernel.h" + typedef struct TP_CTX TP_CTX; /* Number of 256-entry rolling tables passed to tp_new, in this order: @@ -38,9 +40,9 @@ typedef struct TP_CTX TP_CTX; /* Create a kernel context. * tables: TP_TABLES * 256 uint64 entries, see above. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); +TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel); void tp_free(TP_CTX *ctx); diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py index 3a336f80db..eec877000b 100644 --- a/src/borg/testsuite/chunkers/buzhash64_test.py +++ b/src/borg/testsuite/chunkers/buzhash64_test.py @@ -153,13 +153,13 @@ def sizes(chunker): default = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) sizes_default = sizes(default) - os.environ["BORG_BUZHASH64_FORCE_SCALAR"] = "1" + os.environ["BORG_BUZHASH64_KERNEL"] = "scalar" try: forced = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) assert forced.kernel == "scalar" sizes_scalar = sizes(forced) finally: - del os.environ["BORG_BUZHASH64_FORCE_SCALAR"] + del os.environ["BORG_BUZHASH64_KERNEL"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one assert default.kernel in ("avx512", "avx2", "blockwise", "scalar") diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py index 4962bf99eb..8d860c2ab5 100644 --- a/src/borg/testsuite/chunkers/fastcdc_test.py +++ b/src/borg/testsuite/chunkers/fastcdc_test.py @@ -189,13 +189,42 @@ def sizes(chunker): default = ChunkerFastCDC(key0, 10, 16, 14, 2) sizes_default = sizes(default) - os.environ["BORG_FASTCDC_FORCE_SCALAR"] = "1" + os.environ["BORG_FASTCDC_KERNEL"] = "scalar" try: forced = ChunkerFastCDC(key0, 10, 16, 14, 2) assert forced.kernel == "scalar" sizes_scalar = sizes(forced) finally: - del os.environ["BORG_FASTCDC_FORCE_SCALAR"] + del os.environ["BORG_FASTCDC_KERNEL"] assert sizes_default == sizes_scalar # whatever kernel was selected by default, it must be a known one assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar") + + +@pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"]) +def test_kernel_env_rejects_unusable(envvar, monkeypatch): + # A kernel that cannot run here must fail loudly instead of silently + # falling back - a silent fallback would turn a benchmark, or a CI job that + # means to pin one kernel, into a measurement of a different one. + from ...chunkers import ChunkerBuzHash64, ChunkerToeplitzAES + + make = { + "BORG_FASTCDC_KERNEL": lambda k: ChunkerFastCDC(k, 10, 16, 14, 2), + "BORG_BUZHASH64_KERNEL": lambda k: ChunkerBuzHash64(k, 10, 16, 14, 4095, 2), + "BORG_AES_CHUNKER_KERNEL": lambda k: ChunkerToeplitzAES(k, 10, 16, 14, 2), + }[envvar] + key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") + + monkeypatch.setenv(envvar, "no-such-kernel") + with pytest.raises(ValueError, match="no-such-kernel"): + make(key0) + + # "auto" and an unset var both mean "pick the best one", and always work + monkeypatch.setenv(envvar, "auto") + auto = make(key0).kernel + monkeypatch.delenv(envvar) + assert make(key0).kernel == auto + + # asking for whatever auto-selection picked must yield exactly that + monkeypatch.setenv(envvar, auto) + assert make(key0).kernel == auto diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py index 2dc2cac3e6..50b54220f5 100644 --- a/src/borg/testsuite/chunkers/phte_chunkers_test.py +++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py @@ -23,13 +23,16 @@ # from os.urandom(32) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") -# (chunker class, algo name, default params constant, env var forcing the portable kernel) +# all three share one scan-path selector, see phte_kernel.h +KERNEL_ENV = "BORG_AES_CHUNKER_KERNEL" + +# (chunker class, algo name, default params constant) CHUNKERS = [ - (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS, "BORG_RABIN_AES_FORCE_EVP"), - (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS, "BORG_GOLDILOCKS_AES_FORCE_EVP"), - (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS, "BORG_TOEPLITZ_AES_FORCE_EVP"), + (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS), + (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS), + (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS), ] -IDS = [algo for _, algo, _, _ in CHUNKERS] +IDS = [algo for _, algo, _ in CHUNKERS] def H(data): @@ -44,7 +47,7 @@ def chunker_spec(request): def test_kernels_identical(chunker_spec): # the OpenSSL EVP batch path and the AES hardware instruction path (if available # on this platform) must produce identical cut points. - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(4 * 1024 * 1024) def sizes(chunker): @@ -52,20 +55,20 @@ def sizes(chunker): default = cls(key0, 10, 16, 14, 2) sizes_default = sizes(default) - os.environ[env_var] = "1" + os.environ[KERNEL_ENV] = "evp" try: forced = cls(key0, 10, 16, 14, 2) assert forced.kernel == "evp" sizes_evp = sizes(forced) finally: - del os.environ[env_var] + del os.environ[KERNEL_ENV] assert sizes_default == sizes_evp # whatever kernel was selected by default, it must be a known one assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp") def test_chunksize_distribution(chunker_spec): - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(1048576) min_exp, max_exp, mask, nc_level = 10, 16, 14, 2 # chunk size target 16 KiB, clip at 1 KiB and 64 KiB chunker = cls(key0, min_exp, max_exp, mask, nc_level) @@ -95,7 +98,7 @@ def test_chunksize_distribution(chunker_spec): def test_shift_resilience(chunker_spec): # content-defined cuts must survive a prefix insertion (this also validates that the # rolling digest update and the per-chunk window warm-up agree with each other). - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(4 * 1024 * 1024) def chunk_hashes(data): @@ -110,7 +113,7 @@ def chunk_hashes(data): def test_get_chunker(chunker_spec): # without a key, get_chunker uses an all-zero key; chunking must still work and be deterministic - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(2 * 1024 * 1024) a = cf_expand(get_chunker(*params, key=None).chunkify(BytesIO(data))) b = cf_expand(get_chunker(algo, 19, 23, 21, 2, key=None).chunkify(BytesIO(data))) @@ -123,7 +126,7 @@ def test_params_parsing(chunker_spec): from ...helpers import ChunkerParams - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec # , chunk_min, chunk_max, chunk_mask, nc_level (no window field) assert ChunkerParams(f"{algo},19,23,21,2") == (algo, 19, 23, 21, 2) @@ -149,7 +152,7 @@ def test_params_parsing(chunker_spec): @pytest.mark.parametrize("worker", range(os.cpu_count() or 1)) def test_fuzz(chunker_spec, worker): # Fuzz with random and uniform data of misc. sizes and misc keys. - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec # decompose _PARAMS = (algo, min_exp, max_exp, mask_bits, nc_level) params_algo, min_exp, max_exp, mask_bits, nc_level = params From 2163abfe193e51daad8c91720edb859d2db61e0a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 13:49:40 +0200 Subject: [PATCH 12/13] buzhash64: keep the NEON kernel, selectable but not auto-selected Restores the NEON kernel removed in 193a26f4e, with BZ_K_AUTO still resolving to blockwise on aarch64. BORG_BUZHASH64_KERNEL=neon selects it. Removing it outright over-generalized the measurement. It loses to blockwise by ~9% on an Apple M3 Pro P-core, but the reason is core width: the 16 table lookups per block must happen in general registers (NEON has no gather), so the vector form only ADDS the move to the SIMD side plus a cross-lane reduce before the loop branch resolves - all it saves is the 8-lane test, three cheap ops per lane, which Apple's very wide scalar ALUs retire at more than one lane per cycle anyway. That measurement comes from the widest scalar ARM core in existence. Neoverse (Graviton, Ampere), Cortex-A7x and friends are 3-4 wide with comparatively healthy NEON, which is exactly where this should get competitive - and on this machine's much narrower E-cores the 9% gap already collapses into measurement noise. Deciding for all of aarch64 from one unrepresentative core was the wrong call; keeping the kernel costs a dispatch branch and makes flipping the default a one-line change once someone benchmarks such hardware. The cross-kernel identity tests now iterate over every kernel the platform accepts instead of just default-vs-reference, skipping the ones this build or CPU cannot run. That is what keeps a selectable but non-default kernel from bit-rotting, and it also covers whichever tier a given CI machine happens to have. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- docs/changes.rst | 6 +- src/borg/chunkers/buzhash64.pyx | 5 +- src/borg/chunkers/buzhash64_impl.c | 99 +++++++++++++++---- src/borg/chunkers/buzhash64_impl.h | 8 +- src/borg/testsuite/chunkers/buzhash64_test.py | 38 ++++--- src/borg/testsuite/chunkers/fastcdc_test.py | 40 +++++--- .../testsuite/chunkers/phte_chunkers_test.py | 39 +++++--- 7 files changed, 162 insertions(+), 73 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 28872ef66b..592974bc66 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -189,9 +189,11 @@ New features: - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043: - AVX-512 / AVX2 on x86-64 (Intel / AMD) - - fastcdc also NEON on aarch64 (e.g. Apple Silicon); buzhash64 is faster - blockwise there, so it has no vector kernel on aarch64 + - NEON on aarch64 (e.g. Apple Silicon); the default for fastcdc, but not + for buzhash64, which is faster blockwise on the cores measured so far - blockwise scalar elsewhere + - the kernel can be pinned via BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL + / BORG_AES_CHUNKER_KERNEL - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987 - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64 diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx index 26c94de41e..cacee641be 100644 --- a/src/borg/chunkers/buzhash64.pyx +++ b/src/borg/chunkers/buzhash64.pyx @@ -174,7 +174,10 @@ cdef class ChunkerBuzHash64(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'. + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'. + + 'neon' exists on aarch64 but is never auto-selected (it loses to + 'blockwise' there); BORG_BUZHASH64_KERNEL=neon selects it. With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the requested kernel - creating the chunker fails otherwise. diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index 4284a6d6d3..1f24d4eedd 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -22,8 +22,8 @@ * also removes one rotate per byte from the sequential path. * * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blockwise - * scalar everywhere else including aarch64 (which has no vector kernel, see - * the note where it would go), sequential when forced. + * scalar everywhere else, including aarch64 by default - its NEON kernel is + * selectable by name but not auto-selected, see the note above it. * All kernels return bit-identical results. */ #include @@ -114,29 +114,79 @@ static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot, return j; } -/* --- aarch64: no vector kernel ------------------------------------------ +/* --- NEON (aarch64, selectable but not the default) --------------------- * - * There was a NEON kernel here doing the 8-lane test in vector registers. - * It is gone because it lost to the blockwise scalar kernel: measured on an - * Apple M3 Pro (48 MiB, window 4095), blockwise 2590 MB/s vs NEON 2360 MB/s, - * the same ~9% gap at every mask size from 17 to 23 bits. + * This kernel is NOT auto-selected: on an Apple M3 Pro it loses to the + * blockwise scalar one (2590 vs 2360 MB/s, the same ~9% gap at every mask + * size from 17 to 23 bits), so bz64_simd_available() returns 0 here and + * BZ_K_AUTO resolves to blockwise. BORG_BUZHASH64_KERNEL=neon selects it. * - * The reason is that nothing here needs a vector register in the first - * place. clang keeps the block's eight prefix XORs in general registers and - * the scalar test is xor/and/compare per lane, which the wide scalar ALUs - * retire at more than one lane per cycle; the vector form has to move those - * eight values across to the SIMD side and then reduce the 8-lane result - * back to one condition flag (umaxv, ~4 cycles) before the loop branch can - * resolve. The move-and-reduce round trip costs more than the parallel test - * saves. fastcdc keeps its NEON kernel because its test is add/and/compare - * with per-lane shifted masks - enough extra per-lane work to pay for the - * trip - and there it wins by 2x. + * Why it loses there: the 16 table lookups per block have to happen in + * general registers (NEON has no gather), so the vector form only ADDS the + * move to the SIMD side plus a cross-lane reduce (umaxv) before the loop + * branch can resolve - all it saves is the 8-lane test, three cheap ops per + * lane, which Apple's very wide scalar ALUs retire at more than one lane per + * cycle anyway. fastcdc keeps NEON as its default because its per-lane test + * work is larger (add plus per-lane shifted masks), enough to pay for the + * trip; there it wins by 2x. * - * So aarch64 falls through to the blockwise scalar kernel below. */ + * It is kept because that reasoning is about core width, and the measurement + * comes from the widest scalar ARM core there is. Neoverse (Graviton, + * Ampere), Cortex-A7x and friends are 3-4 wide on the scalar side with + * comparatively healthy NEON, which is exactly where this should get + * competitive - on this machine's much narrower E-cores the 9% gap already + * collapses into measurement noise. If you have such hardware, compare + * BORG_BUZHASH64_KERNEL=neon against =blockwise and please report; flipping + * the default is then a one-line change here. */ + +#if defined(__aarch64__) +#define BZ_KIND "neon" + +#include + +static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, + const uint8_t *pr, const uint8_t *pa, + size_t n, uint64_t *sum_io, uint64_t mask) +{ + uint64_t sum = *sum_io; + uint64_t s[8]; + size_t j = 0; + uint64x2_t M12 = {BZ_ROTL(mask, 7), BZ_ROTL(mask, 6)}; + uint64x2_t M34 = {BZ_ROTL(mask, 5), BZ_ROTL(mask, 4)}; + uint64x2_t M56 = {BZ_ROTL(mask, 3), BZ_ROTL(mask, 2)}; + uint64x2_t M78 = {BZ_ROTL(mask, 1), mask}; + + while (j + 8 <= n && (sum & mask) != 0) { + bz64_block_prefix(T, Trot, pr + j, pa + j, s); + uint64_t c = BZ_ROTL(sum, 8); + uint64x2_t Cv = vdupq_n_u64(c); + uint64x2_t z12 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[0], s[1]}), M12)); + uint64x2_t z34 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[2], s[3]}), M34)); + uint64x2_t z56 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[4], s[5]}), M56)); + uint64x2_t z78 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[6], s[7]}), M78)); + uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78)); + if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ s[7]; + j += 8; + } + if (j < n) + j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); + *sum_io = sum; + return j; +} + +static int bz64_simd_available(void) +{ + return 0; /* see above: selectable by name, but never auto-selected */ +} /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */ -#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) #define BZ_KIND "avx2" #include @@ -328,7 +378,9 @@ static int bz64_simd_available(void) const char *bz64_kernel_names(void) { -#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) +#if defined(__aarch64__) + return "auto, neon, blockwise, scalar"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) return "auto, avx512, avx2, blockwise, scalar"; #else return "auto, blockwise, scalar"; @@ -349,7 +401,12 @@ int bz64_kernel_select(const char *name, int *out_id) *out_id = BZ_K_BLOCKWISE; return BZ_KSEL_OK; } -#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) +#if defined(__aarch64__) + if (strcmp(name, "neon") == 0) { + *out_id = BZ_K_VECTOR; /* baseline on aarch64, always runnable */ + return BZ_KSEL_OK; + } +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) if (strcmp(name, "avx2") == 0) { if (!__builtin_cpu_supports("avx2")) return BZ_KSEL_NOCPU; diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h index f6dbfdd900..c75496ce35 100644 --- a/src/borg/chunkers/buzhash64_impl.h +++ b/src/borg/chunkers/buzhash64_impl.h @@ -27,7 +27,7 @@ #define BZ_K_AUTO 0 /* best kernel this CPU can run */ #define BZ_K_SCALAR 1 /* sequential reference loop */ #define BZ_K_BLOCKWISE 2 /* portable 8-lane C */ -#define BZ_K_VECTOR 3 /* avx2 (there is no aarch64 vector kernel) */ +#define BZ_K_VECTOR 3 /* neon / avx2 (neon is not auto-selected) */ #define BZ_K_VECTOR512 4 /* avx512 */ /* Results of bz64_kernel_select(). */ @@ -46,9 +46,9 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, size_t n, uint64_t *sum, uint64_t mask, int kernel); -/* Name of the kernel selects: "avx512", "avx2", "blockwise" or - * "scalar" (there is no vector kernel on aarch64, see buzhash64_impl.c). - * For BZ_K_AUTO, the auto-selected one. */ +/* Name of the kernel selects: "neon", "avx512", "avx2", "blockwise" + * or "scalar"; for BZ_K_AUTO, the auto-selected one. Note "neon" is never + * auto-selected, see buzhash64_impl.c. */ const char *bz64_kernel_name(int kernel); #endif diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py index eec877000b..0848e8f9be 100644 --- a/src/borg/testsuite/chunkers/buzhash64_test.py +++ b/src/borg/testsuite/chunkers/buzhash64_test.py @@ -142,24 +142,32 @@ def rnd_key(): assert reconstructed == data -def test_buzhash64_kernels_identical(): - # the SIMD scan kernel (avx512/avx2/blockwise, auto-selected) and the plain sequential - # loop must produce identical cut points. +ALL_BUZHASH64_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar") + + +def test_buzhash64_kernels_identical(monkeypatch): + # Every scan kernel this platform accepts must produce identical cut points; + # see the fastcdc counterpart. On aarch64 this is what keeps the NEON kernel + # verified - it is selectable but never auto-selected. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) - sizes_default = sizes(default) - os.environ["BORG_BUZHASH64_KERNEL"] = "scalar" - try: - forced = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) - assert forced.kernel == "scalar" - sizes_scalar = sizes(forced) - finally: - del os.environ["BORG_BUZHASH64_KERNEL"] - assert sizes_default == sizes_scalar - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("avx512", "avx2", "blockwise", "scalar") + monkeypatch.delenv("BORG_BUZHASH64_KERNEL", raising=False) + reference = sizes(ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)) + + tested = [] + for name in ALL_BUZHASH64_KERNELS: + monkeypatch.setenv("BORG_BUZHASH64_KERNEL", name) + try: + chunker = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one" + tested.append(name) + + assert "blockwise" in tested and "scalar" in tested diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py index 8d860c2ab5..9e19870ad9 100644 --- a/src/borg/testsuite/chunkers/fastcdc_test.py +++ b/src/borg/testsuite/chunkers/fastcdc_test.py @@ -178,27 +178,37 @@ def rnd_key(): assert b"".join(parts) == data -def test_fastcdc_kernels_identical(): - # the SIMD scan kernel (neon/avx512/avx2/blockwise, auto-selected) and the - # plain sequential Gear loop must produce identical cut points. +ALL_FASTCDC_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar") + + +def test_fastcdc_kernels_identical(monkeypatch): + # Every scan kernel this platform accepts must produce identical cut points. + # Kernels this build/CPU cannot run raise and are skipped, so the same test + # covers whatever tier the machine happens to have - including kernels that + # exist but are not auto-selected, which nothing else would exercise. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = ChunkerFastCDC(key0, 10, 16, 14, 2) - sizes_default = sizes(default) - os.environ["BORG_FASTCDC_KERNEL"] = "scalar" - try: - forced = ChunkerFastCDC(key0, 10, 16, 14, 2) - assert forced.kernel == "scalar" - sizes_scalar = sizes(forced) - finally: - del os.environ["BORG_FASTCDC_KERNEL"] - assert sizes_default == sizes_scalar - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar") + monkeypatch.delenv("BORG_FASTCDC_KERNEL", raising=False) + reference = sizes(ChunkerFastCDC(key0, 10, 16, 14, 2)) + + tested = [] + for name in ALL_FASTCDC_KERNELS: + monkeypatch.setenv("BORG_FASTCDC_KERNEL", name) + try: + chunker = ChunkerFastCDC(key0, 10, 16, 14, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one" + tested.append(name) + + # the portable kernels exist everywhere; if they were skipped, the loop is broken + assert "blockwise" in tested and "scalar" in tested @pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"]) diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py index 50b54220f5..19b89ffac7 100644 --- a/src/borg/testsuite/chunkers/phte_chunkers_test.py +++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py @@ -44,27 +44,36 @@ def chunker_spec(request): return request.param -def test_kernels_identical(chunker_spec): - # the OpenSSL EVP batch path and the AES hardware instruction path (if available - # on this platform) must produce identical cut points. +ALL_AES_KERNELS = ("auto", "vaes", "aes-ni", "aes-arm64", "evp") + + +def test_kernels_identical(chunker_spec, monkeypatch): + # Every scan path this platform accepts - OpenSSL EVP, the 128-bit AES + # instructions, VAES - must produce identical cut points. Paths this + # build/CPU cannot run raise and are skipped. cls, algo, params = chunker_spec data = os.urandom(4 * 1024 * 1024) def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = cls(key0, 10, 16, 14, 2) - sizes_default = sizes(default) - os.environ[KERNEL_ENV] = "evp" - try: - forced = cls(key0, 10, 16, 14, 2) - assert forced.kernel == "evp" - sizes_evp = sizes(forced) - finally: - del os.environ[KERNEL_ENV] - assert sizes_default == sizes_evp - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp") + monkeypatch.delenv(KERNEL_ENV, raising=False) + reference = sizes(cls(key0, 10, 16, 14, 2)) + + tested = [] + for name in ALL_AES_KERNELS: + monkeypatch.setenv(KERNEL_ENV, name) + try: + chunker = cls(key0, 10, 16, 14, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"path {name} disagrees with the default one" + tested.append(name) + + # the portable OpenSSL path exists everywhere + assert "evp" in tested def test_chunksize_distribution(chunker_spec): From f71c3def0df2fea9226289221347b7164b33bb6e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 6 Aug 2026 14:19:54 +0200 Subject: [PATCH 13/13] build: bundle borg.chunkers.kernel_env into the binary The binary build broke with File "src/borg/chunkers/buzhash64.pyx", line 26, in init borg.chunkers.buzhash64 from .kernel_env import kernel_error, requested_kernel ModuleNotFoundError: No module named 'borg.chunkers.kernel_env' kernel_env is imported only from .pyx files, and the chunkers are Cython extensions, so PyInstaller's static analysis of the Python sources cannot see the import at all and never collects the module. The spec already lists borg.chunkers.base and .phte_chunker as hiddenimports for the same reason (they are cimported at C level); this adds kernel_env and widens the comment, since the rule is "anything a compiled chunker pulls in has to be listed here", not just the cimported base classes. Verified both ways with a local pyinstaller build: with the entry the binary runs "borg benchmark cpu" through all chunkers, and rebuilding with the entry removed reproduces the traceback above exactly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm --- scripts/borg.exe.spec | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/borg.exe.spec b/scripts/borg.exe.spec index a705db35b6..e1155cad69 100644 --- a/scripts/borg.exe.spec +++ b/scripts/borg.exe.spec @@ -14,12 +14,16 @@ if is_win32: else: hiddenimports = ['borg.platform.posix', 'borghash', 'rich._unicode_data.unicode17-0-0'] -# The chunkers cimport their shared base classes (ChunkerBase, and ChunkerPHTE -# for the *-aes chunkers), so they import those modules at C level when -# initializing - PyInstaller can not see this by static analysis of the Python -# sources. +# Anything a compiled chunker module pulls in has to be listed here: the +# chunkers are Cython extensions, so PyInstaller's static analysis of the +# Python sources cannot see their imports at all. That covers both the base +# classes they cimport (ChunkerBase, and ChunkerPHTE for the *-aes chunkers), +# which are imported at C level during module init, and plain Python modules +# imported from a .pyx, such as kernel_env - it is imported ONLY from .pyx +# files, so nothing else would drag it in. hiddenimports.append('borg.chunkers.base') hiddenimports.append('borg.chunkers.phte_chunker') +hiddenimports.append('borg.chunkers.kernel_env') block_cipher = None