diff --git a/docs/changes.rst b/docs/changes.rst index a76c3ccb4b..592974bc66 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -186,13 +186,18 @@ New features: - Chunkers: - fastcdc is the new and faster default chunker, #9957 - - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034: - - - NEON on aarch64 (e.g. Apple Silicon) - - AVX2 on x86-64 (Intel / AMD) - - blocked scalar elsewhere + - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043: + + - AVX-512 / AVX2 on x86-64 (Intel / AMD) + - NEON on aarch64 (e.g. Apple Silicon); the default for fastcdc, but not + for buzhash64, which is faster blockwise on the cores measured so far + - blockwise scalar elsewhere + - the kernel can be pinned via BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL + / BORG_AES_CHUNKER_KERNEL - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987 + - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64 + (4 AES blocks per instruction), #10043 - zero-copy fill and lazy buffer compaction optimizations - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice replacement for `borg mount` in some use cases, #9942 diff --git a/scripts/borg.exe.spec b/scripts/borg.exe.spec index a705db35b6..e1155cad69 100644 --- a/scripts/borg.exe.spec +++ b/scripts/borg.exe.spec @@ -14,12 +14,16 @@ if is_win32: else: hiddenimports = ['borg.platform.posix', 'borghash', 'rich._unicode_data.unicode17-0-0'] -# The chunkers cimport their shared base classes (ChunkerBase, and ChunkerPHTE -# for the *-aes chunkers), so they import those modules at C level when -# initializing - PyInstaller can not see this by static analysis of the Python -# sources. +# Anything a compiled chunker module pulls in has to be listed here: the +# chunkers are Cython extensions, so PyInstaller's static analysis of the +# Python sources cannot see their imports at all. That covers both the base +# classes they cimport (ChunkerBase, and ChunkerPHTE for the *-aes chunkers), +# which are imported at C level during module init, and plain Python modules +# imported from a .pyx, such as kernel_env - it is imported ONLY from .pyx +# files, so nothing else would drag it in. hiddenimports.append('borg.chunkers.base') hiddenimports.append('borg.chunkers.phte_chunker') +hiddenimports.append('borg.chunkers.kernel_env') block_cipher = None diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx index 610e80f83f..cacee641be 100644 --- a/src/borg/chunkers/buzhash64.pyx +++ b/src/borg/chunkers/buzhash64.pyx @@ -17,8 +17,13 @@ from .base cimport ChunkerBase cdef extern from "buzhash64_impl.h": size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar) nogil - const char *bz64_kernel_name(int force_scalar) + size_t n, uint64_t *sum, uint64_t mask, int kernel) nogil + const char *bz64_kernel_name(int kernel) + int bz64_kernel_select(const char *name, int *out_id) + const char *bz64_kernel_names() + int BZ_K_AUTO + +from .kernel_env import kernel_error, requested_kernel # Cyclic polynomial / buzhash # @@ -101,6 +106,20 @@ cdef uint64_t _buzhash64_update(uint64_t sum, unsigned char remove, unsigned cha return BARREL_SHIFT64(sum, 1) ^ BARREL_SHIFT64(h[remove], lenmod) ^ h[add] +cdef int _select_kernel() except -1: + """Resolve BORG_BUZHASH64_KERNEL to a kernel id, raising if it cannot be honoured.""" + cdef int kid = BZ_K_AUTO + cdef int rc + want = requested_kernel("BORG_BUZHASH64_KERNEL") + if want is None: + return BZ_K_AUTO + rc = bz64_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_BUZHASH64_KERNEL", want, rc, + (bz64_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerBuzHash64(ChunkerBase): """ Content-Defined Chunker, variable chunk sizes. @@ -118,7 +137,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): """ cdef uint64_t* table cdef uint64_t* table_rot - cdef int force_scalar + cdef int kernel_id cdef size_t window_size def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int hash_window_size, int nc_level=0, size_t normal_size=0, bint sparse=False): @@ -139,7 +158,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): lenmod = hash_window_size & 0x3f for i_rot in range(256): self.table_rot[i_rot] = BARREL_SHIFT64(self.table[i_rot], lenmod) - self.force_scalar = 1 if os.environ.get("BORG_BUZHASH64_FORCE_SCALAR", "") not in ("", "0") else 0 + self.kernel_id = _select_kernel() # buzhash64 output is uniform, so contiguous low-bit masks are used (high_masks=False) self._setup_common("buzhash64", chunk_min_exp, chunk_max_exp, hash_mask_bits, nc_level, normal_size, False, sparse) @@ -155,8 +174,15 @@ cdef class ChunkerBuzHash64(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'.""" - return (bz64_kernel_name(self.force_scalar)).decode("ascii") + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'. + + 'neon' exists on aarch64 but is never auto-selected (it loses to + 'blockwise' there); BORG_BUZHASH64_KERNEL=neon selects it. + + With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the + requested kernel - creating the chunker fails otherwise. + """ + return (bz64_kernel_name(self.kernel_id)).decode("ascii") cdef object process(self): """Process the chunker's buffer and return the next chunk.""" @@ -169,7 +195,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): cdef uint8_t* stop_at cdef uint8_t* nc_stop cdef size_t did_bytes, span - cdef int force_scalar = self.force_scalar + cdef int kernel_id = self.kernel_id if self.done: if self.bytes_read == self.bytes_yielded: @@ -229,7 +255,7 @@ cdef class ChunkerBuzHash64(ChunkerBase): span = stop_at - p with nogil: did_bytes = bz64_scan(self.table, self.table_rot, p, p + window_size, - span, &sum, mask, force_scalar) + span, &sum, mask, kernel_id) self.position += did_bytes self.remaining -= did_bytes diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c index ca43d60a91..1f24d4eedd 100644 --- a/src/borg/chunkers/buzhash64_impl.c +++ b/src/borg/chunkers/buzhash64_impl.c @@ -21,15 +21,12 @@ * Trot[b] = ROTL(T[b], window_size % 64) is precomputed by the caller, which * also removes one rotate per byte from the sequential path. * - * Kernel dispatch mirrors fastcdc_impl.c: NEON on aarch64 (baseline there), - * AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere, sequential - * when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit - * mask): sequential 1150, blocked scalar 2340, NEON 2260 MB/s - NEON and - * blocked are a tie here (throughput and energy, within noise), because the - * XOR/AND/compare test also runs well on the scalar ALUs; the symmetric - * dispatch is kept for consistency across the SIMD chunker kernels. + * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blockwise + * scalar everywhere else, including aarch64 by default - its NEON kernel is + * selectable by name but not auto-selected, see the note above it. * All kernels return bit-identical results. */ +#include #include #include "buzhash64_impl.h" @@ -85,9 +82,9 @@ static inline void bz64_block_prefix(const uint64_t *T, const uint64_t *Trot, s[7] = s8; } -/* --- blocked scalar (the default kernel) --------------------------------- */ +/* --- blockwise scalar (the default kernel) --------------------------------- */ -static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, +static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { @@ -117,7 +114,30 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot, return j; } -/* --- NEON (aarch64 baseline, always available) -------------------------- */ +/* --- NEON (aarch64, selectable but not the default) --------------------- + * + * This kernel is NOT auto-selected: on an Apple M3 Pro it loses to the + * blockwise scalar one (2590 vs 2360 MB/s, the same ~9% gap at every mask + * size from 17 to 23 bits), so bz64_simd_available() returns 0 here and + * BZ_K_AUTO resolves to blockwise. BORG_BUZHASH64_KERNEL=neon selects it. + * + * Why it loses there: the 16 table lookups per block have to happen in + * general registers (NEON has no gather), so the vector form only ADDS the + * move to the SIMD side plus a cross-lane reduce (umaxv) before the loop + * branch can resolve - all it saves is the 8-lane test, three cheap ops per + * lane, which Apple's very wide scalar ALUs retire at more than one lane per + * cycle anyway. fastcdc keeps NEON as its default because its per-lane test + * work is larger (add plus per-lane shifted masks), enough to pay for the + * trip; there it wins by 2x. + * + * It is kept because that reasoning is about core width, and the measurement + * comes from the widest scalar ARM core there is. Neoverse (Graviton, + * Ampere), Cortex-A7x and friends are 3-4 wide on the scalar side with + * comparatively healthy NEON, which is exactly where this should get + * competitive - on this machine's much narrower E-cores the 9% gap already + * collapses into measurement noise. If you have such hardware, compare + * BORG_BUZHASH64_KERNEL=neon against =blockwise and please report; flipping + * the default is then a one-line change here. */ #if defined(__aarch64__) #define BZ_KIND "neon" @@ -161,7 +181,7 @@ static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, static int bz64_simd_available(void) { - return 1; + return 0; /* see above: selectable by name, but never auto-selected */ } /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */ @@ -171,39 +191,158 @@ static int bz64_simd_available(void) #include +/* Load the block's 8 per-byte deltas D_t = Trot[out_t] ^ T[in_t] (via two + * 8-byte data loads), without the rotations and the prefix XOR: the vector + * kernels do those in the vector domain. Endianness-independent. */ +static inline void bz64_block_delta(const uint64_t *T, const uint64_t *Trot, + const uint8_t *pr, const uint8_t *pa, uint64_t d[8]) +{ + uint64_t wr, wa; + memcpy(&wr, pr, 8); + memcpy(&wa, pa, 8); +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wr = __builtin_bswap64(wr); + wa = __builtin_bswap64(wa); +#endif + d[0] = Trot[(uint8_t)wr] ^ T[(uint8_t)wa]; + d[1] = Trot[(uint8_t)(wr >> 8)] ^ T[(uint8_t)(wa >> 8)]; + d[2] = Trot[(uint8_t)(wr >> 16)] ^ T[(uint8_t)(wa >> 16)]; + d[3] = Trot[(uint8_t)(wr >> 24)] ^ T[(uint8_t)(wa >> 24)]; + d[4] = Trot[(uint8_t)(wr >> 32)] ^ T[(uint8_t)(wa >> 32)]; + d[5] = Trot[(uint8_t)(wr >> 40)] ^ T[(uint8_t)(wa >> 40)]; + d[6] = Trot[(uint8_t)(wr >> 48)] ^ T[(uint8_t)(wa >> 48)]; + d[7] = Trot[(uint8_t)(wr >> 56)] ^ T[(uint8_t)(wa >> 56)]; +} + +/* AVX2 has no variable rotate, so ROTL(v, k) is built from a variable left + * and right shift. The k = 0 lane needs a 64-bit right shift, which + * vpsrlvq defines as zero - exactly what (v << 0) | 0 = v needs. */ +__attribute__((target("avx2"))) static inline __m256i bz64_rotl4(__m256i v, __m256i k, __m256i kc) +{ + return _mm256_or_si256(_mm256_sllv_epi64(v, k), _mm256_srlv_epi64(v, kc)); +} + +/* The 8-lane test in two 256-bit vectors, with the aligned domain built in + * the vector domain: per-lane rotations, then a prefix XOR inside each 4-lane + * half (two vpermq+vpxor steps) plus a broadcast of the low half's total into + * the high half. + * + * Only the 2x8 table lookups stay scalar, and they are done one block ahead + * into a double buffer: a 32-byte vector load placed right after the eight + * 8-byte scalar stores that filled it cannot use store-to-load forwarding and + * stalls, twice per block. Loading block j+1 while block j is tested puts a + * full loop body between the stores and the load, so the stall disappears. */ __attribute__((target("avx2"))) static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { uint64_t sum = *sum_io; - uint64_t s[8]; + uint64_t d[16]; /* double buffer: block being tested + block being loaded */ size_t j = 0; + int cur = 0; /* _mm256_set_epi64x takes arguments high lane first */ - __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), - (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); - __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1), - (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3)); - __m256i zero = _mm256_setzero_si256(); + const __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), + (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); + const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1), + (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3)); + const __m256i KL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 rotate by 7..4 */ + const __m256i KH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 rotate by 3..0 */ + const __m256i KLC = _mm256_set_epi64x(60, 59, 58, 57); /* 64 - KL */ + const __m256i KHC = _mm256_set_epi64x(64, 63, 62, 61); /* 64 - KH */ + const __m256i zero = _mm256_setzero_si256(); - while (j + 8 <= n && (sum & mask) != 0) { - bz64_block_prefix(T, Trot, pr + j, pa + j, s); - uint64_t c = BZ_ROTL(sum, 8); - __m256i C = _mm256_set1_epi64x((long long)c); - __m256i H1 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[0])); - __m256i H2 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[4])); - __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); - __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); - __m256i any = _mm256_or_si256(z1, z2); - if (!_mm256_testz_si256(any, any)) { - size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ - *sum_io = sum; - return j + r; + if (n >= 16 && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr, pa, d); + while (j + 16 <= n && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */ + __m256i lo = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur)), KL, KLC); + __m256i hi = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur + 4)), KH, KHC); + /* prefix XOR inside each half: rotate the lanes up by 1 resp. 2 + * (vpermq), zero the lanes rotated in, XOR */ + lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03)); + lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F)); + hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03)); + hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F)); + hi = _mm256_xor_si256(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */ + uint64_t c = BZ_ROTL(sum, 8); + __m256i C = _mm256_set1_epi64x((long long)c); + __m256i H1 = _mm256_xor_si256(C, lo); + __m256i H2 = _mm256_xor_si256(C, hi); + __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); + __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); + __m256i any = _mm256_or_si256(z1, z2); + if (!_mm256_testz_si256(any, any)) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128( + _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */ + j += 8; + cur ^= 8; } - sum = c ^ s[7]; - j += 8; } - if (j < n) + if (j < n) /* up to 15 bytes here (one block more than the scalar kernels) */ + j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); + *sum_io = sum; + return j; +} + +/* --- AVX-512 (x86-64, runtime detected) ---------------------------------- */ + +#define BZ_KIND_512 "avx512" + +/* The same 8-lane test as the AVX2 kernel, but the aligned domain fits in one + * 512-bit vector: vprolvq applies the per-lane rotations (lane k by 7-k) in + * one instruction, three valignq+vpxorq steps turn the rotated deltas into + * the prefix XORs s[0..7] in one pass instead of two halves plus a carry, and + * vptestnmq fuses the AND and the ==0 test into a mask register. + * + * The table lookups are pipelined one block ahead exactly as in the AVX2 + * kernel above, for the same store-to-load forwarding reason. */ +__attribute__((target("avx512f"))) static size_t +bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot, + const uint8_t *pr, const uint8_t *pa, + size_t n, uint64_t *sum_io, uint64_t mask) +{ + uint64_t sum = *sum_io; + uint64_t d[16]; /* double buffer: block being tested + block being loaded */ + size_t j = 0; + int cur = 0; + /* _mm512_set_epi64 takes arguments high lane first: lane k = ROTL(mask, 7-k) */ + const __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1), + (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3), + (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5), + (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7)); + const __m512i RT = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane k: ROTL by 7-k */ + const __m512i Z = _mm512_setzero_si512(); + + if (n >= 16 && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr, pa, d); + while (j + 16 <= n && (sum & mask) != 0) { + bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */ + /* u = rotated deltas, then the prefix XOR over the 8 lanes + * (shift lanes up by 1, 2, 4 and XOR; _mm512_alignr_epi64(u, Z, + * 8-k) shifts up by k, zero-filling) */ + __m512i u = _mm512_rolv_epi64(_mm512_loadu_si512((const void *)(d + cur)), RT); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 7)); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 6)); + u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 4)); + uint64_t c = BZ_ROTL(sum, 8); + __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c), u); + if (_mm512_testn_epi64_mask(H, M)) { + size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */ + *sum_io = sum; + return j + r; + } + sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128( + _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */ + j += 8; + cur ^= 8; + } + } + if (j < n) /* up to 15 bytes here (one block more than the other kernels) */ j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask); *sum_io = sum; return j; @@ -211,17 +350,21 @@ bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, static int bz64_simd_available(void) { +#ifdef BZ_KIND_512 + if (__builtin_cpu_supports("avx512f")) + return 2; +#endif return __builtin_cpu_supports("avx2"); } #else -#define BZ_KIND "blocked" +#define BZ_KIND "blockwise" static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot, const uint8_t *pr, const uint8_t *pa, size_t n, uint64_t *sum_io, uint64_t mask) { - return bz64_scan_blocked(T, Trot, pr, pa, n, sum_io, mask); + return bz64_scan_blockwise(T, Trot, pr, pa, n, sum_io, mask); } static int bz64_simd_available(void) @@ -231,29 +374,109 @@ static int bz64_simd_available(void) #endif +/* --- kernel selection --------------------------------------------------- */ + +const char *bz64_kernel_names(void) +{ +#if defined(__aarch64__) + return "auto, neon, blockwise, scalar"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, avx512, avx2, blockwise, scalar"; +#else + return "auto, blockwise, scalar"; +#endif +} + +int bz64_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = BZ_K_AUTO; + return BZ_KSEL_OK; + } + if (strcmp(name, "scalar") == 0) { + *out_id = BZ_K_SCALAR; + return BZ_KSEL_OK; + } + if (strcmp(name, "blockwise") == 0) { + *out_id = BZ_K_BLOCKWISE; + return BZ_KSEL_OK; + } +#if defined(__aarch64__) + if (strcmp(name, "neon") == 0) { + *out_id = BZ_K_VECTOR; /* baseline on aarch64, always runnable */ + return BZ_KSEL_OK; + } +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "avx2") == 0) { + if (!__builtin_cpu_supports("avx2")) + return BZ_KSEL_NOCPU; + *out_id = BZ_K_VECTOR; + return BZ_KSEL_OK; + } + if (strcmp(name, "avx512") == 0) { +#ifndef BZ_KIND_512 + return BZ_KSEL_NOTBUILT; /* compiler too old for the target attribute */ +#else + if (!__builtin_cpu_supports("avx512f")) + return BZ_KSEL_NOCPU; + *out_id = BZ_K_VECTOR512; + return BZ_KSEL_OK; +#endif + } +#endif + return BZ_KSEL_UNKNOWN; +} + /* --- dispatch ----------------------------------------------------------- */ -/* resolved once; a racy double-init writes the same value, so it is benign */ -static int bz64_use_simd = -1; +/* the kernel BZ_K_AUTO resolves to, worked out once; a racy double-init + * writes the same value, so it is benign */ +static int bz64_auto = -1; + +static int bz64_auto_kernel(void) +{ + int a; + if (bz64_auto < 0) { + a = bz64_simd_available(); + bz64_auto = (a == 2) ? BZ_K_VECTOR512 : (a ? BZ_K_VECTOR : BZ_K_BLOCKWISE); + } + return bz64_auto; +} size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar) + size_t n, uint64_t *sum, uint64_t mask, int kernel) { - if (force_scalar) + if (kernel == BZ_K_AUTO) + kernel = bz64_auto_kernel(); + switch (kernel) { + case BZ_K_SCALAR: return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask); - if (bz64_use_simd < 0) - bz64_use_simd = bz64_simd_available(); - if (bz64_use_simd) + case BZ_K_BLOCKWISE: + return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask); +#ifdef BZ_KIND_512 + case BZ_K_VECTOR512: + return bz64_scan_simd512(table, table_rot, p_rem, p_add, n, sum, mask); +#endif + default: return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask); - return bz64_scan_blocked(table, table_rot, p_rem, p_add, n, sum, mask); + } } -const char *bz64_kernel_name(int force_scalar) +const char *bz64_kernel_name(int kernel) { - if (force_scalar) + if (kernel == BZ_K_AUTO) + kernel = bz64_auto_kernel(); + switch (kernel) { + case BZ_K_SCALAR: return "scalar"; - if (bz64_use_simd < 0) - bz64_use_simd = bz64_simd_available(); - return bz64_use_simd ? BZ_KIND : "blocked"; + case BZ_K_BLOCKWISE: + return "blockwise"; +#ifdef BZ_KIND_512 + case BZ_K_VECTOR512: + return BZ_KIND_512; +#endif + default: + return BZ_KIND; + } } diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h index cd805341c7..c75496ce35 100644 --- a/src/borg/chunkers/buzhash64_impl.h +++ b/src/borg/chunkers/buzhash64_impl.h @@ -1,5 +1,5 @@ /* buzhash64 chunker scan kernel: the cyclic-polynomial rolling hash inner - * loop, with blocked/SIMD implementations that are bit-identical to the + * loop, with blockwise/SIMD implementations that are bit-identical to the * plain sequential loop (see buzhash64_impl.c for the algebra). * * The cut decision granularity stays at 1 byte: every position is tested @@ -21,13 +21,34 @@ * ROTL(table[b], window_size % 64) (precomputed by the caller). * p_rem points at the byte leaving the window, p_add at the byte entering * (p_add = p_rem + window_size); both must have n readable bytes. - * force_scalar != 0 selects the sequential reference loop (for tests); - * all kernels return bit-identical results. */ + * kernel is one of BZ_K_*; BZ_K_AUTO picks the best one this CPU can run. + * All kernels return bit-identical results. */ +/* Scan kernel ids, a tier ladder; see fastcdc_impl.h for the rationale. */ +#define BZ_K_AUTO 0 /* best kernel this CPU can run */ +#define BZ_K_SCALAR 1 /* sequential reference loop */ +#define BZ_K_BLOCKWISE 2 /* portable 8-lane C */ +#define BZ_K_VECTOR 3 /* neon / avx2 (neon is not auto-selected) */ +#define BZ_K_VECTOR512 4 /* avx512 */ + +/* Results of bz64_kernel_select(). */ +#define BZ_KSEL_OK 0 +#define BZ_KSEL_UNKNOWN 1 /* not a kernel name on this platform */ +#define BZ_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define BZ_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a kernel name for this build; see fc_kernel_select(). */ +int bz64_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the kernel names this build accepts. */ +const char *bz64_kernel_names(void); + size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot, const uint8_t *p_rem, const uint8_t *p_add, - size_t n, uint64_t *sum, uint64_t mask, int force_scalar); + size_t n, uint64_t *sum, uint64_t mask, int kernel); -/* Name of the kernel bz64_scan would use: "neon", "avx2", "blocked" or "scalar". */ -const char *bz64_kernel_name(int force_scalar); +/* Name of the kernel selects: "neon", "avx512", "avx2", "blockwise" + * or "scalar"; for BZ_K_AUTO, the auto-selected one. Note "neon" is never + * auto-selected, see buzhash64_impl.c. */ +const char *bz64_kernel_name(int kernel); #endif diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx index 7172fe94fc..ca0d11216c 100644 --- a/src/borg/chunkers/fastcdc.pyx +++ b/src/borg/chunkers/fastcdc.pyx @@ -13,8 +13,13 @@ from ..crypto.low_level import CSPRNG from .base cimport ChunkerBase cdef extern from "fastcdc_impl.h": - int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) nogil - const char *fc_kernel_name(int force_scalar) + int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel) nogil + const char *fc_kernel_name(int kernel) + int fc_kernel_select(const char *name, int *out_id) + const char *fc_kernel_names() + int FC_K_AUTO + +from .kernel_env import kernel_error, requested_kernel # FastCDC content-defined chunker (Xia et al., USENIX ATC 2016). # @@ -33,9 +38,10 @@ cdef extern from "fastcdc_impl.h": # class only provides the keyed Gear table and the _scan() hook calling the C kernel. # # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on -# aarch64, AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to the plain -# sequential Gear loop: every byte position is tested, cuts and hash state are exactly the -# same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop (for tests). +# aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to +# the plain sequential Gear loop: every byte position is tested, cuts and hash state are +# exactly the same, only faster. BORG_FASTCDC_KERNEL pins the kernel instead of letting it +# be auto-selected; see kernel_env.py and the .kernel property. @cython.boundscheck(False) @@ -58,6 +64,20 @@ cdef uint64_t* fastcdc_init_gear(bytes key) except NULL: return gear +cdef int _select_kernel() except -1: + """Resolve BORG_FASTCDC_KERNEL to a kernel id, raising if it cannot be honoured.""" + cdef int kid = FC_K_AUTO + cdef int rc + want = requested_kernel("BORG_FASTCDC_KERNEL") + if want is None: + return FC_K_AUTO + rc = fc_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_FASTCDC_KERNEL", want, rc, + (fc_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerFastCDC(ChunkerBase): """ FastCDC content-defined chunker, variable chunk sizes, keyed Gear hash. @@ -65,11 +85,11 @@ cdef class ChunkerFastCDC(ChunkerBase): Unlike the buzhash chunkers, Gear is window-less, so there is no hash_window_size parameter. """ cdef uint64_t* gear - cdef int force_scalar + cdef int kernel_id def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int nc_level=0, size_t normal_size=0, bint sparse=False): self.gear = NULL - self.force_scalar = 1 if os.environ.get("BORG_FASTCDC_FORCE_SCALAR", "") not in ("", "0") else 0 + self.kernel_id = _select_kernel() self.gear = fastcdc_init_gear(key) # Gear accumulates information in its high bits, so the cut-decision # masks must use the high bits of the hash (high_masks=True). The Gear @@ -85,13 +105,17 @@ cdef class ChunkerFastCDC(ChunkerBase): @property def kernel(self): - """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'.""" - return (fc_kernel_name(self.force_scalar)).decode("ascii") + """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'. + + With BORG_FASTCDC_KERNEL set to anything but "auto", this is always the + requested kernel - creating the chunker fails otherwise. + """ + return (fc_kernel_name(self.kernel_id)).decode("ascii") cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept: cdef int64_t r with nogil: - r = fc_scan(self.gear, p, n, digest, mask, self.force_scalar) + r = fc_scan(self.gear, p, n, digest, mask, self.kernel_id) return r diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c index f7c184caa3..ac58923727 100644 --- a/src/borg/chunkers/fastcdc_impl.c +++ b/src/borg/chunkers/fastcdc_impl.c @@ -21,9 +21,11 @@ * which an exact sequential recheck of the block resolves. The per-lane * masks (mask << 7 .. mask << 0) are constants of the scan. * - * All kernels (sequential, blocked scalar, NEON, AVX2) return bit-identical - * cut positions and fp values; the chunker's golden tests depend on this. */ + * All kernels (sequential, blockwise scalar, NEON, AVX2, AVX-512) return + * bit-identical cut positions and fp values; the chunker's golden tests + * depend on this. */ +#include #include #include "fastcdc_impl.h" @@ -74,9 +76,9 @@ static inline void fc_block_prefix(const uint64_t *gear, const uint8_t *p, uint6 s[7] = s8; } -/* --- blocked scalar (portable C, no intrinsics) ------------------------- */ +/* --- blockwise scalar (portable C, no intrinsics) ------------------------- */ -static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) +static int64_t fc_scan_blockwise(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; uint64_t M[8]; @@ -119,24 +121,35 @@ static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n, #include +/* Per lane, "(x & M) == 0" is replaced by "x <= ~M", turning the and plus + * compare-against-zero into one unsigned compare against a limit computed + * once per scan. + * + * (x & M) == 0 means x's set bits are a subset of ~M's, which implies + * x <= ~M for any M - so this never loses a cut, whatever the mask looks + * like. For the contiguous high-bit masks the chunker actually uses (mask + * has its one-bits at the top and shifting left only drops bits off the top, + * so ~M is 2^(64-k)-1) the two are exactly equivalent, and this stays the + * same superset test as the masked form: the block's exact sequential + * recheck resolves candidates either way. */ static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; uint64_t s[8]; - uint64x2_t M12 = {mask << 7, mask << 6}; - uint64x2_t M34 = {mask << 5, mask << 4}; - uint64x2_t M56 = {mask << 3, mask << 2}; - uint64x2_t M78 = {mask << 1, mask}; + uint64x2_t L12 = {~(mask << 7), ~(mask << 6)}; + uint64x2_t L34 = {~(mask << 5), ~(mask << 4)}; + uint64x2_t L56 = {~(mask << 3), ~(mask << 2)}; + uint64x2_t L78 = {~(mask << 1), ~mask}; size_t i = 0; for (; i + 8 <= n; i += 8) { fc_block_prefix(gear, p + i, s); uint64_t c = fp << 8; uint64x2_t C = vdupq_n_u64(c); - uint64x2_t z12 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[0], s[1]}), M12)); - uint64x2_t z34 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[2], s[3]}), M34)); - uint64x2_t z56 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[4], s[5]}), M56)); - uint64x2_t z78 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[6], s[7]}), M78)); + uint64x2_t z12 = vcgeq_u64(L12, vaddq_u64(C, (uint64x2_t){s[0], s[1]})); + uint64x2_t z34 = vcgeq_u64(L34, vaddq_u64(C, (uint64x2_t){s[2], s[3]})); + uint64x2_t z56 = vcgeq_u64(L56, vaddq_u64(C, (uint64x2_t){s[4], s[5]})); + uint64x2_t z78 = vcgeq_u64(L78, vaddq_u64(C, (uint64x2_t){s[6], s[7]})); uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78)); if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ @@ -171,39 +184,154 @@ static int fc_simd_available(void) #include +/* Load the block's 8 gear values (via one 8-byte data load), without the + * shifts and the prefix sum: the vector kernels do those in the vector + * domain. Endianness-independent: bytes are extracted by shifting. */ +static inline void fc_block_gear(const uint64_t *gear, const uint8_t *p, uint64_t g[8]) +{ + uint64_t w; + memcpy(&w, p, 8); +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + w = __builtin_bswap64(w); +#endif + g[0] = gear[(uint8_t)w]; + g[1] = gear[(uint8_t)(w >> 8)]; + g[2] = gear[(uint8_t)(w >> 16)]; + g[3] = gear[(uint8_t)(w >> 24)]; + g[4] = gear[(uint8_t)(w >> 32)]; + g[5] = gear[(uint8_t)(w >> 40)]; + g[6] = gear[(uint8_t)(w >> 48)]; + g[7] = gear[(uint8_t)(w >> 56)]; +} + +/* The 8-lane candidate test in two 256-bit vectors, with the aligned domain + * built in the vector domain: vpsllvq applies the per-lane shifts, then an + * inclusive prefix sum inside each 4-lane half (two vpermq+vpaddq steps) plus + * a broadcast of the low half's total into the high half. + * + * Only the 8 gear table lookups stay scalar, and they are done one block + * ahead into a double buffer: a 32-byte vector load placed right after the + * eight 8-byte scalar stores that filled it cannot use store-to-load + * forwarding and stalls, twice per block. Loading block i+1 while block i is + * tested puts a full loop body between the stores and the load, so the stores + * have retired by then and the stall disappears. Before this, the stalls made + * the AVX2 kernel slower than the blockwise scalar one. */ __attribute__((target("avx2"))) static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { uint64_t fp = *fp_io; - uint64_t s[8]; + uint64_t g[16]; /* double buffer: block being tested + block being loaded */ /* _mm256_set_epi64x takes arguments high lane first */ - __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5), - (long long)(mask << 6), (long long)(mask << 7)); - __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1), - (long long)(mask << 2), (long long)(mask << 3)); - __m256i zero = _mm256_setzero_si256(); + const __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5), + (long long)(mask << 6), (long long)(mask << 7)); + const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1), + (long long)(mask << 2), (long long)(mask << 3)); + const __m256i SL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 <<= 7..4 */ + const __m256i SH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 <<= 3..0 */ + const __m256i zero = _mm256_setzero_si256(); size_t i = 0; + int cur = 0; - for (; i + 8 <= n; i += 8) { - fc_block_prefix(gear, p + i, s); - uint64_t c = fp << 8; - __m256i C = _mm256_set1_epi64x((long long)c); - __m256i H1 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[0])); - __m256i H2 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[4])); - __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); - __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); - __m256i any = _mm256_or_si256(z1, z2); - if (!_mm256_testz_si256(any, any)) { - int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ - if (r >= 0) { - *fp_io = fp; - return (int64_t)i + r; + if (n >= 16) { + fc_block_gear(gear, p, g); + for (; i + 16 <= n; i += 8, cur ^= 8) { + fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */ + __m256i lo = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur)), SL); + __m256i hi = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur + 4)), SH); + /* inclusive prefix sum inside each half: rotate the lanes up by 1 + * resp. 2 (vpermq), zero the lanes rotated in, add */ + lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03)); + lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F)); + hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03)); + hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F)); + hi = _mm256_add_epi64(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */ + uint64_t c = fp << 8; + __m256i C = _mm256_set1_epi64x((long long)c); + __m256i H1 = _mm256_add_epi64(C, lo); + __m256i H2 = _mm256_add_epi64(C, hi); + __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero); + __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero); + __m256i any = _mm256_or_si256(z1, z2); + if (!_mm256_testz_si256(any, any)) { + int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } else { + fp = c + (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128( + _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */ } - } else { - fp = c + s[7]; } } - if (i < n) { + if (i < n) { /* up to 15 bytes here (one block more than the scalar kernels) */ + int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask); + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } + *fp_io = fp; + return -1; +} + +/* --- AVX-512 (x86-64, runtime detected) --------------------------------- */ + +#define FC_KIND_512 "avx512" + +/* The same 8-lane candidate test as the AVX2 kernel, but the aligned domain + * fits in one 512-bit vector: vpsllvq applies the per-lane shifts (lane j << + * (7-j)), three valignq+vpaddq steps turn the shifted gear values into the + * inclusive prefix sums s[0..7] in one pass instead of two halves plus a + * carry, and vptestnmq fuses the AND and the ==0 test into a mask register. + * + * The gear lookups are pipelined one block ahead exactly as in the AVX2 + * kernel above, for the same store-to-load forwarding reason. + * + * The serial chain stays short: only fp = c + s[7] (scalar add + shift per + * 8 bytes) is carried across blocks, and s[7] is read out of the vector with + * vpermq off that chain. */ +__attribute__((target("avx512f"))) static int64_t +fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) +{ + uint64_t fp = *fp_io; + uint64_t g[16]; /* double buffer: block being tested + block being loaded */ + /* _mm512_set_epi64 takes arguments high lane first: lane j = mask << (7-j) */ + const __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1), + (long long)(mask << 2), (long long)(mask << 3), + (long long)(mask << 4), (long long)(mask << 5), + (long long)(mask << 6), (long long)(mask << 7)); + const __m512i SH = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane j <<= 7-j */ + const __m512i Z = _mm512_setzero_si512(); + size_t i = 0; + int cur = 0; + + if (n >= 16) { + fc_block_gear(gear, p, g); + for (; i + 16 <= n; i += 8, cur ^= 8) { + fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */ + /* u = shifted gear values, then the inclusive prefix sum over the + * 8 lanes (Hillis-Steele: shift lanes up by 1, 2, 4 and add; + * _mm512_alignr_epi64(u, Z, 8-k) shifts up by k, zero-filling) */ + __m512i u = _mm512_sllv_epi64(_mm512_loadu_si512((const void *)(g + cur)), SH); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 7)); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 6)); + u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 4)); + uint64_t c = fp << 8; + __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c), u); + if (_mm512_testn_epi64_mask(H, M)) { + int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */ + if (r >= 0) { + *fp_io = fp; + return (int64_t)i + r; + } + } else { + fp = c + (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128( + _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */ + } + } + } + if (i < n) { /* up to 15 bytes here (one block more than the other kernels) */ int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask); if (r >= 0) { *fp_io = fp; @@ -216,15 +344,19 @@ fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, static int fc_simd_available(void) { +#ifdef FC_KIND_512 + if (__builtin_cpu_supports("avx512f")) + return 2; +#endif return __builtin_cpu_supports("avx2"); } #else -#define FC_KIND "blocked" +#define FC_KIND "blockwise" static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask) { - return fc_scan_blocked(gear, p, n, fp_io, mask); + return fc_scan_blockwise(gear, p, n, fp_io, mask); } static int fc_simd_available(void) @@ -234,27 +366,107 @@ static int fc_simd_available(void) #endif +/* --- kernel selection --------------------------------------------------- */ + +const char *fc_kernel_names(void) +{ +#if defined(__aarch64__) + return "auto, neon, blockwise, scalar"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, avx512, avx2, blockwise, scalar"; +#else + return "auto, blockwise, scalar"; +#endif +} + +int fc_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = FC_K_AUTO; + return FC_KSEL_OK; + } + if (strcmp(name, "scalar") == 0) { + *out_id = FC_K_SCALAR; + return FC_KSEL_OK; + } + if (strcmp(name, "blockwise") == 0) { + *out_id = FC_K_BLOCKWISE; + return FC_KSEL_OK; + } +#if defined(__aarch64__) + if (strcmp(name, "neon") == 0) { + *out_id = FC_K_VECTOR; /* baseline on aarch64, always runnable */ + return FC_KSEL_OK; + } +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "avx2") == 0) { + if (!__builtin_cpu_supports("avx2")) + return FC_KSEL_NOCPU; + *out_id = FC_K_VECTOR; + return FC_KSEL_OK; + } + if (strcmp(name, "avx512") == 0) { +#ifndef FC_KIND_512 + return FC_KSEL_NOTBUILT; /* compiler too old for the target attribute */ +#else + if (!__builtin_cpu_supports("avx512f")) + return FC_KSEL_NOCPU; + *out_id = FC_K_VECTOR512; + return FC_KSEL_OK; +#endif + } +#endif + return FC_KSEL_UNKNOWN; +} + /* --- dispatch ----------------------------------------------------------- */ -/* resolved once; a racy double-init writes the same value, so it is benign */ -static int fc_use_simd = -1; +/* the kernel FC_K_AUTO resolves to, worked out once; a racy double-init + * writes the same value, so it is benign */ +static int fc_auto = -1; + +static int fc_auto_kernel(void) +{ + int a; + if (fc_auto < 0) { + a = fc_simd_available(); + fc_auto = (a == 2) ? FC_K_VECTOR512 : (a ? FC_K_VECTOR : FC_K_BLOCKWISE); + } + return fc_auto; +} -int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) +int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel) { - if (force_scalar) + if (kernel == FC_K_AUTO) + kernel = fc_auto_kernel(); + switch (kernel) { + case FC_K_SCALAR: return fc_scan_seq(gear, p, n, fp, mask); - if (fc_use_simd < 0) - fc_use_simd = fc_simd_available(); - if (fc_use_simd) + case FC_K_BLOCKWISE: + return fc_scan_blockwise(gear, p, n, fp, mask); +#ifdef FC_KIND_512 + case FC_K_VECTOR512: + return fc_scan_simd512(gear, p, n, fp, mask); +#endif + default: return fc_scan_simd(gear, p, n, fp, mask); - return fc_scan_blocked(gear, p, n, fp, mask); + } } -const char *fc_kernel_name(int force_scalar) +const char *fc_kernel_name(int kernel) { - if (force_scalar) + if (kernel == FC_K_AUTO) + kernel = fc_auto_kernel(); + switch (kernel) { + case FC_K_SCALAR: return "scalar"; - if (fc_use_simd < 0) - fc_use_simd = fc_simd_available(); - return fc_use_simd ? FC_KIND : "blocked"; + case FC_K_BLOCKWISE: + return "blockwise"; +#ifdef FC_KIND_512 + case FC_K_VECTOR512: + return FC_KIND_512; +#endif + default: + return FC_KIND; + } } diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h index 10887ec3d8..145127b485 100644 --- a/src/borg/chunkers/fastcdc_impl.h +++ b/src/borg/chunkers/fastcdc_impl.h @@ -11,16 +11,42 @@ #include #include +/* Scan kernel ids, a tier ladder. Which names map onto them depends on the + * build: "neon" exists only on aarch64, "avx2"/"avx512" only on x86-64. */ +#define FC_K_AUTO 0 /* best kernel this CPU can run */ +#define FC_K_SCALAR 1 /* sequential reference loop */ +#define FC_K_BLOCKWISE 2 /* portable 8-lane C */ +#define FC_K_VECTOR 3 /* the platform's vector kernel: neon or avx2 */ +#define FC_K_VECTOR512 4 /* avx512 */ + +/* Results of fc_kernel_select(). */ +#define FC_KSEL_OK 0 +#define FC_KSEL_UNKNOWN 1 /* not a kernel name on this platform */ +#define FC_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define FC_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a kernel name for this build. On FC_KSEL_OK the id is stored in + * *out_id, otherwise *out_id is left alone. The three failures are kept apart + * because they need different fixes: a typo, too old a compiler, or the wrong + * CPU. */ +int fc_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the kernel names this build accepts, for error + * messages. Names a CPU cannot run are still listed. */ +const char *fc_kernel_names(void); + /* Scan up to n positions: for i = 0..n-1 advance fp = (fp << 1) + gear[p[i]] * and test (fp & mask) == 0. * Returns the first i that matched (fp is left at position i), or -1 if none * matched (fp is left at position n-1). - * gear: the keyed 256-entry table. force_scalar != 0 selects the sequential - * reference loop (for tests); otherwise the best kernel for this CPU is used. + * gear: the keyed 256-entry table. kernel is one of FC_K_*; FC_K_AUTO picks + * the best one this CPU can run. Callers are expected to have validated any + * explicit choice with fc_kernel_select() - an unrunnable one falls back to + * FC_K_AUTO rather than crashing. * All kernels return bit-identical results. */ -int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar); +int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel); -/* Name of the kernel fc_scan would use: "neon", "avx2", "blocked" or "scalar". */ -const char *fc_kernel_name(int force_scalar); +/* Name of the kernel selects; for FC_K_AUTO, the auto-selected one. */ +const char *fc_kernel_name(int kernel); #endif diff --git a/src/borg/chunkers/goldilocks_aes.pyx b/src/borg/chunkers/goldilocks_aes.pyx index 50a6e62d28..329577a723 100644 --- a/src/borg/chunkers/goldilocks_aes.pyx +++ b/src/borg/chunkers/goldilocks_aes.pyx @@ -45,7 +45,11 @@ cdef extern from "goldilocks_aes_impl.h": ctypedef struct GL_CTX: pass int GL_TABLES - GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int force_sw) + GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void gl_free(GL_CTX *ctx) const char *gl_kind(const GL_CTX *ctx) uint64_t gl_digest64(const GL_CTX *ctx, const uint8_t *q) @@ -105,6 +109,27 @@ def _derive(bytes key): return aes_key, k, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerGoldilocksAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -128,8 +153,8 @@ cdef class ChunkerGoldilocksAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_GOLDILOCKS_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") self.kernel_str = (gl_kind(self.ctx)).decode("ascii") @@ -176,7 +201,7 @@ def goldilocks_aes_digest64(k, bytes window): for i in range(256): c_tables[t * 256 + i] = tables[t][i] memset(aes_key, 0, 16) - ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1) + ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, PHTE_K_EVP) if ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") d = gl_digest64(ctx, PyBytes_AsString(window)) @@ -205,7 +230,7 @@ def goldilocks_aes_scan_all(k, bytes aes_key, bytes data, mask, bint force_sw=Fa for t in range(len(tables)): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - ctx = gl_new(c_tables, k, (k * k) % _GL_P, PyBytes_AsString(aes_key), 1 if force_sw else 0) + ctx = gl_new(c_tables, k, (k * k) % _GL_P, PyBytes_AsString(aes_key), PHTE_K_EVP if force_sw else PHTE_K_AUTO) if ctx == NULL: raise MemoryError("Failed to set up goldilocks-aes kernel") try: diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c index 5b045e89fc..f6760ce16f 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.c +++ b/src/borg/chunkers/goldilocks_aes_impl.c @@ -31,18 +31,25 @@ struct GL_CTX { /* --- Goldilocks field arithmetic --------------------------------------- * - * All values are kept canonical (< p) at every step: the state is fed to - * AES verbatim, so a non-canonical representation of the same field element - * would change cut decisions. */ + * Every digest the scan produces is canonical (< p): it is fed to AES + * verbatim, so a non-canonical representation of the same field element + * would change cut decisions. The rolls reach that via gl_add, which always + * canonicalizes; only the multiply inside them is allowed to hand on a + * merely reduced (< 2^64) representative, see gl_mul_lazy. */ +/* The reductions are data-dependent and unpredictable (~27% branch-miss rate + * measured), so they must not become branches. Writing them as if() or as a + * ternary lets GCC emit real conditional jumps - the mispredicts alone cost + * more than the arithmetic (gl_mul: 15.1 vs 4.4 cycles per call). The + * __builtin_*_overflow forms keep the carry/borrow in the flags, so the + * compiler settles on sbb/adc + mask instead. Results are unchanged: the + * value is still fully canonical, which matters because it is fed to AES. */ static inline uint64_t gl_add(uint64_t a, uint64_t b) { - uint64_t s = a + b; - if (s < a) /* carry out: 2^64 mod p = GL_EPS; cannot re-carry */ - s += GL_EPS; - if (s >= GL_P) - s -= GL_P; - return s; + uint64_t s, u; + unsigned char carry = __builtin_add_overflow(a, b, &s); + s += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* 2^64 mod p; cannot re-carry */ + return __builtin_sub_overflow(s, GL_P, &u) ? s : u; } /* a * b mod p via the standard 2^64 = 2^32 - 1 folding (risc0/plonky2 style): @@ -53,15 +60,36 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b) __uint128_t t = (__uint128_t)a * b; uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64); uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS; - uint64_t t0 = lo - hi_hi; - if (lo < hi_hi) /* borrow: -2^64 = -eps (mod p) */ - t0 -= GL_EPS; - uint64_t t1 = hi_lo * GL_EPS; /* < 2^64 - 2^33 + 2, no overflow */ - uint64_t r = t0 + t1; - if (r < t1) /* carry out; cannot re-carry (t1 <= 2^64 - 2^33 + 1) */ - r += GL_EPS; - if (r >= GL_P) - r -= GL_P; + uint64_t t0, r, u; + unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0); + t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */ + /* t1 = hi_lo * eps < 2^64 - 2^33 + 2, no overflow */ + unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r); + r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */ + return __builtin_sub_overflow(r, GL_P, &u) ? r : u; +} + +/* gl_mul without the final canonicalization: the result is congruent to a*b + * mod p but only bounded by 2^64, not by p. + * + * That is enough for the rolls, because both feed it straight into a gl_add + * whose other operand is canonical, and that gl_add canonicalizes anyway. + * The bound still holds with a non-canonical first operand: for a < 2^64 and + * b < p, a carry leaves a + b - 2^64 <= p - 2, so adding eps cannot carry a + * second time, and the one conditional subtraction of p then lands below p + * (r - p < 2^32). So the digest handed to AES stays canonical - which it must + * be, since it is fed to AES verbatim - while the multiply chain that limits + * this kernel loses its trailing compare and select. */ +static inline uint64_t gl_mul_lazy(uint64_t a, uint64_t b) +{ + __uint128_t t = (__uint128_t)a * b; + uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64); + uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS; + uint64_t t0, r; + unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0); + t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */ + unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r); + r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */ return r; } @@ -73,7 +101,7 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b) static inline uint64_t gl_roll(const GL_CTX *c, uint64_t s, uint8_t byte_out, uint8_t byte_in) { uint64_t delta = gl_add(c->nout64_tbl[byte_out], byte_in); - return gl_add(gl_mul(s, c->k1), delta); + return gl_add(gl_mul_lazy(s, c->k1), delta); } /* Advance the state by TWO bytes in one step (exact composition of two @@ -87,7 +115,7 @@ static inline uint64_t gl_roll2(const GL_CTX *c, uint64_t s, { uint64_t delta = gl_add(gl_add(c->nout65_tbl[o0], c->nout64_tbl[o1]), gl_add(c->in1_tbl[i0], i1)); - return gl_add(gl_mul(s, c->k2), delta); + return gl_add(gl_mul_lazy(s, c->k2), delta); } uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q) @@ -107,7 +135,7 @@ uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q) #include "phte_scan.h" GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2, - const uint8_t aes_key[16], int force_sw) + const uint8_t aes_key[16], int kernel) { GL_CTX *c = calloc(1, sizeof(GL_CTX)); if (c == NULL) @@ -117,7 +145,7 @@ GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2, memcpy(c->in1_tbl, tables + 2 * 256, sizeof(c->in1_tbl)); c->k1 = k1; c->k2 = k2; - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h index 0b0dd148ab..f1f04a3444 100644 --- a/src/borg/chunkers/goldilocks_aes_impl.h +++ b/src/borg/chunkers/goldilocks_aes_impl.h @@ -21,6 +21,8 @@ #include #include +#include "phte_kernel.h" + typedef struct GL_CTX GL_CTX; /* Number of 256-entry rolling tables passed to gl_new, in this order: @@ -37,13 +39,13 @@ typedef struct GL_CTX GL_CTX; * k1: the secret evaluation point K (canonical, 0 <= K < p). * k2: K^2 mod p. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int force_sw); +GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int kernel); void gl_free(GL_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *gl_kind(const GL_CTX *ctx); /* Full (non-rolling) polynomial hash of the 64 bytes at q (Horner): the diff --git a/src/borg/chunkers/kernel_env.py b/src/borg/chunkers/kernel_env.py new file mode 100644 index 0000000000..aacfcc76ae --- /dev/null +++ b/src/borg/chunkers/kernel_env.py @@ -0,0 +1,42 @@ +"""Shared parsing for the BORG_*_KERNEL scan-kernel selection env vars. + +Each SIMD-accelerated chunker picks its inner scan kernel automatically. The +env vars exist to pin it instead - for benchmarking one kernel against +another, and for CI that wants to prove a particular kernel was exercised +rather than hope it was. + +"auto" (the default) takes the best kernel the CPU can run. Any other value is +a demand, not a preference: if that kernel cannot run here, chunker creation +raises ValueError rather than quietly falling back, because a silent fallback +turns a benchmark or a test into a measurement of something else. + +The C side resolves names, since which names exist depends on the build and +the CPU; this module only turns its verdict into an exception. +""" + +import os + +# must match the FC_KSEL_* / BZ_KSEL_* / PHTE_KSEL_* codes in the C headers +KSEL_OK = 0 +KSEL_UNKNOWN = 1 +KSEL_NOTBUILT = 2 +KSEL_NOCPU = 3 + + +def kernel_error(envvar, want, status, valid): + """The ValueError for a kernel request that cannot be honoured.""" + if status == KSEL_UNKNOWN: + detail = "not a kernel of this build" + elif status == KSEL_NOTBUILT: + detail = "not compiled into this build (needs a newer compiler)" + elif status == KSEL_NOCPU: + detail = "this CPU does not support it" + else: + detail = "unavailable" + return ValueError(f"{envvar}={want!r}: {detail}. Valid values: {valid}") + + +def requested_kernel(envvar): + """The kernel name requested via , or None for auto/unset.""" + want = os.environ.get(envvar, "").strip().lower() + return None if want in ("", "auto") else want diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h index fe9856e6b1..7c7925e698 100644 --- a/src/borg/chunkers/phte_core.h +++ b/src/borg/chunkers/phte_core.h @@ -13,8 +13,11 @@ #ifndef BORG_PHTE_CORE_H #define BORG_PHTE_CORE_H +#include "phte_kernel.h" + #include #include +#include #include #include @@ -25,6 +28,7 @@ typedef struct { uint8_t rk[11][16]; /* AES-128 round keys, for the hardware paths */ EVP_CIPHER_CTX *evp; /* portable path */ int use_hw; + int use_hw512; /* VAES/AVX-512 variant of the hardware path (x86-64) */ } PHTE_BASE; /* --- endianness-explicit helpers (byte loops, so cut points do not depend @@ -130,23 +134,97 @@ phte_aes1_ni(const uint8_t rk[11][16], __m128i b) return _mm_aesenclast_si128(b, _mm_loadu_si128((const __m128i *)rk[10])); } +/* VAES/AVX-512 variant of the hardware path (4 AES blocks per instruction). + * __builtin_cpu_supports("vaes") needs GCC >= 11 / clang >= 14; older + * compilers simply keep the 128-bit AES-NI path. */ +#if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || (defined(__clang__) && __clang_major__ >= 14) +#define PHTE_HAVE_HW512 1 +#define PHTE_KIND_HW512 "vaes" + +static int phte_hw512_available(void) +{ + return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("vaes"); +} +#endif + #else #define PHTE_HAVE_HW 0 #endif +#ifndef PHTE_HAVE_HW512 +#define PHTE_HAVE_HW512 0 +#endif + +/* --- kernel selection --------------------------------------------------- + * + * Shared by all three AES chunkers: the scan path lives in phte_scan.h and + * only the rolling hash differs between them, so there is no machine on which + * one of them has a hardware path and another does not. Hence one selector + * (BORG_AES_CHUNKER_KERNEL) rather than one per chunker. */ + +const char *phte_kernel_names(void) +{ +#if defined(__aarch64__) + return "auto, aes-arm64, evp"; +#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + return "auto, vaes, aes-ni, evp"; +#else + return "auto, evp"; +#endif +} + +int phte_kernel_select(const char *name, int *out_id) +{ + if (strcmp(name, "auto") == 0) { + *out_id = PHTE_K_AUTO; + return PHTE_KSEL_OK; + } + if (strcmp(name, "evp") == 0) { + *out_id = PHTE_K_EVP; + return PHTE_KSEL_OK; + } +#if PHTE_HAVE_HW + if (strcmp(name, PHTE_KIND_HW) == 0) { + if (!phte_hw_available()) + return PHTE_KSEL_NOCPU; + *out_id = PHTE_K_HW; + return PHTE_KSEL_OK; + } +#endif +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) + if (strcmp(name, "vaes") == 0) { +#if !PHTE_HAVE_HW512 + return PHTE_KSEL_NOTBUILT; /* compiler too old for the VAES intrinsics */ +#else + if (!phte_hw512_available()) + return PHTE_KSEL_NOCPU; + *out_id = PHTE_K_HW512; + return PHTE_KSEL_OK; +#endif + } +#endif + return PHTE_KSEL_UNKNOWN; +} + /* --- context base management ------------------------------------------- */ /* Expand the AES key, select the scan path and set up the OpenSSL context. + * kernel is one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns 0 on failure (caller frees its context). */ -static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int force_sw) +static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int kernel) { phte_aes128_expand(aes_key, b->rk); #if PHTE_HAVE_HW - b->use_hw = !force_sw && phte_hw_available(); + b->use_hw = (kernel == PHTE_K_AUTO || kernel == PHTE_K_HW || kernel == PHTE_K_HW512) && phte_hw_available(); #else - (void)force_sw; b->use_hw = 0; #endif +#if PHTE_HAVE_HW512 + b->use_hw512 = b->use_hw && kernel != PHTE_K_HW && phte_hw512_available(); +#else + b->use_hw512 = 0; +#endif + (void)kernel; b->evp = EVP_CIPHER_CTX_new(); if (b->evp == NULL) return 0; @@ -169,6 +247,10 @@ static void phte_base_free(PHTE_BASE *b) static const char *phte_base_kind(const PHTE_BASE *b) { +#if PHTE_HAVE_HW512 + if (b->use_hw512) + return PHTE_KIND_HW512; +#endif #if PHTE_HAVE_HW if (b->use_hw) return PHTE_KIND_HW; diff --git a/src/borg/chunkers/phte_kernel.h b/src/borg/chunkers/phte_kernel.h new file mode 100644 index 0000000000..1340c55d34 --- /dev/null +++ b/src/borg/chunkers/phte_kernel.h @@ -0,0 +1,36 @@ +/* Scan-path selection shared by the three UHF-then-PRF ("Chk-PHTE") chunkers + * (toeplitz-aes, rabin-aes, goldilocks-aes). + * + * They share phte_scan.h and differ only in their rolling hash, so the + * available scan paths are a property of the build and the CPU, never of the + * individual chunker: there is no machine where one of them has a hardware + * path and another does not. Hence one selector for all three. */ + +#ifndef BORG_PHTE_KERNEL_H +#define BORG_PHTE_KERNEL_H + +/* Scan path ids, a tier ladder. Which names map onto them depends on the + * build: "vaes" only on x86-64, and the 128-bit hardware path is spelled + * "aes-ni" on x86-64 and "aes-arm64" on aarch64. */ +#define PHTE_K_AUTO 0 /* best path this CPU can run */ +#define PHTE_K_EVP 1 /* portable OpenSSL EVP batch path */ +#define PHTE_K_HW 2 /* 128-bit AES instructions: aes-ni / aes-arm64 */ +#define PHTE_K_HW512 3 /* VAES/AVX-512, 4 AES blocks per instruction */ + +/* Results of phte_kernel_select(). */ +#define PHTE_KSEL_OK 0 +#define PHTE_KSEL_UNKNOWN 1 /* not a scan path name on this platform */ +#define PHTE_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */ +#define PHTE_KSEL_NOCPU 3 /* known and built, but this CPU cannot run it */ + +/* Resolve a scan path name for this build. On PHTE_KSEL_OK the id is stored + * in *out_id, otherwise *out_id is left alone. The three failures are kept + * apart because they need different fixes: a typo, too old a compiler, or the + * wrong CPU. */ +int phte_kernel_select(const char *name, int *out_id); + +/* Comma-separated list of the scan path names this build accepts, for error + * messages. Names a CPU cannot run are still listed. */ +const char *phte_kernel_names(void); + +#endif /* BORG_PHTE_KERNEL_H */ diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 4695cae788..1362658099 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -19,9 +19,11 @@ * - a portable path batching digests through OpenSSL EVP AES-128-ECB, * - a hardware path using AES instructions directly (arm64 crypto extension, * x86-64 AES-NI), which interleaves the serial rolling-hash chain with - * pipelined AES so the AES work is (mostly) hidden behind it. + * pipelined AES so the AES work is (mostly) hidden behind it, + * - a VAES/AVX-512 variant of the x86-64 hardware path (runtime-detected) + * encrypting 4 AES blocks per instruction. * - * Both paths produce bit-identical cut points. + * All paths produce bit-identical cut points. */ #define PH_CAT2(a, b) a##b @@ -105,11 +107,12 @@ static int64_t PH_FN(scan_evp)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t * /* --- hardware paths ---------------------------------------------------- * - * Both hardware paths process groups of 8 positions: the two rolling lanes - * advance 4 stride-2 steps each (two independent dependency chains), then the - * 8 digests are encrypted with interleaved AES instructions, which execute on - * different ports than the rolling work and thus overlap with the next - * group's. + * The 128-bit hardware paths process groups of 8 positions (the VAES/AVX-512 + * path below uses the same structure with groups of 32): the two rolling + * lanes advance 4 stride-2 steps each (two independent dependency chains), + * then the 8 digests are encrypted with interleaved AES instructions, which + * execute on different ports than the rolling work and thus overlap with the + * next group's. * * Loop invariant: entering a group at position i, db = digest at i and * da = digest at i+1 (both already computed); the group emits digests for @@ -178,18 +181,28 @@ static int64_t PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *d b6 = veorq_u8(vaeseq_u8(b6, k[9]), k[10]); b7 = veorq_u8(vaeseq_u8(b7, k[9]), k[10]); + /* Test all 8 ciphertexts without leaving the vector domain: uzp1 + * packs the low halves (the cut-decision uint64) of two blocks into + * one vector, then and/cmeq-zero/or-reduce. Moving the 8 values to + * general registers instead - one fmov each, plus 8 scalar and/cmp + * pairs - costs noticeably more on the common no-hit path, where + * nothing but the single "any lane hit" bit is ever needed. */ { - uint64_t c0 = vgetq_lane_u64(vreinterpretq_u64_u8(b0), 0); - uint64_t c1 = vgetq_lane_u64(vreinterpretq_u64_u8(b1), 0); - uint64_t c2 = vgetq_lane_u64(vreinterpretq_u64_u8(b2), 0); - uint64_t c3 = vgetq_lane_u64(vreinterpretq_u64_u8(b3), 0); - uint64_t c4 = vgetq_lane_u64(vreinterpretq_u64_u8(b4), 0); - uint64_t c5 = vgetq_lane_u64(vreinterpretq_u64_u8(b5), 0); - uint64_t c6 = vgetq_lane_u64(vreinterpretq_u64_u8(b6), 0); - uint64_t c7 = vgetq_lane_u64(vreinterpretq_u64_u8(b7), 0); - if ((((c0 & mask) == 0) | ((c1 & mask) == 0) | ((c2 & mask) == 0) | ((c3 & mask) == 0) | - ((c4 & mask) == 0) | ((c5 & mask) == 0) | ((c6 & mask) == 0) | ((c7 & mask) == 0))) { - uint64_t cs[8] = {c0, c1, c2, c3, c4, c5, c6, c7}; + const uint64x2_t maskv = vdupq_n_u64(mask); + uint64x2_t h01 = vuzp1q_u64(vreinterpretq_u64_u8(b0), vreinterpretq_u64_u8(b1)); + uint64x2_t h23 = vuzp1q_u64(vreinterpretq_u64_u8(b2), vreinterpretq_u64_u8(b3)); + uint64x2_t h45 = vuzp1q_u64(vreinterpretq_u64_u8(b4), vreinterpretq_u64_u8(b5)); + uint64x2_t h67 = vuzp1q_u64(vreinterpretq_u64_u8(b6), vreinterpretq_u64_u8(b7)); + uint64x2_t any = vorrq_u64(vorrq_u64(vceqzq_u64(vandq_u64(h01, maskv)), + vceqzq_u64(vandq_u64(h23, maskv))), + vorrq_u64(vceqzq_u64(vandq_u64(h45, maskv)), + vceqzq_u64(vandq_u64(h67, maskv)))); + if (vmaxvq_u32(vreinterpretq_u32_u64(any))) { + uint64_t cs[8]; + vst1q_u64(cs + 0, h01); + vst1q_u64(cs + 2, h23); + vst1q_u64(cs + 4, h45); + vst1q_u64(cs + 6, h67); for (int j = 0; j < 8; j++) { if ((cs[j] & mask) == 0) { *digest = dg[j]; @@ -334,10 +347,140 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t #endif /* PHTE_HAVE_HW */ +#if PHTE_HAVE_HW512 + +/* VAES/AVX-512 variant of the x86-64 hardware path: the same two rolling + * lanes, but groups of 32 positions whose digests are encrypted as 8 zmm + * vectors of 4 AES blocks each - 4x fewer AES instructions than the 128-bit + * path, the round keys stay register-resident (32 zmm registers instead of + * 16 xmm, so no per-round key reloads), and the 8 independent chains hide + * the vaesenc latency (2 chains would be latency-bound), and a masked + * vptestnmq tests the 4 ciphertext-low qwords against the mask without + * extracting. Every digest is exact and every position is tested, so cut + * points stay bit-identical to the other paths. + * + * The digests are stored pre-spread: an AES input block wants the digest in + * the low qword and zero in the high one, so dgs[] keeps a digest in every + * even slot and a zero in every odd one. The zeros are written once at entry + * and never touched again, which makes each vector's input a plain aligned + * 512-bit load. Packing the digests and spreading them with vpexpandq (eight + * of those per group) instead costs the same stores and measurably more + * time - about 9% of the whole toeplitz-aes scan, 6% of rabin-aes. */ +__attribute__((target("aes,sse2,vaes,avx512f"))) static int64_t +PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) +{ + __m512i k[11]; + __m512i M = _mm512_set1_epi64((long long)mask); + /* digest at dgs[2j], zero at dgs[2j+1] (= AES input block bytes 8..15) */ + uint64_t dgs[64] __attribute__((aligned(64))); + uint64_t d = *digest, da, db; + const uint8_t *qo = p - 64; + size_t i = 0; + int lanes_live = 0; + + /* broadcast each 128-bit round key to all four block lanes */ + for (int j = 0; j < 11; j++) + k[j] = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c->base.rk[j])); + for (int j = 1; j < 64; j += 2) + dgs[j] = 0; + + if (n >= 34) { + db = PH_ROLL(c, d, qo[0], p[0]); /* d_0 */ + da = PH_ROLL2(c, d, qo[0], qo[1], p[0], p[1]); /* d_1 */ + lanes_live = 1; + } + + while (lanes_live && i + 34 <= n) { + __m512i b0, b1, b2, b3, b4, b5, b6, b7; + __mmask8 h0, h1, h2, h3, h4, h5, h6, h7; + + dgs[0] = db; + dgs[2] = da; + for (int j = 2; j < 32; j += 2) { + db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]); + dgs[2 * j] = db; + da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]); + dgs[2 * j + 2] = da; + } + /* prepare the next group's invariant (digests at i+32, i+33) */ + db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]); + da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]); + + /* each vector's 4 blocks are already laid out in dgs (digest in the + * low qword of every block, zero in the high one) */ + b0 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 0)), k[0]); + b1 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 8)), k[0]); + b2 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 16)), k[0]); + b3 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 24)), k[0]); + b4 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 32)), k[0]); + b5 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 40)), k[0]); + b6 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 48)), k[0]); + b7 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 56)), k[0]); + for (int r = 1; r < 10; r++) { + b0 = _mm512_aesenc_epi128(b0, k[r]); + b1 = _mm512_aesenc_epi128(b1, k[r]); + b2 = _mm512_aesenc_epi128(b2, k[r]); + b3 = _mm512_aesenc_epi128(b3, k[r]); + b4 = _mm512_aesenc_epi128(b4, k[r]); + b5 = _mm512_aesenc_epi128(b5, k[r]); + b6 = _mm512_aesenc_epi128(b6, k[r]); + b7 = _mm512_aesenc_epi128(b7, k[r]); + } + h0 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b0, k[10]), M); + h1 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b1, k[10]), M); + h2 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b2, k[10]), M); + h3 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b3, k[10]), M); + h4 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b4, k[10]), M); + h5 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b5, k[10]), M); + h6 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b6, k[10]), M); + h7 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b7, k[10]), M); + + if (h0 | h1 | h2 | h3 | h4 | h5 | h6 | h7) { + /* block j of vector g is qword lane 2j and position i + 4g + j */ + const uint8_t hs[8] = {h0, h1, h2, h3, h4, h5, h6, h7}; + for (int j = 0; j < 32; j++) { + if ((hs[j >> 2] >> ((j & 3) * 2)) & 1) { + *digest = dgs[2 * j]; + return (int64_t)(i + j); + } + } + } + i += 32; + } + /* tail: single positions, exactly as in the 128-bit path */ + if (lanes_live && i < n) { + __m128i b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)db)); + if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) { + *digest = db; + return (int64_t)i; + } + d = db; + i++; + } + while (i < n) { + __m128i b; + d = PH_ROLL(c, d, qo[i], p[i]); + b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)d)); + if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) { + *digest = d; + return (int64_t)i; + } + i++; + } + *digest = d; + return -1; +} + +#endif /* PHTE_HAVE_HW512 */ + /* --- dispatch (exported) ------------------------------------------------ */ int64_t PH_FN(scan)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) { +#if PHTE_HAVE_HW512 + if (c->base.use_hw512) + return PH_FN(scan_hw512)(c, p, n, digest, mask); +#endif #if PHTE_HAVE_HW if (c->base.use_hw) return PH_FN(scan_hw)(c, p, n, digest, mask); diff --git a/src/borg/chunkers/rabin_aes.pyx b/src/borg/chunkers/rabin_aes.pyx index 8198a8f17b..fa55ded145 100644 --- a/src/borg/chunkers/rabin_aes.pyx +++ b/src/borg/chunkers/rabin_aes.pyx @@ -57,7 +57,11 @@ cdef extern from "rabin_aes_impl.h": ctypedef struct RA_CTX: pass int RA_TABLES - RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw) + RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void ra_free(RA_CTX *ctx) const char *ra_kind(const RA_CTX *ctx) uint64_t ra_digest64(const RA_CTX *ctx, const uint8_t *q) @@ -175,6 +179,27 @@ def _derive(bytes key): return aes_key, p, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerRabinAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -198,8 +223,8 @@ cdef class ChunkerRabinAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_RABIN_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = ra_new(c_tables, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = ra_new(c_tables, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up rabin-aes kernel") self.kernel_str = (ra_kind(self.ctx)).decode("ascii") diff --git a/src/borg/chunkers/rabin_aes_impl.c b/src/borg/chunkers/rabin_aes_impl.c index fc21611f8f..af3ab6d1b1 100644 --- a/src/borg/chunkers/rabin_aes_impl.c +++ b/src/borg/chunkers/rabin_aes_impl.c @@ -66,7 +66,7 @@ uint64_t ra_digest64(const RA_CTX *c, const uint8_t *q) #define PH_DIGEST64 ra_digest64 #include "phte_scan.h" -RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int force_sw) +RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int kernel) { RA_CTX *c = calloc(1, sizeof(RA_CTX)); if (c == NULL) @@ -76,7 +76,7 @@ RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16] memcpy(c->w1_tbl, tables + 2 * 256, sizeof(c->w1_tbl)); memcpy(c->out8_tbl, tables + 3 * 256, sizeof(c->out8_tbl)); memcpy(c->out16_tbl, tables + 4 * 256, sizeof(c->out16_tbl)); - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h index 655c8b206b..5a4fc07e40 100644 --- a/src/borg/chunkers/rabin_aes_impl.h +++ b/src/borg/chunkers/rabin_aes_impl.h @@ -19,6 +19,8 @@ #include #include +#include "phte_kernel.h" + typedef struct RA_CTX RA_CTX; /* Number of 256-entry rolling tables passed to ra_new, in this order: @@ -34,13 +36,13 @@ typedef struct RA_CTX RA_CTX; /* Create a kernel context. * tables: RA_TABLES * 256 uint64 entries, see above. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); +RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel); void ra_free(RA_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *ra_kind(const RA_CTX *ctx); /* Full (non-rolling) Rabin digest of the 64 bytes at q: the window warm-up at diff --git a/src/borg/chunkers/toeplitz_aes.pyx b/src/borg/chunkers/toeplitz_aes.pyx index 482ed4d3db..8bc5ff423c 100644 --- a/src/borg/chunkers/toeplitz_aes.pyx +++ b/src/borg/chunkers/toeplitz_aes.pyx @@ -50,7 +50,11 @@ cdef extern from "toeplitz_aes_impl.h": ctypedef struct TP_CTX: pass int TP_TABLES - TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw) + TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int kernel) + int phte_kernel_select(const char *name, int *out_id) + const char *phte_kernel_names() + int PHTE_K_AUTO + int PHTE_K_EVP void tp_free(TP_CTX *ctx) const char *tp_kind(const TP_CTX *ctx) uint64_t tp_digest64(const TP_CTX *ctx, const uint8_t *q) @@ -107,6 +111,27 @@ def _derive(bytes key): return aes_key, t, tables +from .kernel_env import kernel_error, requested_kernel + + +cdef int _select_kernel() except -1: + """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured. + + One selector for all three AES chunkers: they share phte_scan.h, so the + available paths never differ between them. + """ + cdef int kid = PHTE_K_AUTO + cdef int rc + want = requested_kernel("BORG_AES_CHUNKER_KERNEL") + if want is None: + return PHTE_K_AUTO + rc = phte_kernel_select(want.encode("ascii"), &kid) + if rc != 0: + raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc, + (phte_kernel_names()).decode("ascii")) + return kid + + cdef class ChunkerToeplitzAES(ChunkerPHTE): """ Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision. @@ -130,8 +155,8 @@ cdef class ChunkerToeplitzAES(ChunkerPHTE): for i in range(256): c_tables[t * 256 + i] = tables[t][i] - force_sw = os.environ.get("BORG_TOEPLITZ_AES_FORCE_EVP", "") not in ("", "0") - self.ctx = tp_new(c_tables, aes_key, 1 if force_sw else 0) + kernel = _select_kernel() + self.ctx = tp_new(c_tables, aes_key, kernel) if self.ctx == NULL: raise MemoryError("Failed to set up toeplitz-aes kernel") self.kernel_str = (tp_kind(self.ctx)).decode("ascii") @@ -177,7 +202,7 @@ def toeplitz_aes_digest64(bytes key, bytes window): for i in range(256): c_tables[t * 256 + i] = tables[t][i] memset(aes_key_c, 0, 16) - ctx = tp_new(c_tables, aes_key_c, 1) + ctx = tp_new(c_tables, aes_key_c, PHTE_K_EVP) if ctx == NULL: raise MemoryError("Failed to set up toeplitz-aes kernel") d = tp_digest64(ctx, PyBytes_AsString(window)) diff --git a/src/borg/chunkers/toeplitz_aes_impl.c b/src/borg/chunkers/toeplitz_aes_impl.c index 4e9239bddc..36797d60d3 100644 --- a/src/borg/chunkers/toeplitz_aes_impl.c +++ b/src/borg/chunkers/toeplitz_aes_impl.c @@ -84,7 +84,7 @@ uint64_t tp_digest64(const TP_CTX *c, const uint8_t *q) #define PH_DIGEST64 tp_digest64 #include "phte_scan.h" -TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int force_sw) +TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int kernel) { TP_CTX *c = calloc(1, sizeof(TP_CTX)); if (c == NULL) @@ -93,7 +93,7 @@ TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16] memcpy(c->in1_tbl, tables + 1 * 256, sizeof(c->in1_tbl)); memcpy(c->out64_tbl, tables + 2 * 256, sizeof(c->out64_tbl)); memcpy(c->out65_tbl, tables + 3 * 256, sizeof(c->out65_tbl)); - if (!phte_base_init(&c->base, aes_key, force_sw)) { + if (!phte_base_init(&c->base, aes_key, kernel)) { free(c); return NULL; } diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h index bc005dcf5b..5aa3f61f8b 100644 --- a/src/borg/chunkers/toeplitz_aes_impl.h +++ b/src/borg/chunkers/toeplitz_aes_impl.h @@ -24,6 +24,8 @@ #include #include +#include "phte_kernel.h" + typedef struct TP_CTX TP_CTX; /* Number of 256-entry rolling tables passed to tp_new, in this order: @@ -38,13 +40,13 @@ typedef struct TP_CTX TP_CTX; /* Create a kernel context. * tables: TP_TABLES * 256 uint64 entries, see above. * aes_key: 16 bytes (AES-128). - * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks). + * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run. * Returns NULL on allocation/OpenSSL failure. */ -TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw); +TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel); void tp_free(TP_CTX *ctx); -/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */ +/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */ const char *tp_kind(const TP_CTX *ctx); /* Full (non-rolling) digest of the 64 bytes at q: the window warm-up at the diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py index 452a5b0583..0848e8f9be 100644 --- a/src/borg/testsuite/chunkers/buzhash64_test.py +++ b/src/borg/testsuite/chunkers/buzhash64_test.py @@ -142,24 +142,32 @@ def rnd_key(): assert reconstructed == data -def test_buzhash64_kernels_identical(): - # the blocked/AVX2 scan kernel (auto-selected) and the plain sequential - # loop must produce identical cut points. +ALL_BUZHASH64_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar") + + +def test_buzhash64_kernels_identical(monkeypatch): + # Every scan kernel this platform accepts must produce identical cut points; + # see the fastcdc counterpart. On aarch64 this is what keeps the NEON kernel + # verified - it is selectable but never auto-selected. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) - sizes_default = sizes(default) - os.environ["BORG_BUZHASH64_FORCE_SCALAR"] = "1" - try: - forced = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) - assert forced.kernel == "scalar" - sizes_scalar = sizes(forced) - finally: - del os.environ["BORG_BUZHASH64_FORCE_SCALAR"] - assert sizes_default == sizes_scalar - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx2", "blocked", "scalar") + monkeypatch.delenv("BORG_BUZHASH64_KERNEL", raising=False) + reference = sizes(ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)) + + tested = [] + for name in ALL_BUZHASH64_KERNELS: + monkeypatch.setenv("BORG_BUZHASH64_KERNEL", name) + try: + chunker = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one" + tested.append(name) + + assert "blockwise" in tested and "scalar" in tested diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py index 68812f64ca..9e19870ad9 100644 --- a/src/borg/testsuite/chunkers/fastcdc_test.py +++ b/src/borg/testsuite/chunkers/fastcdc_test.py @@ -178,24 +178,63 @@ def rnd_key(): assert b"".join(parts) == data -def test_fastcdc_kernels_identical(): - # the SIMD scan kernel (neon/avx2/blocked, auto-selected) and the plain - # sequential Gear loop must produce identical cut points. +ALL_FASTCDC_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar") + + +def test_fastcdc_kernels_identical(monkeypatch): + # Every scan kernel this platform accepts must produce identical cut points. + # Kernels this build/CPU cannot run raise and are skipped, so the same test + # covers whatever tier the machine happens to have - including kernels that + # exist but are not auto-selected, which nothing else would exercise. data = os.urandom(4 * 1024 * 1024) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = ChunkerFastCDC(key0, 10, 16, 14, 2) - sizes_default = sizes(default) - os.environ["BORG_FASTCDC_FORCE_SCALAR"] = "1" - try: - forced = ChunkerFastCDC(key0, 10, 16, 14, 2) - assert forced.kernel == "scalar" - sizes_scalar = sizes(forced) - finally: - del os.environ["BORG_FASTCDC_FORCE_SCALAR"] - assert sizes_default == sizes_scalar - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("neon", "avx2", "blocked", "scalar") + monkeypatch.delenv("BORG_FASTCDC_KERNEL", raising=False) + reference = sizes(ChunkerFastCDC(key0, 10, 16, 14, 2)) + + tested = [] + for name in ALL_FASTCDC_KERNELS: + monkeypatch.setenv("BORG_FASTCDC_KERNEL", name) + try: + chunker = ChunkerFastCDC(key0, 10, 16, 14, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one" + tested.append(name) + + # the portable kernels exist everywhere; if they were skipped, the loop is broken + assert "blockwise" in tested and "scalar" in tested + + +@pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"]) +def test_kernel_env_rejects_unusable(envvar, monkeypatch): + # A kernel that cannot run here must fail loudly instead of silently + # falling back - a silent fallback would turn a benchmark, or a CI job that + # means to pin one kernel, into a measurement of a different one. + from ...chunkers import ChunkerBuzHash64, ChunkerToeplitzAES + + make = { + "BORG_FASTCDC_KERNEL": lambda k: ChunkerFastCDC(k, 10, 16, 14, 2), + "BORG_BUZHASH64_KERNEL": lambda k: ChunkerBuzHash64(k, 10, 16, 14, 4095, 2), + "BORG_AES_CHUNKER_KERNEL": lambda k: ChunkerToeplitzAES(k, 10, 16, 14, 2), + }[envvar] + key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") + + monkeypatch.setenv(envvar, "no-such-kernel") + with pytest.raises(ValueError, match="no-such-kernel"): + make(key0) + + # "auto" and an unset var both mean "pick the best one", and always work + monkeypatch.setenv(envvar, "auto") + auto = make(key0).kernel + monkeypatch.delenv(envvar) + assert make(key0).kernel == auto + + # asking for whatever auto-selection picked must yield exactly that + monkeypatch.setenv(envvar, auto) + assert make(key0).kernel == auto diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py index 2ef3f0c219..19b89ffac7 100644 --- a/src/borg/testsuite/chunkers/phte_chunkers_test.py +++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py @@ -23,13 +23,16 @@ # from os.urandom(32) key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da") -# (chunker class, algo name, default params constant, env var forcing the portable kernel) +# all three share one scan-path selector, see phte_kernel.h +KERNEL_ENV = "BORG_AES_CHUNKER_KERNEL" + +# (chunker class, algo name, default params constant) CHUNKERS = [ - (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS, "BORG_RABIN_AES_FORCE_EVP"), - (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS, "BORG_GOLDILOCKS_AES_FORCE_EVP"), - (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS, "BORG_TOEPLITZ_AES_FORCE_EVP"), + (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS), + (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS), + (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS), ] -IDS = [algo for _, algo, _, _ in CHUNKERS] +IDS = [algo for _, algo, _ in CHUNKERS] def H(data): @@ -41,31 +44,40 @@ def chunker_spec(request): return request.param -def test_kernels_identical(chunker_spec): - # the OpenSSL EVP batch path and the AES hardware instruction path (if available - # on this platform) must produce identical cut points. - cls, algo, params, env_var = chunker_spec +ALL_AES_KERNELS = ("auto", "vaes", "aes-ni", "aes-arm64", "evp") + + +def test_kernels_identical(chunker_spec, monkeypatch): + # Every scan path this platform accepts - OpenSSL EVP, the 128-bit AES + # instructions, VAES - must produce identical cut points. Paths this + # build/CPU cannot run raise and are skipped. + cls, algo, params = chunker_spec data = os.urandom(4 * 1024 * 1024) def sizes(chunker): return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))] - default = cls(key0, 10, 16, 14, 2) - sizes_default = sizes(default) - os.environ[env_var] = "1" - try: - forced = cls(key0, 10, 16, 14, 2) - assert forced.kernel == "evp" - sizes_evp = sizes(forced) - finally: - del os.environ[env_var] - assert sizes_default == sizes_evp - # whatever kernel was selected by default, it must be a known one - assert default.kernel in ("aes-arm64", "aes-ni", "evp") + monkeypatch.delenv(KERNEL_ENV, raising=False) + reference = sizes(cls(key0, 10, 16, 14, 2)) + + tested = [] + for name in ALL_AES_KERNELS: + monkeypatch.setenv(KERNEL_ENV, name) + try: + chunker = cls(key0, 10, 16, 14, 2) + except ValueError: + continue # not available on this build/CPU + if name != "auto": + assert chunker.kernel == name + assert sizes(chunker) == reference, f"path {name} disagrees with the default one" + tested.append(name) + + # the portable OpenSSL path exists everywhere + assert "evp" in tested def test_chunksize_distribution(chunker_spec): - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(1048576) min_exp, max_exp, mask, nc_level = 10, 16, 14, 2 # chunk size target 16 KiB, clip at 1 KiB and 64 KiB chunker = cls(key0, min_exp, max_exp, mask, nc_level) @@ -95,7 +107,7 @@ def test_chunksize_distribution(chunker_spec): def test_shift_resilience(chunker_spec): # content-defined cuts must survive a prefix insertion (this also validates that the # rolling digest update and the per-chunk window warm-up agree with each other). - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(4 * 1024 * 1024) def chunk_hashes(data): @@ -110,7 +122,7 @@ def chunk_hashes(data): def test_get_chunker(chunker_spec): # without a key, get_chunker uses an all-zero key; chunking must still work and be deterministic - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec data = os.urandom(2 * 1024 * 1024) a = cf_expand(get_chunker(*params, key=None).chunkify(BytesIO(data))) b = cf_expand(get_chunker(algo, 19, 23, 21, 2, key=None).chunkify(BytesIO(data))) @@ -123,7 +135,7 @@ def test_params_parsing(chunker_spec): from ...helpers import ChunkerParams - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec # , chunk_min, chunk_max, chunk_mask, nc_level (no window field) assert ChunkerParams(f"{algo},19,23,21,2") == (algo, 19, 23, 21, 2) @@ -149,7 +161,7 @@ def test_params_parsing(chunker_spec): @pytest.mark.parametrize("worker", range(os.cpu_count() or 1)) def test_fuzz(chunker_spec, worker): # Fuzz with random and uniform data of misc. sizes and misc keys. - cls, algo, params, env_var = chunker_spec + cls, algo, params = chunker_spec # decompose _PARAMS = (algo, min_exp, max_exp, mask_bits, nc_level) params_algo, min_exp, max_exp, mask_bits, nc_level = params