From 1712cfee4ec6059a85e66b794c89df3733bff71e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 2 Sep 2026 02:13:27 +0200 Subject: [PATCH 1/2] AES chunkers: unroll the aes-ni scan path, test the 8 lanes branch-free The 128-bit hardware path (aes-ni, i.e. every x86-64 CPU without VAES) looped over the 9 AES rounds and over the 8 ciphertexts to test. gcc at -O2 - Debian's python flags - keeps both loops rolled: 12 instructions per round instead of 8, and two branches per tested position; clang serialises the early-exit test loop into 8 dependent 40-cycle AES chains. Both cost about a fifth of the path's time on a Zen 4. Spell the rounds out by macro and test all 8 lanes at once in the vector domain: pack the low ciphertext qwords pairwise, mask, compare with zero (pcmpeqq, SSE4.1 - every CPU with AES-NI has it), or-reduce, and only a hit enters the lane search. phte_hw_available() now checks SSE4.1 too. Zen 4, gcc 14 -O2, borg benchmark cpu with BORG_AES_CHUNKER_KERNEL=aes-ni: toeplitz-aes 735 -> 960 MB/s, rabin-aes 720 -> 870, goldilocks-aes 472 -> 505. Cut points are unchanged (the hw/vaes/evp equivalence tests pass). Co-Authored-By: Claude Fable 5.1 --- src/borg/chunkers/phte_core.h | 3 +- src/borg/chunkers/phte_scan.h | 86 ++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h index 5a9dbbe4b4..1df426b221 100644 --- a/src/borg/chunkers/phte_core.h +++ b/src/borg/chunkers/phte_core.h @@ -150,7 +150,8 @@ PHTE_HW_TARGET static inline uint8x16_t phte_aes1_neon(const uint8x16_t k[11], u static int phte_hw_available(void) { - return __builtin_cpu_supports("aes") && __builtin_cpu_supports("sse2"); + /* SSE4.1 for the scan's pcmpeqq; every CPU with AES-NI has it */ + return __builtin_cpu_supports("aes") && __builtin_cpu_supports("sse4.1"); } __attribute__((target("aes,sse2"))) static inline __m128i diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 4377cb160a..e083a8409d 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -246,13 +246,27 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t #elif PHTE_HAVE_HW -__attribute__((target("aes,sse2"))) static int64_t +/* apply one AES round (or the last one) to the 8 blocks in flight */ +#define PH_R8(op, k) \ + b0 = op(b0, k); b1 = op(b1, k); b2 = op(b2, k); b3 = op(b3, k); \ + b4 = op(b4, k); b5 = op(b5, k); b6 = op(b6, k); b7 = op(b7, k) + +/* The AES rounds and the 8-way test are spelled out rather than looped: + * gcc at -O2 keeps such loops rolled (12 instructions per round, two + * branches per tested position) and clang serialises the early-exit test + * loop into 8 dependent chains; both cost about a fifth of the path's time + * on a Zen 4. The test is branch-free in the vector domain: the 8 low + * ciphertext qwords are packed pairwise (unpcklqdq), masked, compared to + * zero (pcmpeqq, SSE4.1 - every CPU with AES-NI has it), or-reduced, and + * only a hit enters the lane search. */ +__attribute__((target("aes,sse4.1"))) static int64_t PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) { uint64_t d = *digest, da, db; const uint8_t *qo = p - 64; size_t i = 0; int lanes_live = 0; + const __m128i M = _mm_set1_epi64x((long long)mask), Z = _mm_setzero_si128(); if (n >= 10) { db = PH_ROLL(c, d, qo[0], p[0]); /* d_0 */ @@ -278,50 +292,38 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t /* 8 blocks in flight; round keys are re-loaded per round (they stay * hot in L1) to keep register pressure within the 16 XMM registers */ kr = _mm_loadu_si128((const __m128i *)c->base.rk[0]); - b0 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[0]), kr); - b1 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[1]), kr); - b2 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[2]), kr); - b3 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[3]), kr); - b4 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[4]), kr); - b5 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[5]), kr); - b6 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[6]), kr); - b7 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[7]), kr); - for (int r = 1; r < 10; r++) { - kr = _mm_loadu_si128((const __m128i *)c->base.rk[r]); - b0 = _mm_aesenc_si128(b0, kr); - b1 = _mm_aesenc_si128(b1, kr); - b2 = _mm_aesenc_si128(b2, kr); - b3 = _mm_aesenc_si128(b3, kr); - b4 = _mm_aesenc_si128(b4, kr); - b5 = _mm_aesenc_si128(b5, kr); - b6 = _mm_aesenc_si128(b6, kr); - b7 = _mm_aesenc_si128(b7, kr); - } + b0 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[0]), kr); + b1 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[1]), kr); + b2 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[2]), kr); + b3 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[3]), kr); + b4 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[4]), kr); + b5 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[5]), kr); + b6 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[6]), kr); + b7 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[7]), kr); +#define PH_ROUND(r) \ + kr = _mm_loadu_si128((const __m128i *)c->base.rk[r]); \ + PH_R8(_mm_aesenc_si128, kr) + PH_ROUND(1); PH_ROUND(2); PH_ROUND(3); PH_ROUND(4); PH_ROUND(5); + PH_ROUND(6); PH_ROUND(7); PH_ROUND(8); PH_ROUND(9); +#undef PH_ROUND kr = _mm_loadu_si128((const __m128i *)c->base.rk[10]); - b0 = _mm_aesenclast_si128(b0, kr); - b1 = _mm_aesenclast_si128(b1, kr); - b2 = _mm_aesenclast_si128(b2, kr); - b3 = _mm_aesenclast_si128(b3, kr); - b4 = _mm_aesenclast_si128(b4, kr); - b5 = _mm_aesenclast_si128(b5, kr); - b6 = _mm_aesenclast_si128(b6, kr); - b7 = _mm_aesenclast_si128(b7, kr); + PH_R8(_mm_aesenclast_si128, kr); { - uint64_t cs[8]; - cs[0] = (uint64_t)_mm_cvtsi128_si64(b0); - cs[1] = (uint64_t)_mm_cvtsi128_si64(b1); - cs[2] = (uint64_t)_mm_cvtsi128_si64(b2); - cs[3] = (uint64_t)_mm_cvtsi128_si64(b3); - cs[4] = (uint64_t)_mm_cvtsi128_si64(b4); - cs[5] = (uint64_t)_mm_cvtsi128_si64(b5); - cs[6] = (uint64_t)_mm_cvtsi128_si64(b6); - cs[7] = (uint64_t)_mm_cvtsi128_si64(b7); - for (int j = 0; j < 8; j++) { - if ((cs[j] & mask) == 0) { - *digest = dg[j]; - return (int64_t)(i + j); - } + __m128i e01 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b0, b1), M), Z); + __m128i e23 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b2, b3), M), Z); + __m128i e45 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b4, b5), M), Z); + __m128i e67 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b6, b7), M), Z); + __m128i any = _mm_or_si128(_mm_or_si128(e01, e23), _mm_or_si128(e45, e67)); + if (_mm_movemask_epi8(any)) { + /* bit j of hits = position i + j hit; the lowest one wins */ + unsigned hits = (unsigned)_mm_movemask_pd(_mm_castsi128_pd(e01)) | + ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e23)) << 2) | + ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e45)) << 4) | + ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e67)) << 6); + int j = __builtin_ctz(hits); + *digest = dg[j]; + return (int64_t)(i + (size_t)j); } } i += 8; From 9d831da9daa4c0e37e55f9c58341ffdf83563fbf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 2 Sep 2026 02:16:02 +0200 Subject: [PATCH 2/2] AES chunkers: unroll the vaes scan path's roll steps and AES rounds gcc at -O2 - Debian's python flags - keeps the VAES/AVX-512 path's two loops rolled: the 15 stride-2 roll steps run with an index chain and the 9 AES rounds over 8 zmm vectors come with 72 zmm register copies per group. Spelled out by macro (like the aes-ni path now), the group loop is straight-line code. Zen 4, gcc 14 -O2, borg benchmark cpu with BORG_AES_CHUNKER_KERNEL=vaes: toeplitz-aes 871 -> 1000 MB/s, rabin-aes 879 -> ~1010, goldilocks-aes 489 -> ~555. At the kernel level toeplitz-aes goes from 7.06 to 6.40 cycles per byte. Cut points are unchanged (the hw/vaes/evp equivalence tests pass). Co-Authored-By: Claude Fable 5.1 --- docs/changes.rst | 3 +++ src/borg/chunkers/phte_scan.h | 45 ++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 1f757d1c0b..7651f1b8a5 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -226,6 +226,9 @@ Other changes: - fastcdc: faster blockwise kernel (the candidate test as a short-circuit chain), 1.75x on an Apple M3 Pro and 3x on a Zen 4 with gcc -O2; it is the default kernel on platforms other than x86-64 and aarch64 +- toeplitz-aes, rabin-aes, goldilocks-aes: faster x86-64 scan paths, the AES rounds + and roll steps are unrolled and the lane test is branch-free; on a Zen 4 with gcc -O2 + aes-ni +7..+30%, vaes +5..+10% - archive: resolve the item metadata stream chunk ids lazily, big win for repo-list on remote repos, #10204 - lock exceptions: tell who holds the lock, #2261 diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index e083a8409d..47440c3c99 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -400,38 +400,35 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6 __m512i b0, b1, b2, b3, b4, b5, b6, b7; __mmask8 h0, h1, h2, h3, h4, h5, h6, h7; + /* The 15 stride-2 roll steps and the 9 AES rounds are spelled out by + * macro: gcc at -O2 keeps them as loops, which for the rounds means + * 72 zmm register copies per group and for the rolls an index chain + * (Zen 4, toeplitz-aes: 7.06 -> 6.40 cycles per byte unrolled). */ dgs[0] = db; dgs[2] = da; - for (int j = 2; j < 32; j += 2) { - db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]); - dgs[2 * j] = db; - da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]); - dgs[2 * j + 2] = da; - } +#define PH_STEP(j) \ + db = PH_ROLL2(c, db, qo[i + (j) - 1], qo[i + (j)], p[i + (j) - 1], p[i + (j)]); dgs[2 * (j)] = db; \ + da = PH_ROLL2(c, da, qo[i + (j)], qo[i + (j) + 1], p[i + (j)], p[i + (j) + 1]); dgs[2 * (j) + 2] = da + PH_STEP(2); PH_STEP(4); PH_STEP(6); PH_STEP(8); PH_STEP(10); PH_STEP(12); PH_STEP(14); PH_STEP(16); + PH_STEP(18); PH_STEP(20); PH_STEP(22); PH_STEP(24); PH_STEP(26); PH_STEP(28); PH_STEP(30); +#undef PH_STEP /* prepare the next group's invariant (digests at i+32, i+33) */ db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]); da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]); /* each vector's 4 blocks are already laid out in dgs (digest in the * low qword of every block, zero in the high one) */ - b0 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 0)), k[0]); - b1 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 8)), k[0]); - b2 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 16)), k[0]); - b3 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 24)), k[0]); - b4 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 32)), k[0]); - b5 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 40)), k[0]); - b6 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 48)), k[0]); - b7 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 56)), k[0]); - for (int r = 1; r < 10; r++) { - b0 = _mm512_aesenc_epi128(b0, k[r]); - b1 = _mm512_aesenc_epi128(b1, k[r]); - b2 = _mm512_aesenc_epi128(b2, k[r]); - b3 = _mm512_aesenc_epi128(b3, k[r]); - b4 = _mm512_aesenc_epi128(b4, k[r]); - b5 = _mm512_aesenc_epi128(b5, k[r]); - b6 = _mm512_aesenc_epi128(b6, k[r]); - b7 = _mm512_aesenc_epi128(b7, k[r]); - } + b0 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 0)), k[0]); + b1 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 8)), k[0]); + b2 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 16)), k[0]); + b3 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 24)), k[0]); + b4 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 32)), k[0]); + b5 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 40)), k[0]); + b6 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 48)), k[0]); + b7 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 56)), k[0]); + PH_R8(_mm512_aesenc_epi128, k[1]); PH_R8(_mm512_aesenc_epi128, k[2]); PH_R8(_mm512_aesenc_epi128, k[3]); + PH_R8(_mm512_aesenc_epi128, k[4]); PH_R8(_mm512_aesenc_epi128, k[5]); PH_R8(_mm512_aesenc_epi128, k[6]); + PH_R8(_mm512_aesenc_epi128, k[7]); PH_R8(_mm512_aesenc_epi128, k[8]); PH_R8(_mm512_aesenc_epi128, k[9]); h0 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b0, k[10]), M); h1 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b1, k[10]), M); h2 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b2, k[10]), M);