AES chunkers: unroll the x86-64 aes-ni and vaes scan paths, branch-free lane test

docs/changes.rst

@@ -226,6 +226,9 @@ Other changes:
 - fastcdc: faster blockwise kernel (the candidate test as a short-circuit chain), 1.75x
   on an Apple M3 Pro and 3x on a Zen 4 with gcc -O2; it is the default kernel on
   platforms other than x86-64 and aarch64
+- toeplitz-aes, rabin-aes, goldilocks-aes: faster x86-64 scan paths, the AES rounds
+  and roll steps are unrolled and the lane test is branch-free; on a Zen 4 with gcc -O2
+  aes-ni +7..+30%, vaes +5..+10%
 - archive: resolve the item metadata stream chunk ids lazily, big win for repo-list
   on remote repos, #10204
 - lock exceptions: tell who holds the lock, #2261

src/borg/chunkers/phte_core.h

@@ -150,7 +150,8 @@ PHTE_HW_TARGET static inline uint8x16_t phte_aes1_neon(const uint8x16_t k[11], u
 
 static int phte_hw_available(void)
 {
-    return __builtin_cpu_supports("aes") && __builtin_cpu_supports("sse2");
+    /* SSE4.1 for the scan's pcmpeqq; every CPU with AES-NI has it */
+    return __builtin_cpu_supports("aes") && __builtin_cpu_supports("sse4.1");
 }
 
 __attribute__((target("aes,sse2"))) static inline __m128i

src/borg/chunkers/phte_scan.h

@@ -246,13 +246,27 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t
 
 #elif PHTE_HAVE_HW
 
-__attribute__((target("aes,sse2"))) static int64_t
+/* apply one AES round (or the last one) to the 8 blocks in flight */
+#define PH_R8(op, k) \
+    b0 = op(b0, k); b1 = op(b1, k); b2 = op(b2, k); b3 = op(b3, k); \
+    b4 = op(b4, k); b5 = op(b5, k); b6 = op(b6, k); b7 = op(b7, k)
+
+/* The AES rounds and the 8-way test are spelled out rather than looped:
+ * gcc at -O2 keeps such loops rolled (12 instructions per round, two
+ * branches per tested position) and clang serialises the early-exit test
+ * loop into 8 dependent chains; both cost about a fifth of the path's time
+ * on a Zen 4. The test is branch-free in the vector domain: the 8 low
+ * ciphertext qwords are packed pairwise (unpcklqdq), masked, compared to
+ * zero (pcmpeqq, SSE4.1 - every CPU with AES-NI has it), or-reduced, and
+ * only a hit enters the lane search. */
+__attribute__((target("aes,sse4.1"))) static int64_t
 PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
 {
     uint64_t d = *digest, da, db;
     const uint8_t *qo = p - 64;
     size_t i = 0;
     int lanes_live = 0;
+    const __m128i M = _mm_set1_epi64x((long long)mask), Z = _mm_setzero_si128();
 
     if (n >= 10) {
         db = PH_ROLL(c, d, qo[0], p[0]);               /* d_0 */
@@ -278,50 +292,38 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t
         /* 8 blocks in flight; round keys are re-loaded per round (they stay
          * hot in L1) to keep register pressure within the 16 XMM registers */
         kr = _mm_loadu_si128((const __m128i *)c->base.rk[0]);
-        b0 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[0]), kr);
-        b1 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[1]), kr);
-        b2 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[2]), kr);
-        b3 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[3]), kr);
-        b4 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[4]), kr);
-        b5 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[5]), kr);
-        b6 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[6]), kr);
-        b7 = _mm_xor_si128(_mm_set_epi64x(0, (long long)dg[7]), kr);
-        for (int r = 1; r < 10; r++) {
-            kr = _mm_loadu_si128((const __m128i *)c->base.rk[r]);
-            b0 = _mm_aesenc_si128(b0, kr);
-            b1 = _mm_aesenc_si128(b1, kr);
-            b2 = _mm_aesenc_si128(b2, kr);
-            b3 = _mm_aesenc_si128(b3, kr);
-            b4 = _mm_aesenc_si128(b4, kr);
-            b5 = _mm_aesenc_si128(b5, kr);
-            b6 = _mm_aesenc_si128(b6, kr);
-            b7 = _mm_aesenc_si128(b7, kr);
-        }
+        b0 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[0]), kr);
+        b1 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[1]), kr);
+        b2 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[2]), kr);
+        b3 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[3]), kr);
+        b4 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[4]), kr);
+        b5 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[5]), kr);
+        b6 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[6]), kr);
+        b7 = _mm_xor_si128(_mm_cvtsi64_si128((long long)dg[7]), kr);
+#define PH_ROUND(r) \
+        kr = _mm_loadu_si128((const __m128i *)c->base.rk[r]); \
+        PH_R8(_mm_aesenc_si128, kr)
+        PH_ROUND(1); PH_ROUND(2); PH_ROUND(3); PH_ROUND(4); PH_ROUND(5);
+        PH_ROUND(6); PH_ROUND(7); PH_ROUND(8); PH_ROUND(9);
+#undef PH_ROUND
         kr = _mm_loadu_si128((const __m128i *)c->base.rk[10]);
-        b0 = _mm_aesenclast_si128(b0, kr);
-        b1 = _mm_aesenclast_si128(b1, kr);
-        b2 = _mm_aesenclast_si128(b2, kr);
-        b3 = _mm_aesenclast_si128(b3, kr);
-        b4 = _mm_aesenclast_si128(b4, kr);
-        b5 = _mm_aesenclast_si128(b5, kr);
-        b6 = _mm_aesenclast_si128(b6, kr);
-        b7 = _mm_aesenclast_si128(b7, kr);
+        PH_R8(_mm_aesenclast_si128, kr);
 
         {
-            uint64_t cs[8];
-            cs[0] = (uint64_t)_mm_cvtsi128_si64(b0);
-            cs[1] = (uint64_t)_mm_cvtsi128_si64(b1);
-            cs[2] = (uint64_t)_mm_cvtsi128_si64(b2);
-            cs[3] = (uint64_t)_mm_cvtsi128_si64(b3);
-            cs[4] = (uint64_t)_mm_cvtsi128_si64(b4);
-            cs[5] = (uint64_t)_mm_cvtsi128_si64(b5);
-            cs[6] = (uint64_t)_mm_cvtsi128_si64(b6);
-            cs[7] = (uint64_t)_mm_cvtsi128_si64(b7);
-            for (int j = 0; j < 8; j++) {
-                if ((cs[j] & mask) == 0) {
-                    *digest = dg[j];
-                    return (int64_t)(i + j);
-                }
+            __m128i e01 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b0, b1), M), Z);
+            __m128i e23 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b2, b3), M), Z);
+            __m128i e45 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b4, b5), M), Z);
+            __m128i e67 = _mm_cmpeq_epi64(_mm_and_si128(_mm_unpacklo_epi64(b6, b7), M), Z);
+            __m128i any = _mm_or_si128(_mm_or_si128(e01, e23), _mm_or_si128(e45, e67));
+            if (_mm_movemask_epi8(any)) {
+                /* bit j of hits = position i + j hit; the lowest one wins */
+                unsigned hits = (unsigned)_mm_movemask_pd(_mm_castsi128_pd(e01)) |
+                                ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e23)) << 2) |
+                                ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e45)) << 4) |
+                                ((unsigned)_mm_movemask_pd(_mm_castsi128_pd(e67)) << 6);
+                int j = __builtin_ctz(hits);
+                *digest = dg[j];
+                return (int64_t)(i + (size_t)j);
             }
         }
         i += 8;
@@ -398,38 +400,35 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6
         __m512i b0, b1, b2, b3, b4, b5, b6, b7;
         __mmask8 h0, h1, h2, h3, h4, h5, h6, h7;
 
+        /* The 15 stride-2 roll steps and the 9 AES rounds are spelled out by
+         * macro: gcc at -O2 keeps them as loops, which for the rounds means
+         * 72 zmm register copies per group and for the rolls an index chain
+         * (Zen 4, toeplitz-aes: 7.06 -> 6.40 cycles per byte unrolled). */
         dgs[0] = db;
         dgs[2] = da;
-        for (int j = 2; j < 32; j += 2) {
-            db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]);
-            dgs[2 * j] = db;
-            da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]);
-            dgs[2 * j + 2] = da;
-        }
+#define PH_STEP(j) \
+        db = PH_ROLL2(c, db, qo[i + (j) - 1], qo[i + (j)], p[i + (j) - 1], p[i + (j)]); dgs[2 * (j)] = db; \
+        da = PH_ROLL2(c, da, qo[i + (j)], qo[i + (j) + 1], p[i + (j)], p[i + (j) + 1]); dgs[2 * (j) + 2] = da
+        PH_STEP(2); PH_STEP(4); PH_STEP(6); PH_STEP(8); PH_STEP(10); PH_STEP(12); PH_STEP(14); PH_STEP(16);
+        PH_STEP(18); PH_STEP(20); PH_STEP(22); PH_STEP(24); PH_STEP(26); PH_STEP(28); PH_STEP(30);
+#undef PH_STEP
         /* prepare the next group's invariant (digests at i+32, i+33) */
         db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]);
         da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]);
 
         /* each vector's 4 blocks are already laid out in dgs (digest in the
          * low qword of every block, zero in the high one) */
-        b0 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 0)), k[0]);
-        b1 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 8)), k[0]);
-        b2 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 16)), k[0]);
-        b3 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 24)), k[0]);
-        b4 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 32)), k[0]);
-        b5 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 40)), k[0]);
-        b6 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 48)), k[0]);
-        b7 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 56)), k[0]);
-        for (int r = 1; r < 10; r++) {
-            b0 = _mm512_aesenc_epi128(b0, k[r]);
-            b1 = _mm512_aesenc_epi128(b1, k[r]);
-            b2 = _mm512_aesenc_epi128(b2, k[r]);
-            b3 = _mm512_aesenc_epi128(b3, k[r]);
-            b4 = _mm512_aesenc_epi128(b4, k[r]);
-            b5 = _mm512_aesenc_epi128(b5, k[r]);
-            b6 = _mm512_aesenc_epi128(b6, k[r]);
-            b7 = _mm512_aesenc_epi128(b7, k[r]);
-        }
+        b0 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 0)), k[0]);
+        b1 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 8)), k[0]);
+        b2 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 16)), k[0]);
+        b3 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 24)), k[0]);
+        b4 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 32)), k[0]);
+        b5 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 40)), k[0]);
+        b6 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 48)), k[0]);
+        b7 = _mm512_xor_si512(_mm512_load_si512((const void *)(dgs + 56)), k[0]);
+        PH_R8(_mm512_aesenc_epi128, k[1]); PH_R8(_mm512_aesenc_epi128, k[2]); PH_R8(_mm512_aesenc_epi128, k[3]);
+        PH_R8(_mm512_aesenc_epi128, k[4]); PH_R8(_mm512_aesenc_epi128, k[5]); PH_R8(_mm512_aesenc_epi128, k[6]);
+        PH_R8(_mm512_aesenc_epi128, k[7]); PH_R8(_mm512_aesenc_epi128, k[8]); PH_R8(_mm512_aesenc_epi128, k[9]);
         h0 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b0, k[10]), M);
         h1 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b1, k[10]), M);
         h2 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b2, k[10]), M);