diff --git a/docs/changes.rst b/docs/changes.rst index 8af6211e2f..133455c9e2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -193,6 +193,11 @@ Fixes: - borgfs: adopt the "mount:" config file section - let Ctrl-C / SIGINT abort interactive prompts (y/n and passphrase), #8521 - goldilocks-aes: fix build on 32-bit archs (no __uint128_t there) +- toeplitz-aes, rabin-aes, goldilocks-aes: use the AES instructions on Linux and + FreeBSD arm64. The hardware scan path was only built when the compiler targeted + the crypto extension as a whole (Apple's default does, a Linux/BSD python + extension build does not), so those builds silently ran the 1.5-1.9x slower + portable OpenSSL path and rejected BORG_AES_CHUNKER_KERNEL=aes-arm64. - repo-list: fix --format help, it did not show the actual default, #10204 - windows: diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h index d4d97d7842..5a9dbbe4b4 100644 --- a/src/borg/chunkers/phte_core.h +++ b/src/borg/chunkers/phte_core.h @@ -86,28 +86,56 @@ static void phte_aes128_expand(const uint8_t key[16], uint8_t rk[11][16]) /* --- hardware AES availability and single-block helpers ---------------- */ -#if defined(__aarch64__) && defined(__ARM_FEATURE_AES) +/* aarch64: the AES intrinsics are only usable in a function compiled with the + * crypto extension enabled. Python extension builds get no -march flags, so on + * Linux and the BSDs they target the armv8-a baseline, which does not include + * it (__ARM_FEATURE_AES is undefined there); only Apple's default target has + * it. So the extension is enabled per function via a target attribute (gcc >= + * 6 / clang >= 14), the same way the x86-64 path uses target("aes,sse2"), and + * whether this CPU actually has the instructions is decided at run time by + * phte_hw_available(). A build that targets +crypto as a whole needs neither. */ +#if defined(__aarch64__) && \ + (defined(__ARM_FEATURE_AES) || (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 6) || \ + (defined(__clang__) && __clang_major__ >= 14)) #define PHTE_HAVE_HW 1 #define PHTE_KIND_HW "aes-arm64" #include -#if defined(__linux__) -#include -#ifndef HWCAP_AES +#if defined(__linux__) || defined(__FreeBSD__) +#include /* getauxval resp. elf_aux_info; FreeBSD's also brings HWCAP_AES */ +#endif +#if (defined(__linux__) || defined(__FreeBSD__)) && !defined(HWCAP_AES) #define HWCAP_AES (1 << 3) #endif + +#if defined(__ARM_FEATURE_AES) +#define PHTE_HW_TARGET /* the whole build targets +crypto already */ +#else +#define PHTE_HW_TARGET __attribute__((target("+crypto"))) #endif +/* Whether this CPU has the AES instructions. Where the OS cannot be asked, + * only a build that targets +crypto as a whole may assume so (it would not + * run on a lesser CPU anyway); everything else stays on the portable path. + * TODO: NetBSD (machdep.cpuN.cpu_id sysctl) and OpenBSD (elf_aux_info since + * 7.6) could be asked too. */ static int phte_hw_available(void) { -#if defined(__linux__) +#if defined(__APPLE__) + return 1; /* every Apple Silicon CPU has the crypto extension */ +#elif defined(__linux__) return (getauxval(AT_HWCAP) & HWCAP_AES) != 0; +#elif defined(__FreeBSD__) + unsigned long hwcap = 0; + return elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0 && (hwcap & HWCAP_AES) != 0; +#elif defined(__ARM_FEATURE_AES) + return 1; #else - return 1; /* compiler targeted +aes; on Apple Silicon it is always there */ + return 0; #endif } -static inline uint8x16_t phte_aes1_neon(const uint8x16_t k[11], uint8x16_t b) +PHTE_HW_TARGET static inline uint8x16_t phte_aes1_neon(const uint8x16_t k[11], uint8x16_t b) { for (int r = 0; r < 9; r++) b = vaesmcq_u8(vaeseq_u8(b, k[r])); @@ -186,6 +214,9 @@ int phte_kernel_select(const char *name, int *out_id) *out_id = PHTE_K_HW; return PHTE_KSEL_OK; } +#elif defined(__aarch64__) + if (strcmp(name, "aes-arm64") == 0) + return PHTE_KSEL_NOTBUILT; /* compiler too old for the target attribute */ #endif #if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) if (strcmp(name, "vaes") == 0) { diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h index 1362658099..e0b18ae9b7 100644 --- a/src/borg/chunkers/phte_scan.h +++ b/src/borg/chunkers/phte_scan.h @@ -121,7 +121,8 @@ static int64_t PH_FN(scan_evp)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t * #if PHTE_HAVE_HW && defined(__aarch64__) -static int64_t PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) +PHTE_HW_TARGET static int64_t +PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) { uint8x16_t k[11]; uint64_t d = *digest, da, db; diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py index 7279f8890c..d032d20fd4 100644 --- a/src/borg/testsuite/chunkers/phte_chunkers_test.py +++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py @@ -10,6 +10,7 @@ from hashlib import sha256 from io import BytesIO import os +import platform import random import pytest @@ -75,6 +76,34 @@ def test_evp_kernel_available(chunker_spec, monkeypatch): assert cls(key0, 10, 16, 14, 2).kernel == "evp" +def aarch64_cpu_has_aes(): + """Whether this aarch64 CPU has the AES instructions, or None if there is no way to tell here.""" + system = platform.system() + if system == "Darwin": + return True # every Apple Silicon CPU has the crypto extension + if system == "Linux": + with open("/proc/cpuinfo") as f: + for line in f: + if line.startswith("Features"): + return "aes" in line.split(":", 1)[1].split() + return None + + +@pytest.mark.skipif(platform.machine() not in ("aarch64", "arm64"), reason="aarch64 only") +def test_aarch64_hw_kernel_is_default(chunker_spec, monkeypatch): + # The hardware path must be part of every aarch64 build, not only of those + # whose compiler targets +crypto as a whole (Apple's default target does, a + # Linux/BSD python extension build does not), and it must be the default + # wherever the CPU has the AES instructions. Regression test: Linux arm64 + # builds used to run the portable EVP path silently. + cls, algo, params = chunker_spec + has_aes = aarch64_cpu_has_aes() + if has_aes is None: + pytest.skip("cannot tell whether this CPU has the AES instructions") + monkeypatch.delenv(KERNEL_ENV, raising=False) + assert cls(key0, 10, 16, 14, 2).kernel == ("aes-arm64" if has_aes else "evp") + + def test_chunksize_distribution(chunker_spec): cls, algo, params = chunker_spec data = os.urandom(1048576)