Add AES-256 IGE decrypt helper with ARMv8 fast path

aes256_decrypt.go

@@ -0,0 +1,35 @@
+package ige
+
+import "crypto/aes"
+
+// DecryptAES256Blocks decrypts src using AES-256 in IGE mode and writes the
+// result to dst. It uses an architecture-specific implementation when one is
+// available and otherwise falls back to crypto/aes and DecryptBlocks.
+//
+// The key must be exactly 32 bytes. The IV must contain two AES blocks. Src
+// must contain whole AES blocks and dst must be at least as long as src. Dst
+// and src must not overlap.
+func DecryptAES256Blocks(key, iv, dst, src []byte) {
+	if len(key) != 32 {
+		panic(aes.KeySizeError(len(key)))
+	}
+	if len(iv) != aes.BlockSize*2 {
+		panic(ErrInvalidIV)
+	}
+	if len(src)%aes.BlockSize != 0 {
+		panic("src not full blocks")
+	}
+	if len(dst) < len(src) {
+		panic("len(dst) < len(src)")
+	}
+
+	if decryptAES256BlocksHardware(key, iv, dst, src) {
+		return
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		panic(err)
+	}
+	DecryptBlocks(block, iv, dst, src)
+}

aes256_decrypt_arm.go

@@ -0,0 +1,33 @@
+//go:build arm && cgo && (linux || android)
+// +build arm
+// +build cgo
+// +build linux android
+
+package ige
+
+/*
+#cgo CFLAGS: -O3 -march=armv8-a+crypto -mfpu=crypto-neon-fp-armv8
+#cgo LDFLAGS: -ldl
+#include <stddef.h>
+#include <stdint.h>
+
+int aes256_ige_decrypt(const uint8_t *key, const uint8_t *iv,
+                       uint8_t *dst, const uint8_t *src, size_t len);
+*/
+import "C"
+import "unsafe"
+
+func decryptAES256BlocksHardware(key, iv, dst, src []byte) bool {
+	if len(src) == 0 {
+		return false
+	}
+
+	rc := C.aes256_ige_decrypt(
+		(*C.uint8_t)(unsafe.Pointer(&key[0])),
+		(*C.uint8_t)(unsafe.Pointer(&iv[0])),
+		(*C.uint8_t)(unsafe.Pointer(&dst[0])),
+		(*C.uint8_t)(unsafe.Pointer(&src[0])),
+		C.size_t(len(src)),
+	)
+	return rc == 0
+}

aes256_decrypt_arm_test.go

@@ -0,0 +1,48 @@
+//go:build arm && cgo && (linux || android)
+// +build arm
+// +build cgo
+// +build linux android
+
+package ige
+
+import (
+	"bytes"
+	"crypto/aes"
+	"testing"
+)
+
+func TestDecryptAES256BlocksHardware(t *testing.T) {
+	key, iv, ciphertext := aes256IGEDecryptTestInput(16 * 16)
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := make([]byte, len(ciphertext))
+	DecryptBlocks(block, iv, want, ciphertext)
+
+	got := make([]byte, len(ciphertext))
+	if !decryptAES256BlocksHardware(key, iv, got, ciphertext) {
+		t.Skip("ARMv8 AES extension is not available")
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatal("hardware AES-256 IGE decrypt result differs from DecryptBlocks")
+	}
+}
+
+func BenchmarkDecryptAES256BlocksHardware(b *testing.B) {
+	key, iv, ciphertext := aes256IGEDecryptTestInput(1024 * 1024)
+	plaintext := make([]byte, len(ciphertext))
+	if !decryptAES256BlocksHardware(key, iv, plaintext, ciphertext) {
+		b.Skip("ARMv8 AES extension is not available")
+	}
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(ciphertext)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !decryptAES256BlocksHardware(key, iv, plaintext, ciphertext) {
+			b.Fatal("ARMv8 AES extension became unavailable")
+		}
+	}
+}

aes256_decrypt_stub.go

@@ -0,0 +1,8 @@
+//go:build !arm || !cgo || (!linux && !android)
+// +build !arm !cgo !linux,!android
+
+package ige
+
+func decryptAES256BlocksHardware(key, iv, dst, src []byte) bool {
+	return false
+}

aes256_decrypt_test.go

@@ -0,0 +1,40 @@
+package ige
+
+import (
+	"bytes"
+	"crypto/aes"
+	"testing"
+)
+
+func TestDecryptAES256Blocks(t *testing.T) {
+	key, iv, ciphertext := aes256IGEDecryptTestInput(16 * 16)
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := make([]byte, len(ciphertext))
+	DecryptBlocks(block, iv, want, ciphertext)
+
+	got := make([]byte, len(ciphertext))
+	DecryptAES256Blocks(key, iv, got, ciphertext)
+	if !bytes.Equal(got, want) {
+		t.Fatal("AES-256 IGE decrypt result differs from DecryptBlocks")
+	}
+}
+
+func aes256IGEDecryptTestInput(size int) (key, iv, ciphertext []byte) {
+	key = make([]byte, 32)
+	iv = make([]byte, 32)
+	ciphertext = make([]byte, size)
+	for i := range key {
+		key[i] = byte(i*3 + 1)
+	}
+	for i := range iv {
+		iv[i] = byte(i*5 + 2)
+	}
+	for i := range ciphertext {
+		ciphertext[i] = byte(i*7 + 3)
+	}
+	return key, iv, ciphertext
+}

aes256_ige_arm.c

@@ -0,0 +1,134 @@
+//go:build arm && cgo && (linux || android)
+// +build arm
+// +build cgo
+// +build linux android
+
+// AES-256 IGE decrypt using ARMv8 Cryptography Extension on AArch32.
+#include <arm_neon.h>
+#include <dlfcn.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifndef AT_HWCAP2
+#define AT_HWCAP2 26
+#endif
+#ifndef HWCAP2_AES
+#define HWCAP2_AES (1 << 0)
+#endif
+
+static int aes_hw_available(void) {
+    typedef unsigned long (*getauxval_fn)(unsigned long);
+    const char *libc_name =
+#if defined(__ANDROID__)
+        "libc.so";
+#else
+        "libc.so.6";
+#endif
+    void *libc = dlopen(libc_name, RTLD_LAZY | RTLD_LOCAL);
+    if (libc == NULL) return 0;
+    getauxval_fn getauxval = (getauxval_fn)dlsym(libc, "getauxval");
+    if (getauxval == NULL) {
+        dlclose(libc);
+        return 0;
+    }
+    unsigned long hw2 = getauxval(AT_HWCAP2);
+    dlclose(libc);
+    return (hw2 & HWCAP2_AES) ? 1 : 0;
+}
+
+static uint8_t gf_mul(uint8_t a, uint8_t b) {
+    uint8_t result = 0;
+    for (int i = 0; i < 8; i++) {
+        uint8_t select = (uint8_t)-(int)(b & 1);
+        result ^= a & select;
+        uint8_t high = a >> 7;
+        a = (uint8_t)((a << 1) ^ ((uint8_t)-(int)high & 0x1b));
+        b >>= 1;
+    }
+    return result;
+}
+
+static uint8_t rotate_left(uint8_t value, int shift) {
+    return (uint8_t)((value << shift) | (value >> (8 - shift)));
+}
+
+static uint8_t aes_sbox(uint8_t value) {
+    uint8_t value2 = gf_mul(value, value);
+    uint8_t value4 = gf_mul(value2, value2);
+    uint8_t value8 = gf_mul(value4, value4);
+    uint8_t value16 = gf_mul(value8, value8);
+    uint8_t value32 = gf_mul(value16, value16);
+    uint8_t value64 = gf_mul(value32, value32);
+    uint8_t value128 = gf_mul(value64, value64);
+    uint8_t inverse = gf_mul(value128, value64);
+    inverse = gf_mul(inverse, value32);
+    inverse = gf_mul(inverse, value16);
+    inverse = gf_mul(inverse, value8);
+    inverse = gf_mul(inverse, value4);
+    inverse = gf_mul(inverse, value2);
+    return inverse ^ rotate_left(inverse, 1) ^ rotate_left(inverse, 2) ^
+           rotate_left(inverse, 3) ^ rotate_left(inverse, 4) ^ 0x63;
+}
+
+// AES-256 key expansion to 15 encryption round keys.
+static void expand_key(const uint8_t *key, uint8_t ek[15][16]) {
+    uint8_t w[60][4];
+    memcpy(w, key, 32);
+    uint8_t rcon = 1;
+    for (int i = 8; i < 60; i++) {
+        uint8_t t[4];
+        memcpy(t, w[i - 1], 4);
+        if (i % 8 == 0) {
+            uint8_t tmp = t[0];
+            t[0] = t[1];
+            t[1] = t[2];
+            t[2] = t[3];
+            t[3] = tmp;
+            for (int j = 0; j < 4; j++) t[j] = aes_sbox(t[j]);
+            t[0] ^= rcon;
+            rcon = (uint8_t)((rcon << 1) ^ ((rcon >> 7) * 0x1b));
+        } else if (i % 8 == 4) {
+            for (int j = 0; j < 4; j++) t[j] = aes_sbox(t[j]);
+        }
+        for (int j = 0; j < 4; j++) w[i][j] = w[i - 8][j] ^ t[j];
+    }
+    for (int r = 0; r < 15; r++) memcpy(ek[r], w[4 * r], 16);
+}
+
+// IGE decrypt: c is the previous ciphertext and m is the previous plaintext.
+// Returns 0 on success, -1 if hardware AES is unavailable, and -2 on bad input.
+int aes256_ige_decrypt(const uint8_t *key, const uint8_t *iv,
+                       uint8_t *dst, const uint8_t *src, size_t len) {
+    if (!aes_hw_available()) return -1;
+    if (len == 0 || len % 16 != 0) return -2;
+
+    uint8_t ek[15][16];
+    expand_key(key, ek);
+
+    uint8x16_t drk[15];
+    drk[0] = vld1q_u8(ek[14]);
+    for (int j = 1; j < 14; j++) drk[j] = vaesimcq_u8(vld1q_u8(ek[14 - j]));
+    drk[14] = vld1q_u8(ek[0]);
+
+    uint8x16_t c = vld1q_u8(iv);
+    uint8x16_t m = vld1q_u8(iv + 16);
+
+    for (size_t o = 0; o < len; o += 16) {
+        uint8x16_t ct = vld1q_u8(src + o);
+        uint8x16_t s = veorq_u8(ct, m);
+        s = vaesdq_u8(s, drk[0]);
+        s = vaesimcq_u8(s);
+        for (int i = 1; i < 13; i++) {
+            s = vaesdq_u8(s, drk[i]);
+            s = vaesimcq_u8(s);
+        }
+        s = vaesdq_u8(s, drk[13]);
+        s = veorq_u8(s, drk[14]);
+        s = veorq_u8(s, c);
+        vst1q_u8(dst + o, s);
+        m = s;
+        c = ct;
+    }
+    return 0;
+}