Page MenuHomeFreeBSD
Files F173951586

D60097.id187905.diff
No OneTemporary
Actions

D60097.id187905.diff
View Options

diff --git a/FreeBSD/sys/conf/files.arm64 b/FreeBSD/sys/conf/files.arm64
--- a/FreeBSD/sys/conf/files.arm64
+++ b/FreeBSD/sys/conf/files.arm64
@@ -197,14 +197,21 @@
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "aesv8-armx.o"
-ghashv8-armx.o optional armv8crypto \
+ghashv8-armx.o optional armv8crypto | ossl \
dependency "$S/crypto/openssl/aarch64/ghashv8-armx.S" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "ghashv8-armx.o"
+aes-gcm-armv8_64.o optional ossl \
+ dependency "$S/crypto/openssl/aarch64/aes-gcm-armv8_64.S" \
+ compile-with "${CC} -c ${CFLAGS:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aes-gcm-armv8_64.o"
+
crypto/des/des_enc.c optional netsmb
crypto/openssl/ossl_aarch64.c optional ossl
+crypto/openssl/ossl_aes_gcm.c optional ossl
crypto/openssl/aarch64/chacha-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}"
crypto/openssl/aarch64/chacha-armv8-sve.S optional ossl \
diff --git a/FreeBSD/sys/crypto/openssl/ossl.c b/FreeBSD/sys/crypto/openssl/ossl.c
--- a/FreeBSD/sys/crypto/openssl/ossl.c
+++ b/FreeBSD/sys/crypto/openssl/ossl.c
@@ -200,6 +200,10 @@
if (csp->csp_auth_mlen != 0 &&
csp->csp_auth_mlen != GMAC_DIGEST_LEN)
return (EINVAL);
+#ifdef __aarch64__
+ /* Prefer the fused implementation over armv8crypto. */
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE + 1);
+#endif
break;
default:
return (EINVAL);
@@ -479,6 +483,30 @@
sizeof(struct ossl_softc)
};
+#ifdef __aarch64__
+static void
+ossl_cpuid_late(void *arg __unused)
+{
+ devclass_t dc;
+ device_t dev;
+
+ dc = devclass_find("ossl");
+ if (dc == NULL)
+ return;
+ dev = devclass_get_device(dc, 0);
+ if (dev == NULL || !device_is_attached(dev))
+ return;
+
+ /*
+ * elf_hwcap is populated after normal nexus children attach.
+ * Recheck it before consumers begin creating crypto sessions.
+ */
+ ossl_cpuid(device_get_softc(dev));
+}
+SYSINIT(ossl_cpuid_late, SI_SUB_CONFIGURE, SI_ORDER_LAST,
+ ossl_cpuid_late, NULL);
+#endif
+
DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL);
MODULE_VERSION(ossl, 1);
MODULE_DEPEND(ossl, crypto, 1, 1, 1);
diff --git a/FreeBSD/sys/crypto/openssl/ossl_aarch64.c b/FreeBSD/sys/crypto/openssl/ossl_aarch64.c
--- a/FreeBSD/sys/crypto/openssl/ossl_aarch64.c
+++ b/FreeBSD/sys/crypto/openssl/ossl_aarch64.c
@@ -41,9 +41,13 @@
* Feature bits defined in arm_arch.h
*/
unsigned int OPENSSL_armcap_P;
+#define AES_PMULL_CAPABLE \
+ ((OPENSSL_armcap_P & (ARMV8_AES | ARMV8_PMULL)) == \
+ (ARMV8_AES | ARMV8_PMULL))
ossl_cipher_setkey_t aes_v8_set_encrypt_key;
ossl_cipher_setkey_t aes_v8_set_decrypt_key;
+ossl_cipher_setkey_t ossl_aes_gcm_setkey;
ossl_cipher_setkey_t vpaes_set_encrypt_key;
ossl_cipher_setkey_t vpaes_set_decrypt_key;
@@ -78,4 +82,10 @@
ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key;
ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key;
}
+
+ if (AES_PMULL_CAPABLE) {
+ ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey;
+ ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey;
+ sc->has_aes_gcm = true;
+ }
}
diff --git a/FreeBSD/sys/crypto/openssl/ossl_aes_gcm.c b/FreeBSD/sys/crypto/openssl/ossl_aes_gcm.c
--- a/FreeBSD/sys/crypto/openssl/ossl_aes_gcm.c
+++ b/FreeBSD/sys/crypto/openssl/ossl_aes_gcm.c
@@ -10,10 +10,10 @@
*/
/*
- * This file contains an AES-GCM wrapper implementation from OpenSSL, using
- * AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from
- * cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation
- * for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
+ * This file contains AES-GCM wrapper implementations from OpenSSL, using
+ * AES-NI (x86), ARMv8 Crypto Extensions, or POWER8 Crypto Extensions. It
+ * makes use of a generic C implementation for partial blocks, ported from
+ * gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
*/
#include <sys/endian.h>
@@ -53,6 +53,86 @@
void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
size_t len);
+#elif defined(__aarch64__)
+#define AES_set_encrypt_key aes_v8_set_encrypt_key
+#define AES_gcm_encrypt armv8_aes_gcm_encrypt
+#define AES_gcm_decrypt armv8_aes_gcm_decrypt
+#define AES_encrypt aes_v8_encrypt
+#define AES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
+#define GCM_init gcm_init_v8
+#define GCM_gmult gcm_gmult_v8
+#define GCM_ghash gcm_ghash_v8
+
+int AES_set_encrypt_key(const unsigned char *key, int bits, void *ctx);
+void AES_encrypt(const unsigned char *in, unsigned char *out, const void *ks);
+void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+ size_t blocks, const void *ks, const unsigned char *iv);
+
+void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]);
+void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
+void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
+ size_t len);
+
+typedef size_t armv8_gcm_kernel_t(const unsigned char *, uint64_t,
+ unsigned char *, uint64_t *, unsigned char *, const void *);
+
+armv8_gcm_kernel_t aes_gcm_enc_128_kernel;
+armv8_gcm_kernel_t aes_gcm_enc_192_kernel;
+armv8_gcm_kernel_t aes_gcm_enc_256_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_128_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_192_kernel;
+armv8_gcm_kernel_t aes_gcm_dec_256_kernel;
+
+static size_t
+armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t Xi[2])
+{
+ const struct ossl_aes_keysched *aes_key;
+ size_t bulk;
+
+ aes_key = key;
+ bulk = len - len % AES_BLOCK_LEN;
+ switch (aes_key->rounds) {
+ case 10:
+ (void)aes_gcm_enc_128_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 12:
+ (void)aes_gcm_enc_192_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 14:
+ (void)aes_gcm_enc_256_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ default:
+ return (0);
+ }
+ return (bulk);
+}
+
+static size_t
+armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t Xi[2])
+{
+ const struct ossl_aes_keysched *aes_key;
+ size_t bulk;
+
+ aes_key = key;
+ bulk = len - len % AES_BLOCK_LEN;
+ switch (aes_key->rounds) {
+ case 10:
+ (void)aes_gcm_dec_128_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 12:
+ (void)aes_gcm_dec_192_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ case 14:
+ (void)aes_gcm_dec_256_kernel(in, bulk * 8, out, Xi, ivec, key);
+ break;
+ default:
+ return (0);
+ }
+ return (bulk);
+}
+
#elif defined(__powerpc64__)
#define AES_set_encrypt_key aes_p8_set_encrypt_key
#define AES_gcm_encrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,1)
diff --git a/FreeBSD/sys/modules/ossl/Makefile b/FreeBSD/sys/modules/ossl/Makefile
--- a/FreeBSD/sys/modules/ossl/Makefile
+++ b/FreeBSD/sys/modules/ossl/Makefile
@@ -28,14 +28,17 @@
ossl_aes_gcm_neon.c
SRCS.aarch64= \
+ aes-gcm-armv8_64.S \
chacha-armv8.S \
chacha-armv8-sve.S \
+ ghashv8-armx.S \
poly1305-armv8.S \
sha1-armv8.S \
sha256-armv8.S \
sha512-armv8.S \
vpaes-armv8.S \
- ossl_aarch64.c
+ ossl_aarch64.c \
+ ossl_aes_gcm.c
SRCS.amd64= \
aes-gcm-avx512.S \

File Metadata

Mime Type
text/plain
Expires
Wed, Sep 30, 2:41 PM (20 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39899139
Default Alt Text
D60097.id187905.diff (7 KB)

Event Timeline