Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -140,6 +140,13 @@ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/vpaes-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" +sha256c_arm64.o optional crypto | \ + ekcd | geom_bde | ipsec | ipsec_support | !random_loadable | \ + sctp | zfs \ + dependency "$S/crypto/sha2/sha256c_arm64.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/sha2/ -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} -march=armv8-a+crypto -DARM64_SHA2 ${.IMPSRC}" \ + no-implicit-rule \ + clean "sha256c_arm64.o" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi Index: sys/crypto/sha2/sha256c.c =================================================================== --- sys/crypto/sha2/sha256c.c +++ sys/crypto/sha2/sha256c.c @@ -42,6 +42,7 @@ #if defined(ARM64_SHA2) #include +#include #include #endif @@ -201,6 +202,20 @@ SHA256_Transform_arm64_impl(state, block, K); } +#ifdef _KERNEL +DEFINE_IFUNC(static, void, SHA256_Transform, + (uint32_t * state, const unsigned char block[64])) +{ + uint64_t reg; + + if (get_kernel_reg(ID_AA64ISAR0_EL1, ®)) { + if (ID_AA64ISAR0_SHA2_VAL(reg) >= ID_AA64ISAR0_SHA2_BASE) + return (SHA256_Transform_arm64); + } + + return (SHA256_Transform_c); +} +#else /* _KERNEL */ DEFINE_UIFUNC(static, void, SHA256_Transform, (uint32_t * state, const unsigned char block[64])) { @@ -209,17 +224,19 @@ if (elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) { if ((hwcap & HWCAP_SHA2) != 0) return (SHA256_Transform_arm64); + } } return (SHA256_Transform_c); } -#else +#endif /* _KERNEL */ +#else /* ARM64_SHA2 */ static void SHA256_Transform(uint32_t * state, const unsigned char block[64]) { SHA256_Transform_c(state, block); } -#endif +#endif /* ARM64_SHA2 */ static unsigned char PAD[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Index: sys/modules/crypto/Makefile =================================================================== --- sys/modules/crypto/Makefile +++ sys/modules/crypto/Makefile @@ -27,6 +27,16 @@ SRCS += camellia.c camellia-api.c SRCS += des_ecb.c des_enc.c des_setkey.c SRCS += sha1.c sha256c.c sha512c.c +.if ${MACHINE_ARCH} == "aarch64" +# arm_neon.h wants stdint.h +CFLAGS+= -I${SRCTOP}/sys/crypto/armv8 +OBJS+= sha256c_arm64.o +# Remove -nostdinc so we can get the intrinsics. +sha256c_arm64.o: sha256c_arm64.c sha256c_impl.h + ${CC} ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ + -march=armv8-a+crypto -DARM64_SHA2 -c ${.IMPSRC} + ${CTFCONVERT_CMD} +.endif SRCS += skein.c skein_block.c # unroll the 256 and 512 loops, half unroll the 1024 CFLAGS.skein_block.c += -DSKEIN_LOOP=995