diff --git a/sys/conf/files.arm b/sys/conf/files.arm --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -135,6 +135,16 @@ libkern/udivdi3.c standard libkern/umoddi3.c standard +crypto/openssl/ossl_arm.c optional ossl +crypto/openssl/arm/aes-armv4.S optional ossl +crypto/openssl/arm/bsaes-armv7.S optional ossl \ + compile-with "${CC} -D__KERNEL__ -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" +crypto/openssl/arm/chacha-armv4.S optional ossl +crypto/openssl/arm/poly1305-armv4.S optional ossl +crypto/openssl/arm/sha1-armv4-large.S optional ossl +crypto/openssl/arm/sha256-armv4.S optional ossl +crypto/openssl/arm/sha512-armv4.S optional ossl + # Annapurna support arm/annapurna/alpine/alpine_ccu.c optional al_ccu fdt arm/annapurna/alpine/alpine_nb_service.c optional al_nb_service fdt diff --git a/sys/crypto/openssl/arm/arm_arch.h b/sys/crypto/openssl/arm/arm_arch.h new file mode 100644 --- /dev/null +++ b/sys/crypto/openssl/arm/arm_arch.h @@ -0,0 +1,84 @@ +/* + * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ARM_ARCH_H +# define OSSL_CRYPTO_ARM_ARCH_H + +# if !defined(__ARM_ARCH__) +# if defined(__CC_ARM) +# define __ARM_ARCH__ __TARGET_ARCH_ARM +# if defined(__BIG_ENDIAN) +# define __ARMEB__ +# else +# define __ARMEL__ +# endif +# elif defined(__GNUC__) +# if defined(__aarch64__) +# define __ARM_ARCH__ 8 +# if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define __ARMEB__ +# else +# define __ARMEL__ +# endif + /* + * Why doesn't gcc define __ARM_ARCH__? Instead it defines + * bunch of below macros. See all_architectures[] table in + * gcc/config/arm/arm.c. On a side note it defines + * __ARMEL__/__ARMEB__ for little-/big-endian. + */ +# elif defined(__ARM_ARCH) +# define __ARM_ARCH__ __ARM_ARCH +# elif defined(__ARM_ARCH_8A__) +# define __ARM_ARCH__ 8 +# elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ + defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) || \ + defined(__ARM_ARCH_7EM__) +# define __ARM_ARCH__ 7 +# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ + defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__) || \ + defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__) || \ + defined(__ARM_ARCH_6T2__) +# define __ARM_ARCH__ 6 +# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \ + defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__) || \ + defined(__ARM_ARCH_5TEJ__) +# define __ARM_ARCH__ 5 +# elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) +# define __ARM_ARCH__ 4 +# else +# error "unsupported ARM architecture" +# endif +# endif +# endif + +# if !defined(__ARM_MAX_ARCH__) +# define __ARM_MAX_ARCH__ __ARM_ARCH__ +# endif + +# if __ARM_MAX_ARCH__<__ARM_ARCH__ +# error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" +# elif __ARM_MAX_ARCH__!=__ARM_ARCH__ +# if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__) +# error "can't build universal big-endian binary" +# endif +# endif + +# ifndef __ASSEMBLER__ +extern unsigned int OPENSSL_armcap_P; +# endif + +# define ARMV7_NEON (1<<0) +# define ARMV7_TICK (1<<1) +# define ARMV8_AES (1<<2) +# define ARMV8_SHA1 (1<<3) +# define ARMV8_SHA256 (1<<4) +# define ARMV8_PMULL (1<<5) +# define ARMV8_SHA512 (1<<6) + +#endif diff --git a/sys/crypto/openssl/ossl_aes.c b/sys/crypto/openssl/ossl_aes.c --- a/sys/crypto/openssl/ossl_aes.c +++ b/sys/crypto/openssl/ossl_aes.c @@ -39,6 +39,8 @@ #include #elif defined (__aarch64__) #include +#elif defined (__arm__) +#include #endif static ossl_cipher_process_t ossl_aes_cbc; diff --git a/sys/crypto/openssl/ossl_aes_gcm.h b/sys/crypto/openssl/ossl_aes_gcm.h --- a/sys/crypto/openssl/ossl_aes_gcm.h +++ b/sys/crypto/openssl/ossl_aes_gcm.h @@ -28,7 +28,6 @@ #define _OSSL_AES_GCM_H_ #include -#include struct ossl_gcm_context; @@ -64,10 +63,7 @@ unsigned int mres, ares; } gcm; - struct { - uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; - int rounds; - } aes_ks; + struct ossl_aes_keysched aes_ks; const struct ossl_aes_gcm_ops *ops; }; diff --git a/sys/crypto/openssl/ossl_arm.h b/sys/crypto/openssl/ossl_arm.h new file mode 100644 --- /dev/null +++ b/sys/crypto/openssl/ossl_arm.h @@ -0,0 +1,94 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2023 Stormshield + * Copyright (c) 2023 Semihalf + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any + * redistribution must be conditioned upon including a substantially + * similar Disclaimer requirement for further binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGES. + */ + +#ifndef __OSSL_ARM__ +#define __OSSL_ARM__ + +#include +#include + +#include + +struct bsaes_key { + struct ossl_aes_keysched ks; + int converted; +#define BSAES_KEY_SIZE (128 * (RIJNDAEL_MAXNR - 1) + 2 * AES_BLOCK_LEN) + uint8_t bitslice[BSAES_KEY_SIZE] __aligned(8); +} __aligned(8); + +ossl_cipher_encrypt_t ossl_bsaes_cbc_encrypt; + +void AES_encrypt(const void *, void *, const void *); + +static inline void +AES_CBC_ENCRYPT(const unsigned char *in, unsigned char *out, + size_t length, const void *key, unsigned char *iv, int encrypt) +{ + struct bsaes_key bsks; + uint32_t iv32[4], scratch[4]; + + /* + * bsaes_cbc_encrypt has some special requirements w.r.t input data. + * The key buffer, that normally holds round keys is used as a scratch + * space. 128 bytes per round of extra space is required. + * Another thing is that only decryption is supported. + * In the case of encryption block chaining has to be done in C. + */ + if (!encrypt) { + memcpy(&bsks.ks, key, sizeof(bsks.ks)); + bsks.converted = 0; + ossl_bsaes_cbc_encrypt(in, out, length, &bsks, iv, false); + return; + } + + length /= AES_BLOCK_LEN; + memcpy(iv32, iv, AES_BLOCK_LEN); + + while (length-- > 0) { + memcpy(scratch, in, AES_BLOCK_LEN); + + /* XOR plaintext with IV. */ + scratch[0] ^= iv32[0]; + scratch[1] ^= iv32[1]; + scratch[2] ^= iv32[2]; + scratch[3] ^= iv32[3]; + + AES_encrypt(scratch, out, key); + + memcpy(iv32, out, AES_BLOCK_LEN); + in += AES_BLOCK_LEN; + out += AES_BLOCK_LEN; + } + + memcpy(iv, iv32, AES_BLOCK_LEN); +} + +#endif /* __OSSL_ARM__ */ diff --git a/sys/crypto/openssl/ossl_arm.c b/sys/crypto/openssl/ossl_arm.c new file mode 100644 --- /dev/null +++ b/sys/crypto/openssl/ossl_arm.c @@ -0,0 +1,59 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2023 Stormshield + * Copyright (c) 2023 Semihalf + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any + * redistribution must be conditioned upon including a substantially + * similar Disclaimer requirement for further binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGES. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#include +#include + +#include +#include +#include + +ossl_cipher_setkey_t AES_set_encrypt_key; +ossl_cipher_setkey_t AES_set_decrypt_key; + +unsigned int OPENSSL_armcap_P; + +void +ossl_cpuid(struct ossl_softc *sc) +{ + if (elf_hwcap & HWCAP_NEON) { + OPENSSL_armcap_P |= ARMV7_NEON; + + sc->has_aes = true; + ossl_cipher_aes_cbc.set_encrypt_key = AES_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = AES_set_decrypt_key; + } +} diff --git a/sys/crypto/openssl/ossl_cipher.h b/sys/crypto/openssl/ossl_cipher.h --- a/sys/crypto/openssl/ossl_cipher.h +++ b/sys/crypto/openssl/ossl_cipher.h @@ -28,6 +28,9 @@ #ifndef __OSSL_CIPHER_H__ #define __OSSL_CIPHER_H__ +#include +#include + struct ossl_session_cipher; struct cryptop; struct crypto_session_params; @@ -50,4 +53,9 @@ ossl_cipher_process_t *process; }; +struct ossl_aes_keysched { + uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; + int rounds; +}; + #endif diff --git a/sys/modules/Makefile b/sys/modules/Makefile --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -565,11 +565,15 @@ .if ${MK_SOURCELESS_UCODE} != "no" _iwlwififw= iwlwififw .endif -_ossl= ossl _rtw88= rtw88 _vmware= vmware .endif +.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \ + ${MACHINE_CPUARCH} == "i386" || ${MACHINE_ARCH} == "armv7" +_ossl= ossl +.endif + # MAC framework .if ${KERN_OPTS:MMAC} || defined(ALL_MODULES) _mac_biba= mac_biba diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -16,6 +16,16 @@ ossl_sha512.c \ ${SRCS.${MACHINE_CPUARCH}} +SRCS.arm= \ + aes-armv4.S \ + bsaes-armv7.S \ + chacha-armv4.S \ + poly1305-armv4.S \ + sha1-armv4-large.S \ + sha256-armv4.S \ + sha512-armv4.S \ + ossl_arm.c + SRCS.aarch64= \ chacha-armv8.S \ poly1305-armv8.S \ @@ -47,6 +57,8 @@ sha512-586.S \ ossl_x86.c +CFLAGS.bsaes-armv7.S+= -D__KERNEL__ + # For arm64, we are forced to rewrite the compiler invocation for the assembly # files, to remove -mgeneral-regs-only. ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S