Index: share/man/man4/ossl.4 =================================================================== --- share/man/man4/ossl.4 +++ share/man/man4/ossl.4 @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 3, 2021 +.Dd September 24, 2021 .Dt OSSL 4 .Os .Sh NAME @@ -74,6 +74,8 @@ .Pp .Bl -bullet -compact .It +AES-CBC +.It ChaCha20 .It ChaCha20-Poly1305 (RFC 8439) Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -716,6 +716,7 @@ crypto/des/des_ecb.c optional netsmb crypto/des/des_setkey.c optional netsmb crypto/openssl/ossl.c optional ossl +crypto/openssl/ossl_aes.c optional ossl crypto/openssl/ossl_chacha20.c optional ossl crypto/openssl/ossl_poly1305.c optional ossl crypto/openssl/ossl_sha1.c optional ossl Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -88,6 +88,7 @@ cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb +crypto/openssl/amd64/aesni-x86_64.S optional ossl crypto/openssl/amd64/chacha-x86_64.S optional ossl crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -124,6 +124,8 @@ crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl +crypto/openssl/aarch64/aesv8-armx.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} -march=armv8-a+crypto ${.IMPSRC}" crypto/openssl/aarch64/chacha-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/poly1305-armv8.S optional ossl \ @@ -134,6 +136,8 @@ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha512-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" +crypto/openssl/aarch64/vpaes-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi Index: sys/conf/files.i386 =================================================================== --- sys/conf/files.i386 +++ sys/conf/files.i386 @@ -15,6 +15,7 @@ cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_i386.S optional aesni crypto/des/arch/i386/des_enc.S optional netsmb +crypto/openssl/i386/aesni-x86.S optional ossl crypto/openssl/i386/chacha-x86.S optional ossl crypto/openssl/i386/poly1305-x86.S optional ossl crypto/openssl/i386/sha1-586.S optional ossl Index: sys/crypto/openssl/ossl.h =================================================================== --- sys/crypto/openssl/ossl.h +++ sys/crypto/openssl/ossl.h @@ -37,19 +37,28 @@ struct cryptop; struct crypto_session_params; +struct ossl_softc { + int32_t sc_cid; + bool has_aes; +}; + int ossl_chacha20(struct cryptop *crp, const struct crypto_session_params *csp); int ossl_chacha20_poly1305_decrypt(struct cryptop *crp, const struct crypto_session_params *csp); int ossl_chacha20_poly1305_encrypt(struct cryptop *crp, const struct crypto_session_params *csp); -void ossl_cpuid(void); +void ossl_cpuid(struct ossl_softc *sc); /* Needs to be big enough to hold any hash context. */ struct ossl_hash_context { uint32_t dummy[61]; } __aligned(32); +struct ossl_cipher_context { + uint32_t dummy[61]; +} __aligned(32); + extern struct auth_hash ossl_hash_poly1305; extern struct auth_hash ossl_hash_sha1; extern struct auth_hash ossl_hash_sha224; @@ -57,4 +66,6 @@ extern struct auth_hash ossl_hash_sha384; extern struct auth_hash ossl_hash_sha512; +extern struct ossl_cipher ossl_cipher_aes_cbc; + #endif /* !__OSSL_H__ */ Index: sys/crypto/openssl/ossl.c =================================================================== --- sys/crypto/openssl/ossl.c +++ sys/crypto/openssl/ossl.c @@ -49,13 +49,10 @@ #include #include +#include #include "cryptodev_if.h" -struct ossl_softc { - int32_t sc_cid; -}; - struct ossl_session_hash { struct ossl_hash_context ictx; struct ossl_hash_context octx; @@ -63,7 +60,14 @@ u_int mlen; }; +struct ossl_session_cipher { + struct ossl_cipher_context dec_ctx; + struct ossl_cipher_context enc_ctx; + struct ossl_cipher *cipher; +}; + struct ossl_session { + struct ossl_session_cipher cipher; struct ossl_session_hash hash; }; @@ -92,7 +96,7 @@ sc = device_get_softc(dev); - ossl_cpuid(); + ossl_cpuid(sc); sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); @@ -143,9 +147,41 @@ } } +static struct ossl_cipher* +ossl_lookup_cipher(const struct crypto_session_params *csp) +{ + struct ossl_cipher *cipher; + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + cipher = &ossl_cipher_aes_cbc; + break; + default: + return (NULL); + } + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + switch (csp->csp_cipher_klen * 8) { + case 128: + case 192: + case 256: + break; + default: + return (NULL); + } + break; + default: + __assert_unreachable(); + } + + return (cipher); +} + static int ossl_probesession(device_t dev, const struct crypto_session_params *csp) { + struct ossl_softc *sc = device_get_softc(dev); if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) @@ -162,7 +198,8 @@ return (EINVAL); break; default: - return (EINVAL); + if (!sc->has_aes || ossl_lookup_cipher(csp) == NULL) + return (EINVAL); } break; case CSP_MODE_AEAD: @@ -213,20 +250,57 @@ } } +static int +ossl_newsession_cipher(struct ossl_session *s, + const struct crypto_session_params *csp) +{ + struct ossl_cipher *cipher; + int error; + + if (csp->csp_cipher_alg == CRYPTO_CHACHA20) + return (0); + + cipher = ossl_lookup_cipher(csp); + if (cipher == NULL) + return (EINVAL); + + s->cipher.cipher = cipher; + + if (csp->csp_cipher_key == NULL) + return (0); + + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + error = cipher->set_encrypt_key(csp->csp_cipher_key, + 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx); + if (error != 0) { + fpu_kern_leave(curthread, NULL); + return (error); + } + error = cipher->set_decrypt_key(csp->csp_cipher_key, + 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx); + fpu_kern_leave(curthread, NULL); + + return (error); +} + static int ossl_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ossl_session *s; + int error = 0; s = crypto_get_driver_session(cses); switch (csp->csp_mode) { case CSP_MODE_DIGEST: ossl_newsession_hash(s, csp); break; + case CSP_MODE_CIPHER: + error = ossl_newsession_cipher(s, csp); + break; } - return (0); + return (error); } static int @@ -296,6 +370,116 @@ explicit_bzero(&ctx, sizeof(ctx)); return (error); } +static int +ossl_process_cipher(struct ossl_session *s, struct cryptop *crp, + const struct crypto_session_params *csp) +{ + struct crypto_buffer_cursor cc_in, cc_out; + unsigned char block[EALG_MAX_BLOCK_LEN]; + unsigned char iv[EALG_MAX_BLOCK_LEN]; + const unsigned char *in, *inseg; + unsigned char *out, *outseg; + size_t plen, seglen, inlen, outlen; + struct ossl_cipher_context key; + struct ossl_cipher *cipher; + int blocklen, error; + bool encrypt; + + /* CHACHA20 is handled separately for now. */ + if (csp->csp_cipher_alg == CRYPTO_CHACHA20) + return (ossl_chacha20(crp, csp)); + + cipher = s->cipher.cipher; + encrypt = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); + plen = crp->crp_payload_length; + blocklen = cipher->blocksize; + + if (!cipher->stream_cipher && (plen % blocklen)) + return (EINVAL); + + if (crp->crp_cipher_key != NULL) { + if (encrypt) + error = cipher->set_encrypt_key(crp->crp_cipher_key, + 8 * csp->csp_cipher_klen, &key); + else + error = cipher->set_decrypt_key(crp->crp_cipher_key, + 8 * csp->csp_cipher_klen, &key); + if (error) + return (error); + } else { + if (encrypt) + key = s->cipher.enc_ctx; + else + key = s->cipher.dec_ctx; + } + + crypto_read_iv(crp, iv); + if (cipher->set_iv != NULL) + cipher->set_iv(iv, csp->csp_ivlen, iv); + + /* Derived from ossl_chacha20.c */ + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_payload_start); + inseg = crypto_cursor_segment(&cc_in, &inlen); + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + crypto_cursor_init(&cc_out, &crp->crp_obuf); + crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); + } else { + cc_out = cc_in; + } + outseg = crypto_cursor_segment(&cc_out, &outlen); + + while (plen >= blocklen) { + if (inlen < blocklen) { + crypto_cursor_copydata(&cc_in, blocklen, block); + in = block; + inlen = blocklen; + } else { + in = inseg; + } + if (outlen < blocklen) { + out = block; + outlen = blocklen; + } else { + out = outseg; + } + + /* Figure out how many blocks we can encrypt/decrypt at once. */ + seglen = rounddown(MIN(plen, MIN(inlen, outlen)), blocklen); + + cipher->encrypt(in, out, seglen, &key, iv, encrypt); + + if (out == block) { + crypto_cursor_copyback(&cc_out, blocklen, block); + outseg = crypto_cursor_segment(&cc_out, &outlen); + } else { + crypto_cursor_advance(&cc_out, seglen); + outseg += seglen; + outlen -= seglen; + } + if (in == block) { + inseg = crypto_cursor_segment(&cc_in, &inlen); + } else { + crypto_cursor_advance(&cc_in, seglen); + inseg += seglen; + inlen -= seglen; + } + plen -= seglen; + } + + if (plen > 0) { + MPASS(cipher->stream_cipher); + memset(block, 0, sizeof(block)); + crypto_cursor_copydata(&cc_in, plen, block); + cipher->encrypt(block, block, blocklen, &key, iv, encrypt); + crypto_cursor_copyback(&cc_out, plen, block); + } + + explicit_bzero(block, sizeof(block)); + explicit_bzero(iv, sizeof(iv)); + explicit_bzero(&key, sizeof(key)); + return (0); +} static int ossl_process(device_t dev, struct cryptop *crp, int hint) @@ -320,7 +504,7 @@ error = ossl_process_hash(s, crp, csp); break; case CSP_MODE_CIPHER: - error = ossl_chacha20(crp, csp); + error = ossl_process_cipher(s, crp, csp); break; case CSP_MODE_AEAD: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) Index: sys/crypto/openssl/ossl_aarch64.c =================================================================== --- sys/crypto/openssl/ossl_aarch64.c +++ sys/crypto/openssl/ossl_aarch64.c @@ -36,15 +36,26 @@ #include #include +#include #include /* * Feature bits defined in arm_arch.h */ unsigned int OPENSSL_armcap_P; +#define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES) +#define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) + +ossl_cipher_setkey_t aes_v8_set_encrypt_key; +ossl_cipher_setkey_t aes_v8_set_decrypt_key; +ossl_cipher_encrypt_t aes_v8_cbc_encrypt; + +ossl_cipher_setkey_t vpaes_set_encrypt_key; +ossl_cipher_setkey_t vpaes_set_decrypt_key; +ossl_cipher_encrypt_t vpaes_cbc_encrypt; void -ossl_cpuid(void) +ossl_cpuid(struct ossl_softc *sc) { /* SHA features */ if ((elf_hwcap & HWCAP_SHA1) != 0) @@ -59,4 +70,19 @@ OPENSSL_armcap_P |= ARMV8_AES; if ((elf_hwcap & HWCAP_PMULL) != 0) OPENSSL_armcap_P |= ARMV8_PMULL; + + if (!HWAES_CAPABLE && !VPAES_CAPABLE) { + sc->has_aes = false; + return; + } + sc->has_aes = true; + if (HWAES_CAPABLE) { + ossl_cipher_aes_cbc.set_encrypt_key = aes_v8_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = aes_v8_set_decrypt_key; + ossl_cipher_aes_cbc.encrypt = aes_v8_cbc_encrypt; + } else { + ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key; + ossl_cipher_aes_cbc.encrypt = vpaes_cbc_encrypt; + } } Index: sys/crypto/openssl/ossl_aes.c =================================================================== --- /dev/null +++ sys/crypto/openssl/ossl_aes.c @@ -0,0 +1,49 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Stormshield. + * Copyright (c) 2021 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +#include + +struct ossl_cipher ossl_cipher_aes_cbc = { + .type = CRYPTO_AES_CBC, + .blocksize = AES_BLOCK_LEN, + .ivsize = AES_BLOCK_LEN, + .stream_cipher = false, + + /* Filled during initialization based on CPU caps. */ + .set_encrypt_key = NULL, + .set_decrypt_key = NULL, + .set_iv = NULL, + .encrypt = NULL +}; Index: sys/crypto/openssl/ossl_cipher.h =================================================================== --- /dev/null +++ sys/crypto/openssl/ossl_cipher.h @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Stormshield. + * Copyright (c) 2021 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OSSL_CIPHER_H__ +#define __OSSL_CIPHER_H__ + +typedef int (ossl_cipher_setkey_t)(const unsigned char*, int, void*); +typedef void (ossl_cipher_encrypt_t)(const unsigned char*, unsigned char*, size_t, + const void*, unsigned char*, int); + +struct ossl_cipher { + int type; + uint16_t blocksize; + uint16_t ivsize; + bool stream_cipher; + + ossl_cipher_setkey_t *set_encrypt_key; + ossl_cipher_setkey_t *set_decrypt_key; + ossl_cipher_setkey_t *set_iv; + ossl_cipher_encrypt_t *encrypt; +}; + +#endif Index: sys/crypto/openssl/ossl_x86.c =================================================================== --- sys/crypto/openssl/ossl_x86.c +++ sys/crypto/openssl/ossl_x86.c @@ -39,6 +39,7 @@ #include #include +#include /* * See OPENSSL_ia32cap(3). @@ -49,9 +50,14 @@ * [3] = 0 */ unsigned int OPENSSL_ia32cap_P[4]; +#define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) + +ossl_cipher_setkey_t aesni_set_encrypt_key; +ossl_cipher_setkey_t aesni_set_decrypt_key; +ossl_cipher_encrypt_t aesni_cbc_encrypt; void -ossl_cpuid(void) +ossl_cpuid(struct ossl_softc *sc) { uint64_t xcr0; u_int regs[4]; @@ -112,4 +118,13 @@ OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; } + + if (!AESNI_CAPABLE) { + sc->has_aes = false; + return; + } + sc->has_aes = true; + ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key; + ossl_cipher_aes_cbc.encrypt = aesni_cbc_encrypt; } Index: sys/modules/ossl/Makefile =================================================================== --- sys/modules/ossl/Makefile +++ sys/modules/ossl/Makefile @@ -4,10 +4,12 @@ .PATH: ${SRCTOP}/sys/crypto/openssl/${MACHINE_CPUARCH} KMOD= ossl +OBJS+= ${OBJS.${MACHINE_CPUARCH}} SRCS= bus_if.h \ cryptodev_if.h \ device_if.h \ ossl.c \ + ossl_aes.c \ ossl_chacha20.c \ ossl_poly1305.c \ ossl_sha1.c \ @@ -21,9 +23,11 @@ sha1-armv8.S \ sha256-armv8.S \ sha512-armv8.S \ + vpaes-armv8.S \ ossl_aarch64.c SRCS.amd64= \ + aesni-x86_64.S \ chacha-x86_64.S \ poly1305-x86_64.S \ sha1-x86_64.S \ @@ -32,6 +36,7 @@ ossl_x86.c SRCS.i386= \ + aesni-x86.S \ chacha-x86.S \ poly1305-x86.S \ sha1-586.S \ @@ -45,4 +50,13 @@ ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC} ${CTFCONVERT_CMD} +# Based on modules/armv8crypto/Makefile. +# Clang doesn't recognize "aes*" instructions without -march set. +aesv8-armx.o: aesv8-armx.S + ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + +OBJS.aarch64= aesv8-armx.o + .include