Index: head/share/man/man4/Makefile =================================================================== --- head/share/man/man4/Makefile +++ head/share/man/man4/Makefile @@ -398,6 +398,7 @@ ocs_fc.4\ ohci.4 \ orm.4 \ + ${_ossl.4} \ ow.4 \ ow_temp.4 \ owc.4 \ @@ -819,6 +820,7 @@ _nvd.4= nvd.4 _nvme.4= nvme.4 _nvram.4= nvram.4 +_ossl.4= ossl.4 _padlock.4= padlock.4 _pchtherm.4= pchtherm.4 _rr232x.4= rr232x.4 Index: head/share/man/man4/ossl.4 =================================================================== --- head/share/man/man4/ossl.4 +++ head/share/man/man4/ossl.4 @@ -0,0 +1,105 @@ +.\" Copyright (c) 2020 Netflix, Inc +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer, +.\" without modification. +.\" 2. Redistributions in binary form must reproduce at minimum a disclaimer +.\" similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any +.\" redistribution must be conditioned upon including a substantially +.\" similar Disclaimer requirement for further binary redistribution. +.\" +.\" NO WARRANTY +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY +.\" AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +.\" THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, +.\" OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +.\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +.\" THE POSSIBILITY OF SUCH DAMAGES. +.\" +.\" $FreeBSD$ +.\" +.Dd October 19, 2020 +.Dt OSSL 4 +.Os +.Sh NAME +.Nm ossl +.Nd "driver using OpenSSL assembly routines on x86 CPUs" +.Sh SYNOPSIS +To compile this driver into the kernel, +place the following lines in your +kernel configuration file: +.Bd -ragged -offset indent +.Cd "device crypto" +.Cd "device cryptodev" +.Cd "device ossl" +.Ed +.Pp +Alternatively, to load the driver as a +module at boot time, place the following line in +.Xr loader.conf 5 : +.Bd -literal -offset indent +ossl_load="YES" +.Ed +.Sh DESCRIPTION +The OpenSSL distribution includes architecture-specific +implementations for some commonly used cryptographic algorithms. +This driver adds a wrapper around these routines permitting them to be +used by in-kernel cryptography consumers such as kernel TLS and IPsec. +.Pp +The +.Nm +driver includes architecture-specific implementations for the following +architectures: +.Pp +.Bl -bullet -compact +.It +amd64 +.It +i386 +.El +.Pp +The +.Nm +driver includes support for the following algorithms: +.Pp +.Bl -bullet -compact +.It +SHA1 +.It +SHA1-HMAC +.It +SHA2-224 +.It +SHA2-224-HMAC +.It +SHA2-256 +.It +SHA2-256-HMAC +.It +SHA2-384 +.It +SHA2-384-HMAC +.It +SHA2-512 +.It +SHA2-512-HMAC +.El +.Sh SEE ALSO +.Xr crypto 4 , +.Xr intro 4 , +.Xr ipsec 4 , +.Xr crypto 7 , +.Xr crypto 9 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 13.0 . Index: head/sys/amd64/conf/NOTES =================================================================== --- head/sys/amd64/conf/NOTES +++ head/sys/amd64/conf/NOTES @@ -533,6 +533,7 @@ device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG device aesni # AES-NI OpenCrypto module +device ossl # OpenSSL OpenCrypto module device ioat # Intel I/OAT DMA engine # Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 +++ head/sys/conf/files.amd64 @@ -137,6 +137,9 @@ cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb +crypto/openssl/amd64/sha1-x86_64.S optional ossl +crypto/openssl/amd64/sha256-x86_64.S optional ossl +crypto/openssl/amd64/sha512-x86_64.S optional ossl dev/acpi_support/acpi_wmi_if.m standard dev/agp/agp_amd64.c optional agp dev/agp/agp_i810.c optional agp Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 +++ head/sys/conf/files.i386 @@ -77,6 +77,9 @@ compat/ndis/winx32_wrap.S optional ndisapi pci crypto/aesni/aeskeys_i386.S optional aesni crypto/des/arch/i386/des_enc.S optional netsmb +crypto/openssl/i386/sha1-586.S optional ossl +crypto/openssl/i386/sha256-586.S optional ossl +crypto/openssl/i386/sha512-586.S optional ossl dev/agp/agp_ali.c optional agp dev/agp/agp_amd.c optional agp dev/agp/agp_amd64.c optional agp Index: head/sys/conf/files.x86 =================================================================== --- head/sys/conf/files.x86 +++ head/sys/conf/files.x86 @@ -53,6 +53,10 @@ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" +crypto/openssl/ossl.c optional ossl +crypto/openssl/ossl_sha1.c optional ossl +crypto/openssl/ossl_sha256.c optional ossl +crypto/openssl/ossl_sha512.c optional ossl crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock Index: head/sys/crypto/openssl/ossl.h =================================================================== --- head/sys/crypto/openssl/ossl.h +++ head/sys/crypto/openssl/ossl.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 Netflix, Inc + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any + * redistribution must be conditioned upon including a substantially + * similar Disclaimer requirement for further binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGES. + * + * $FreeBSD$ + */ + +#ifndef __OSSL_H__ +#define __OSSL_H__ + +/* Compatibility shims. */ +#define OPENSSL_cleanse explicit_bzero + +/* Used by assembly routines to select CPU-specific variants. */ +extern unsigned int OPENSSL_ia32cap_P[4]; + +/* Needs to be big enough to hold any hash context. */ +struct ossl_hash_context { + uint32_t dummy[54]; +} __aligned(32); + +extern struct auth_hash ossl_hash_sha1; +extern struct auth_hash ossl_hash_sha224; +extern struct auth_hash ossl_hash_sha256; +extern struct auth_hash ossl_hash_sha384; +extern struct auth_hash ossl_hash_sha512; + +#endif /* !__OSSL_H__ */ Index: head/sys/crypto/openssl/ossl.c =================================================================== --- head/sys/crypto/openssl/ossl.c +++ head/sys/crypto/openssl/ossl.c @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2020 Netflix, Inc + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any + * redistribution must be conditioned upon including a substantially + * similar Disclaimer requirement for further binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGES. + */ + +/* + * A driver for the OpenCrypto framework which uses assembly routines + * from OpenSSL. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "cryptodev_if.h" + +struct ossl_softc { + int32_t sc_cid; +}; + +struct ossl_session_hash { + struct ossl_hash_context ictx; + struct ossl_hash_context octx; + struct auth_hash *axf; + u_int mlen; +}; + +struct ossl_session { + struct ossl_session_hash hash; +}; + +/* + * See OPENSSL_ia32cap(3). + * + * [0] = cpu_feature but with a few custom bits + * [1] = cpu_feature2 but with AMD XOP in bit 11 + * [2] = cpu_stdext_feature + * [3] = 0 + */ +unsigned int OPENSSL_ia32cap_P[4]; + +static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); + +static void +ossl_cpuid(void) +{ + uint64_t xcr0; + u_int regs[4]; + u_int max_cores; + + /* Derived from OpenSSL_ia32_cpuid. */ + + OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64); + if (cpu_vendor_id == CPU_VENDOR_INTEL) { + OPENSSL_ia32cap_P[0] |= CPUID_IA64; + if ((cpu_id & 0xf00) != 0xf00) + OPENSSL_ia32cap_P[0] |= CPUID_B20; + } + + /* Only leave CPUID_HTT on if HTT is present. */ + if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) { + max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1; + if (cpu_feature & CPUID_HTT) { + if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores) + OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; + } + } else { + if (cpu_high >= 4) { + cpuid_count(4, 0, regs); + max_cores = (regs[0] >> 26) & 0xfff; + } else + max_cores = -1; + } + if (max_cores == 0) + OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; + else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0) + OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; + + OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP; + if (cpu_vendor_id == CPU_VENDOR_AMD) + OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP; + + OPENSSL_ia32cap_P[2] = cpu_stdext_feature; + if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0) + OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F | + CPUID_STDEXT_AVX512DQ); + + /* Disable AVX512F on Skylake-X. */ + if ((cpu_id & 0x0fff0ff0) == 0x00050650) + OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F); + + if (cpu_feature2 & CPUID2_OSXSAVE) + xcr0 = rxcr(0); + else + xcr0 = 0; + + if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) != + (XFEATURE_AVX512 | XFEATURE_AVX)) + OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL | + CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA | + CPUID_STDEXT_AVX512F); + if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) { + OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); + OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; + } +} + +static void +ossl_identify(driver_t *driver, device_t parent) +{ + + if (device_find_child(parent, "ossl", -1) == NULL) + BUS_ADD_CHILD(parent, 10, "ossl", -1); +} + +static int +ossl_probe(device_t dev) +{ + + device_set_desc(dev, "OpenSSL crypto"); + return (BUS_PROBE_DEFAULT); +} + +static int +ossl_attach(device_t dev) +{ + struct ossl_softc *sc; + + sc = device_get_softc(dev); + + ossl_cpuid(); + sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), + CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | + CRYPTOCAP_F_ACCEL_SOFTWARE); + if (sc->sc_cid < 0) { + device_printf(dev, "failed to allocate crypto driver id\n"); + return (ENXIO); + } + + return (0); +} + +static int +ossl_detach(device_t dev) +{ + struct ossl_softc *sc; + + sc = device_get_softc(dev); + + crypto_unregister_all(sc->sc_cid); + + return (0); +} + +static struct auth_hash * +ossl_lookup_hash(const struct crypto_session_params *csp) +{ + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (&ossl_hash_sha1); + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + return (&ossl_hash_sha224); + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + return (&ossl_hash_sha256); + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + return (&ossl_hash_sha384); + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + return (&ossl_hash_sha512); + default: + return (NULL); + } +} + +static int +ossl_probesession(device_t dev, const struct crypto_session_params *csp) +{ + + if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != + 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + if (ossl_lookup_hash(csp) == NULL) + return (EINVAL); + break; + default: + return (EINVAL); + } + + return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); +} + +static void +ossl_setkey_hmac(struct ossl_session *s, const void *key, int klen) +{ + + hmac_init_ipad(s->hash.axf, key, klen, &s->hash.ictx); + hmac_init_opad(s->hash.axf, key, klen, &s->hash.octx); +} + +static int +ossl_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct ossl_session *s; + struct auth_hash *axf; + + s = crypto_get_driver_session(cses); + + axf = ossl_lookup_hash(csp); + s->hash.axf = axf; + if (csp->csp_auth_mlen == 0) + s->hash.mlen = axf->hashsize; + else + s->hash.mlen = csp->csp_auth_mlen; + + if (csp->csp_auth_klen == 0) { + axf->Init(&s->hash.ictx); + } else { + if (csp->csp_auth_key != NULL) { + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + ossl_setkey_hmac(s, csp->csp_auth_key, + csp->csp_auth_klen); + fpu_kern_leave(curthread, NULL); + } + } + return (0); +} + +static int +ossl_process(device_t dev, struct cryptop *crp, int hint) +{ + struct ossl_hash_context ctx; + char digest[HASH_MAX_LEN]; + const struct crypto_session_params *csp; + struct ossl_session *s; + struct auth_hash *axf; + int error; + bool fpu_entered; + + s = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + axf = s->hash.axf; + + if (is_fpu_kern_thread(0)) { + fpu_entered = false; + } else { + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + fpu_entered = true; + } + + if (crp->crp_auth_key != NULL) + ossl_setkey_hmac(s, crp->crp_auth_key, csp->csp_auth_klen); + + ctx = s->hash.ictx; + + if (crp->crp_aad != NULL) + error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); + else + error = crypto_apply(crp, crp->crp_aad_start, + crp->crp_aad_length, axf->Update, &ctx); + if (error) + goto out; + + error = crypto_apply(crp, crp->crp_payload_start, + crp->crp_payload_length, axf->Update, &ctx); + if (error) + goto out; + + axf->Final(digest, &ctx); + + if (csp->csp_auth_klen != 0) { + ctx = s->hash.octx; + axf->Update(&ctx, digest, axf->hashsize); + axf->Final(digest, &ctx); + } + + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + char digest2[HASH_MAX_LEN]; + + crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen, + digest2); + if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0) + error = EBADMSG; + explicit_bzero(digest2, sizeof(digest2)); + } else { + crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen, + digest); + } + explicit_bzero(digest, sizeof(digest)); + +out: + if (fpu_entered) + fpu_kern_leave(curthread, NULL); + + crp->crp_etype = error; + crypto_done(crp); + + explicit_bzero(&ctx, sizeof(ctx)); + return (0); +} + +static device_method_t ossl_methods[] = { + DEVMETHOD(device_identify, ossl_identify), + DEVMETHOD(device_probe, ossl_probe), + DEVMETHOD(device_attach, ossl_attach), + DEVMETHOD(device_detach, ossl_detach), + + DEVMETHOD(cryptodev_probesession, ossl_probesession), + DEVMETHOD(cryptodev_newsession, ossl_newsession), + DEVMETHOD(cryptodev_process, ossl_process), + + DEVMETHOD_END +}; + +static driver_t ossl_driver = { + "ossl", + ossl_methods, + sizeof(struct ossl_softc) +}; + +static devclass_t ossl_devclass; + +DRIVER_MODULE(ossl, nexus, ossl_driver, ossl_devclass, NULL, NULL); +MODULE_VERSION(ossl, 1); +MODULE_DEPEND(ossl, crypto, 1, 1, 1); Index: head/sys/crypto/openssl/ossl_hash.h =================================================================== --- head/sys/crypto/openssl/ossl_hash.h +++ head/sys/crypto/openssl/ossl_hash.h @@ -0,0 +1,146 @@ +/* + * Copyright 1999-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * $FreeBSD$ + */ + +/* + * Derived from include/crypto/md32_common.h + * + * HASH_UPDATE and HASH_FINAL have been updated to work with the + * auth_hash interface. + */ + +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + +# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++))) ) ) +# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff), \ + l) + +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + +# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<<24) ) +# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + l) + +#endif + +/* + * Time for some action :-) + */ + +static int +HASH_UPDATE(void *c_, const void *data_, unsigned int len) +{ + HASH_CTX *c = c_; + const unsigned char *data = data_; + unsigned char *p; + HASH_LONG l; + size_t n; + + if (len == 0) + return 0; + + l = (c->Nl + (((HASH_LONG) len) << 3)) & 0xffffffffUL; + if (l < c->Nl) /* overflow */ + c->Nh++; + c->Nh += (HASH_LONG) (len >> 29); /* might cause compiler warning on + * 16-bit */ + c->Nl = l; + + n = c->num; + if (n != 0) { + p = (unsigned char *)c->data; + + if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK) { + memcpy(p + n, data, HASH_CBLOCK - n); + HASH_BLOCK_DATA_ORDER(c, p, 1); + n = HASH_CBLOCK - n; + data += n; + len -= n; + c->num = 0; + /* + * We use memset rather than OPENSSL_cleanse() here deliberately. + * Using OPENSSL_cleanse() here could be a performance issue. It + * will get properly cleansed on finalisation so this isn't a + * security problem. + */ + memset(p, 0, HASH_CBLOCK); /* keep it zeroed */ + } else { + memcpy(p + n, data, len); + c->num += (unsigned int)len; + return 0; + } + } + + n = len / HASH_CBLOCK; + if (n > 0) { + HASH_BLOCK_DATA_ORDER(c, data, n); + n *= HASH_CBLOCK; + data += n; + len -= n; + } + + if (len != 0) { + p = (unsigned char *)c->data; + c->num = (unsigned int)len; + memcpy(p, data, len); + } + return 0; +} + +static void +HASH_FINAL(uint8_t *md, void *c_) +{ + HASH_CTX *c = c_; + unsigned char *p = (unsigned char *)c->data; + size_t n = c->num; + + p[n] = 0x80; /* there is always room for one */ + n++; + + if (n > (HASH_CBLOCK - 8)) { + memset(p + n, 0, HASH_CBLOCK - n); + n = 0; + HASH_BLOCK_DATA_ORDER(c, p, 1); + } + memset(p + n, 0, HASH_CBLOCK - 8 - n); + + p += HASH_CBLOCK - 8; +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + (void)HOST_l2c(c->Nh, p); + (void)HOST_l2c(c->Nl, p); +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + (void)HOST_l2c(c->Nl, p); + (void)HOST_l2c(c->Nh, p); +#endif + p -= HASH_CBLOCK; + HASH_BLOCK_DATA_ORDER(c, p, 1); + c->num = 0; + OPENSSL_cleanse(p, HASH_CBLOCK); + +#ifndef HASH_MAKE_STRING +# error "HASH_MAKE_STRING must be defined!" +#else + HASH_MAKE_STRING(c, md); +#endif + + return; +} Index: head/sys/crypto/openssl/ossl_sha.h =================================================================== --- head/sys/crypto/openssl/ossl_sha.h +++ head/sys/crypto/openssl/ossl_sha.h @@ -0,0 +1,68 @@ +/* + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * $FreeBSD$ + */ + +#ifndef __OSSL_SHA_H__ +#define __OSSL_SHA_H__ + +/* + * This is always included last which permits the namespace hacks below + * to work. + */ +#define SHA256_CTX OSSL_SHA256_CTX +#define SHA512_CTX OSSL_SHA512_CTX + +/* From include/openssl/sha.h */ +# define SHA_LONG unsigned int + +# define SHA_LBLOCK 16 +# define SHA_CBLOCK (SHA_LBLOCK*4)/* SHA treats input data as a + * contiguous array of 32 bit wide + * big-endian values. */ + +typedef struct SHAstate_st { + SHA_LONG h0, h1, h2, h3, h4; + SHA_LONG Nl, Nh; + SHA_LONG data[SHA_LBLOCK]; + unsigned int num; +} SHA_CTX; + +# define SHA256_CBLOCK (SHA_LBLOCK*4)/* SHA-256 treats input data as a + * contiguous array of 32 bit wide + * big-endian values. */ + +typedef struct SHA256state_st { + SHA_LONG h[8]; + SHA_LONG Nl, Nh; + SHA_LONG data[SHA_LBLOCK]; + unsigned int num, md_len; +} SHA256_CTX; + +/* + * SHA-512 treats input data as a + * contiguous array of 64 bit + * wide big-endian values. + */ +# define SHA512_CBLOCK (SHA_LBLOCK*8) + +# define SHA_LONG64 unsigned long long +# define U64(C) C##ULL + +typedef struct SHA512state_st { + SHA_LONG64 h[8]; + SHA_LONG64 Nl, Nh; + union { + SHA_LONG64 d[SHA_LBLOCK]; + unsigned char p[SHA512_CBLOCK]; + } u; + unsigned int num, md_len; +} SHA512_CTX; + +#endif /* !__OSSL_SHA_H__ */ Index: head/sys/crypto/openssl/ossl_sha1.c =================================================================== --- head/sys/crypto/openssl/ossl_sha1.c +++ head/sys/crypto/openssl/ossl_sha1.c @@ -0,0 +1,77 @@ +/* + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include + +#include +#include + +/* sha1-x86_64.S */ +void sha1_block_data_order(SHA_CTX *c, const void *p, size_t len); + +/* From crypto/sha/sha_local.h */ +#define DATA_ORDER_IS_BIG_ENDIAN + +#define HASH_LONG SHA_LONG +#define HASH_CTX SHA_CTX +#define HASH_CBLOCK SHA_CBLOCK +#define HASH_MAKE_STRING(c,s) do { \ + unsigned long ll; \ + ll=(c)->h0; (void)HOST_l2c(ll,(s)); \ + ll=(c)->h1; (void)HOST_l2c(ll,(s)); \ + ll=(c)->h2; (void)HOST_l2c(ll,(s)); \ + ll=(c)->h3; (void)HOST_l2c(ll,(s)); \ + ll=(c)->h4; (void)HOST_l2c(ll,(s)); \ + } while (0) + +#define HASH_UPDATE ossl_sha1_update +#define HASH_FINAL ossl_sha1_final +#define HASH_INIT ossl_sha1_init +#define HASH_BLOCK_DATA_ORDER sha1_block_data_order + +#define INIT_DATA_h0 0x67452301UL +#define INIT_DATA_h1 0xefcdab89UL +#define INIT_DATA_h2 0x98badcfeUL +#define INIT_DATA_h3 0x10325476UL +#define INIT_DATA_h4 0xc3d2e1f0UL + +static void +HASH_INIT(void *c_) +{ + SHA_CTX *c = c_; + memset(c, 0, sizeof(*c)); + c->h0 = INIT_DATA_h0; + c->h1 = INIT_DATA_h1; + c->h2 = INIT_DATA_h2; + c->h3 = INIT_DATA_h3; + c->h4 = INIT_DATA_h4; +} + +#include "ossl_hash.h" + +struct auth_hash ossl_hash_sha1 = { + .type = CRYPTO_SHA1, + .name = "OpenSSL-SHA1", + .hashsize = SHA1_HASH_LEN, + .ctxsize = sizeof(SHA_CTX), + .blocksize = SHA1_BLOCK_LEN, + .Init = HASH_INIT, + .Update = HASH_UPDATE, + .Final = HASH_FINAL, +}; + +_Static_assert(sizeof(SHA_CTX) <= sizeof(struct ossl_hash_context), + "ossl_hash_context too small"); Index: head/sys/crypto/openssl/ossl_sha256.c =================================================================== --- head/sys/crypto/openssl/ossl_sha256.c +++ head/sys/crypto/openssl/ossl_sha256.c @@ -0,0 +1,121 @@ +/* + * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include + +#include +#include + +/* sha256-x86_64.S */ +void sha256_block_data_order(SHA256_CTX *c, const void *in, size_t num); + +/* From crypto/sha/sha256.c */ + +static void +ossl_sha224_init(void *c_) +{ + SHA256_CTX *c = c_; + memset(c, 0, sizeof(*c)); + c->h[0] = 0xc1059ed8UL; + c->h[1] = 0x367cd507UL; + c->h[2] = 0x3070dd17UL; + c->h[3] = 0xf70e5939UL; + c->h[4] = 0xffc00b31UL; + c->h[5] = 0x68581511UL; + c->h[6] = 0x64f98fa7UL; + c->h[7] = 0xbefa4fa4UL; + c->md_len = SHA224_DIGEST_LENGTH; +} + +static void +ossl_sha256_init(void *c_) +{ + SHA256_CTX *c = c_; + memset(c, 0, sizeof(*c)); + c->h[0] = 0x6a09e667UL; + c->h[1] = 0xbb67ae85UL; + c->h[2] = 0x3c6ef372UL; + c->h[3] = 0xa54ff53aUL; + c->h[4] = 0x510e527fUL; + c->h[5] = 0x9b05688cUL; + c->h[6] = 0x1f83d9abUL; + c->h[7] = 0x5be0cd19UL; + c->md_len = SHA256_DIGEST_LENGTH; +} + + +#define DATA_ORDER_IS_BIG_ENDIAN + +#define HASH_LONG SHA_LONG +#define HASH_CTX SHA256_CTX +#define HASH_CBLOCK SHA_CBLOCK + +/* + * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." + * default: case below covers for it. It's not clear however if it's + * permitted to truncate to amount of bytes not divisible by 4. I bet not, + * but if it is, then default: case shall be extended. For reference. + * Idea behind separate cases for pre-defined lengths is to let the + * compiler decide if it's appropriate to unroll small loops. + */ +#define HASH_MAKE_STRING(c,s) do { \ + unsigned long ll; \ + unsigned int nn; \ + switch ((c)->md_len) \ + { case SHA224_DIGEST_LENGTH: \ + for (nn=0;nnh[nn]; (void)HOST_l2c(ll,(s)); } \ + break; \ + case SHA256_DIGEST_LENGTH: \ + for (nn=0;nnh[nn]; (void)HOST_l2c(ll,(s)); } \ + break; \ + default: \ + __assert_unreachable(); \ + break; \ + } \ + } while (0) + +#define HASH_UPDATE ossl_sha256_update +#define HASH_FINAL ossl_sha256_final +#define HASH_BLOCK_DATA_ORDER sha256_block_data_order + +#include "ossl_hash.h" + +struct auth_hash ossl_hash_sha224 = { + .type = CRYPTO_SHA2_224, + .name = "OpenSSL-SHA2-224", + .hashsize = SHA2_224_HASH_LEN, + .ctxsize = sizeof(SHA256_CTX), + .blocksize = SHA2_224_BLOCK_LEN, + .Init = ossl_sha224_init, + .Update = HASH_UPDATE, + .Final = HASH_FINAL, +}; + +struct auth_hash ossl_hash_sha256 = { + .type = CRYPTO_SHA2_256, + .name = "OpenSSL-SHA2-256", + .hashsize = SHA2_256_HASH_LEN, + .ctxsize = sizeof(SHA256_CTX), + .blocksize = SHA2_256_BLOCK_LEN, + .Init = ossl_sha256_init, + .Update = HASH_UPDATE, + .Final = HASH_FINAL, +}; + +_Static_assert(sizeof(SHA256_CTX) <= sizeof(struct ossl_hash_context), + "ossl_hash_context too small"); Index: head/sys/crypto/openssl/ossl_sha512.c =================================================================== --- head/sys/crypto/openssl/ossl_sha512.c +++ head/sys/crypto/openssl/ossl_sha512.c @@ -0,0 +1,258 @@ +/* + * Copyright 2004-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include + +#include +#include + +/* sha512-x86_64.S */ +void sha512_block_data_order(SHA512_CTX *c, const void *in, size_t num); + +/* From crypto/sha/sha512.c */ + +#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) +# define SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA +#endif + +static void +ossl_sha384_init(void *c_) +{ + SHA512_CTX *c = c_; + c->h[0] = U64(0xcbbb9d5dc1059ed8); + c->h[1] = U64(0x629a292a367cd507); + c->h[2] = U64(0x9159015a3070dd17); + c->h[3] = U64(0x152fecd8f70e5939); + c->h[4] = U64(0x67332667ffc00b31); + c->h[5] = U64(0x8eb44a8768581511); + c->h[6] = U64(0xdb0c2e0d64f98fa7); + c->h[7] = U64(0x47b5481dbefa4fa4); + + c->Nl = 0; + c->Nh = 0; + c->num = 0; + c->md_len = SHA384_DIGEST_LENGTH; +} + +static void +ossl_sha512_init(void *c_) +{ + SHA512_CTX *c = c_; + c->h[0] = U64(0x6a09e667f3bcc908); + c->h[1] = U64(0xbb67ae8584caa73b); + c->h[2] = U64(0x3c6ef372fe94f82b); + c->h[3] = U64(0xa54ff53a5f1d36f1); + c->h[4] = U64(0x510e527fade682d1); + c->h[5] = U64(0x9b05688c2b3e6c1f); + c->h[6] = U64(0x1f83d9abfb41bd6b); + c->h[7] = U64(0x5be0cd19137e2179); + + c->Nl = 0; + c->Nh = 0; + c->num = 0; + c->md_len = SHA512_DIGEST_LENGTH; +} + +static void +ossl_sha512_final(uint8_t *md, void *c_) +{ + SHA512_CTX *c = c_; + unsigned char *p = (unsigned char *)c->u.p; + size_t n = c->num; + + p[n] = 0x80; /* There always is a room for one */ + n++; + if (n > (sizeof(c->u) - 16)) { + memset(p + n, 0, sizeof(c->u) - n); + n = 0; + sha512_block_data_order(c, p, 1); + } + + memset(p + n, 0, sizeof(c->u) - 16 - n); +#if _BYTE_ORDER == _BIG_ENDIAN + c->u.d[SHA_LBLOCK - 2] = c->Nh; + c->u.d[SHA_LBLOCK - 1] = c->Nl; +#else + p[sizeof(c->u) - 1] = (unsigned char)(c->Nl); + p[sizeof(c->u) - 2] = (unsigned char)(c->Nl >> 8); + p[sizeof(c->u) - 3] = (unsigned char)(c->Nl >> 16); + p[sizeof(c->u) - 4] = (unsigned char)(c->Nl >> 24); + p[sizeof(c->u) - 5] = (unsigned char)(c->Nl >> 32); + p[sizeof(c->u) - 6] = (unsigned char)(c->Nl >> 40); + p[sizeof(c->u) - 7] = (unsigned char)(c->Nl >> 48); + p[sizeof(c->u) - 8] = (unsigned char)(c->Nl >> 56); + p[sizeof(c->u) - 9] = (unsigned char)(c->Nh); + p[sizeof(c->u) - 10] = (unsigned char)(c->Nh >> 8); + p[sizeof(c->u) - 11] = (unsigned char)(c->Nh >> 16); + p[sizeof(c->u) - 12] = (unsigned char)(c->Nh >> 24); + p[sizeof(c->u) - 13] = (unsigned char)(c->Nh >> 32); + p[sizeof(c->u) - 14] = (unsigned char)(c->Nh >> 40); + p[sizeof(c->u) - 15] = (unsigned char)(c->Nh >> 48); + p[sizeof(c->u) - 16] = (unsigned char)(c->Nh >> 56); +#endif + + sha512_block_data_order(c, p, 1); + + switch (c->md_len) { + /* Let compiler decide if it's appropriate to unroll... */ + case SHA224_DIGEST_LENGTH: + for (n = 0; n < SHA224_DIGEST_LENGTH / 8; n++) { + SHA_LONG64 t = c->h[n]; + + *(md++) = (unsigned char)(t >> 56); + *(md++) = (unsigned char)(t >> 48); + *(md++) = (unsigned char)(t >> 40); + *(md++) = (unsigned char)(t >> 32); + *(md++) = (unsigned char)(t >> 24); + *(md++) = (unsigned char)(t >> 16); + *(md++) = (unsigned char)(t >> 8); + *(md++) = (unsigned char)(t); + } + /* + * For 224 bits, there are four bytes left over that have to be + * processed separately. + */ + { + SHA_LONG64 t = c->h[SHA224_DIGEST_LENGTH / 8]; + + *(md++) = (unsigned char)(t >> 56); + *(md++) = (unsigned char)(t >> 48); + *(md++) = (unsigned char)(t >> 40); + *(md++) = (unsigned char)(t >> 32); + } + break; + case SHA256_DIGEST_LENGTH: + for (n = 0; n < SHA256_DIGEST_LENGTH / 8; n++) { + SHA_LONG64 t = c->h[n]; + + *(md++) = (unsigned char)(t >> 56); + *(md++) = (unsigned char)(t >> 48); + *(md++) = (unsigned char)(t >> 40); + *(md++) = (unsigned char)(t >> 32); + *(md++) = (unsigned char)(t >> 24); + *(md++) = (unsigned char)(t >> 16); + *(md++) = (unsigned char)(t >> 8); + *(md++) = (unsigned char)(t); + } + break; + case SHA384_DIGEST_LENGTH: + for (n = 0; n < SHA384_DIGEST_LENGTH / 8; n++) { + SHA_LONG64 t = c->h[n]; + + *(md++) = (unsigned char)(t >> 56); + *(md++) = (unsigned char)(t >> 48); + *(md++) = (unsigned char)(t >> 40); + *(md++) = (unsigned char)(t >> 32); + *(md++) = (unsigned char)(t >> 24); + *(md++) = (unsigned char)(t >> 16); + *(md++) = (unsigned char)(t >> 8); + *(md++) = (unsigned char)(t); + } + break; + case SHA512_DIGEST_LENGTH: + for (n = 0; n < SHA512_DIGEST_LENGTH / 8; n++) { + SHA_LONG64 t = c->h[n]; + + *(md++) = (unsigned char)(t >> 56); + *(md++) = (unsigned char)(t >> 48); + *(md++) = (unsigned char)(t >> 40); + *(md++) = (unsigned char)(t >> 32); + *(md++) = (unsigned char)(t >> 24); + *(md++) = (unsigned char)(t >> 16); + *(md++) = (unsigned char)(t >> 8); + *(md++) = (unsigned char)(t); + } + break; + /* ... as well as make sure md_len is not abused. */ + default: + __assert_unreachable(); + } +} + +static int +ossl_sha512_update(void *c_, const void *_data, unsigned int len) +{ + SHA512_CTX *c = c_; + SHA_LONG64 l; + unsigned char *p = c->u.p; + const unsigned char *data = (const unsigned char *)_data; + + if (len == 0) + return 0; + + l = (c->Nl + (((SHA_LONG64) len) << 3)) & U64(0xffffffffffffffff); + if (l < c->Nl) + c->Nh++; + if (sizeof(len) >= 8) + c->Nh += (((SHA_LONG64) len) >> 61); + c->Nl = l; + + if (c->num != 0) { + size_t n = sizeof(c->u) - c->num; + + if (len < n) { + memcpy(p + c->num, data, len), c->num += (unsigned int)len; + return 0; + } else { + memcpy(p + c->num, data, n), c->num = 0; + len -= n, data += n; + sha512_block_data_order(c, p, 1); + } + } + + if (len >= sizeof(c->u)) { +#ifndef SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA + if ((size_t)data % sizeof(c->u.d[0]) != 0) + while (len >= sizeof(c->u)) + memcpy(p, data, sizeof(c->u)), + sha512_block_data_order(c, p, 1), + len -= sizeof(c->u), data += sizeof(c->u); + else +#endif + sha512_block_data_order(c, data, len / sizeof(c->u)), + data += len, len %= sizeof(c->u), data -= len; + } + + if (len != 0) + memcpy(p, data, len), c->num = (int)len; + + return 0; +} + +struct auth_hash ossl_hash_sha384 = { + .type = CRYPTO_SHA2_384, + .name = "OpenSSL-SHA2-384", + .hashsize = SHA2_384_HASH_LEN, + .ctxsize = sizeof(SHA512_CTX), + .blocksize = SHA2_384_BLOCK_LEN, + .Init = ossl_sha384_init, + .Update = ossl_sha512_update, + .Final = ossl_sha512_final, +}; + +struct auth_hash ossl_hash_sha512 = { + .type = CRYPTO_SHA2_512, + .name = "OpenSSL-SHA2-512", + .hashsize = SHA2_512_HASH_LEN, + .ctxsize = sizeof(SHA512_CTX), + .blocksize = SHA2_512_BLOCK_LEN, + .Init = ossl_sha512_init, + .Update = ossl_sha512_update, + .Final = ossl_sha512_final, +}; + +_Static_assert(sizeof(SHA512_CTX) <= sizeof(struct ossl_hash_context), + "ossl_hash_context too small"); Index: head/sys/i386/conf/NOTES =================================================================== --- head/sys/i386/conf/NOTES +++ head/sys/i386/conf/NOTES @@ -768,6 +768,7 @@ device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG device aesni # AES-NI OpenCrypto module +device ossl # OpenSSL OpenCrypto module # # Laptop/Notebook options: Index: head/sys/modules/Makefile =================================================================== --- head/sys/modules/Makefile +++ head/sys/modules/Makefile @@ -274,6 +274,7 @@ ${_nvram} \ oce \ ${_ocs_fc} \ + ${_ossl} \ otus \ ${_otusfw} \ ow \ @@ -626,6 +627,7 @@ _ndis= ndis _ntb= ntb _ocs_fc= ocs_fc +_ossl= ossl _pccard= pccard .if ${MK_OFED} != "no" || defined(ALL_MODULES) _rdma= rdma Index: head/sys/modules/ossl/Makefile =================================================================== --- head/sys/modules/ossl/Makefile +++ head/sys/modules/ossl/Makefile @@ -0,0 +1,26 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/crypto/openssl +.PATH: ${SRCTOP}/sys/crypto/openssl/${MACHINE_CPUARCH} + +KMOD= ossl +SRCS= bus_if.h \ + cryptodev_if.h \ + device_if.h \ + ossl.c \ + ossl_sha1.c \ + ossl_sha256.c \ + ossl_sha512.c \ + ${SRCS.${MACHINE_CPUARCH}} + +SRCS.amd64= \ + sha1-x86_64.S \ + sha256-x86_64.S \ + sha512-x86_64.S + +SRCS.i386= \ + sha1-586.S \ + sha256-586.S \ + sha512-586.S + +.include