diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4 --- a/share/man/man4/ossl.4 +++ b/share/man/man4/ossl.4 @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 4, 2020 +.Dd March 3, 2021 .Dt OSSL 4 .Os .Sh NAME @@ -74,6 +74,8 @@ .Pp .Bl -bullet -compact .It +Poly1305 +.It SHA1 .It SHA1-HMAC diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -739,6 +739,7 @@ crypto/des/des_ecb.c optional netsmb crypto/des/des_setkey.c optional netsmb crypto/openssl/ossl.c optional ossl +crypto/openssl/ossl_poly1305.c optional ossl crypto/openssl/ossl_sha1.c optional ossl crypto/openssl/ossl_sha256.c optional ossl crypto/openssl/ossl_sha512.c optional ossl diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -137,6 +137,7 @@ cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb +crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl crypto/openssl/amd64/sha256-x86_64.S optional ossl crypto/openssl/amd64/sha512-x86_64.S optional ossl diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -125,6 +125,8 @@ crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl +crypto/openssl/aarch64/poly1305-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/sha1-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/sha256-armv8.S optional ossl \ diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -77,6 +77,7 @@ compat/linux/linux.c optional compat_linux crypto/aesni/aeskeys_i386.S optional aesni crypto/des/arch/i386/des_enc.S optional netsmb +crypto/openssl/i386/poly1305-x86.S optional ossl crypto/openssl/i386/sha1-586.S optional ossl crypto/openssl/i386/sha256-586.S optional ossl crypto/openssl/i386/sha512-586.S optional ossl diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h --- a/sys/crypto/openssl/ossl.h +++ b/sys/crypto/openssl/ossl.h @@ -38,9 +38,10 @@ /* Needs to be big enough to hold any hash context. */ struct ossl_hash_context { - uint32_t dummy[54]; + uint32_t dummy[61]; } __aligned(32); +extern struct auth_hash ossl_hash_poly1305; extern struct auth_hash ossl_hash_sha1; extern struct auth_hash ossl_hash_sha224; extern struct auth_hash ossl_hash_sha256; diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c --- a/sys/crypto/openssl/ossl.c +++ b/sys/crypto/openssl/ossl.c @@ -135,6 +135,8 @@ case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (&ossl_hash_sha512); + case CRYPTO_POLY1305: + return (&ossl_hash_poly1305); default: return (NULL); } @@ -159,14 +161,6 @@ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } -static void -ossl_setkey_hmac(struct ossl_session *s, const void *key, int klen) -{ - - hmac_init_ipad(s->hash.axf, key, klen, &s->hash.ictx); - hmac_init_opad(s->hash.axf, key, klen, &s->hash.octx); -} - static int ossl_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) @@ -188,8 +182,16 @@ } else { if (csp->csp_auth_key != NULL) { fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); - ossl_setkey_hmac(s, csp->csp_auth_key, - csp->csp_auth_klen); + if (axf->Setkey != NULL) { + axf->Init(&s->hash.ictx); + axf->Setkey(&s->hash.ictx, csp->csp_auth_key, + csp->csp_auth_klen); + } else { + hmac_init_ipad(axf, csp->csp_auth_key, + csp->csp_auth_klen, &s->hash.ictx); + hmac_init_opad(axf, csp->csp_auth_key, + csp->csp_auth_klen, &s->hash.octx); + } fpu_kern_leave(curthread, NULL); } } @@ -218,10 +220,18 @@ fpu_entered = true; } - if (crp->crp_auth_key != NULL) - ossl_setkey_hmac(s, crp->crp_auth_key, csp->csp_auth_klen); - - ctx = s->hash.ictx; + if (crp->crp_auth_key == NULL) { + ctx = s->hash.ictx; + } else { + if (axf->Setkey != NULL) { + axf->Init(&ctx); + axf->Setkey(&ctx, crp->crp_auth_key, + csp->csp_auth_klen); + } else { + hmac_init_ipad(axf, crp->crp_auth_key, + csp->csp_auth_klen, &ctx); + } + } if (crp->crp_aad != NULL) error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); @@ -238,8 +248,12 @@ axf->Final(digest, &ctx); - if (csp->csp_auth_klen != 0) { - ctx = s->hash.octx; + if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) { + if (crp->crp_auth_key == NULL) + ctx = s->hash.octx; + else + hmac_init_opad(axf, crp->crp_auth_key, + csp->csp_auth_klen, &ctx); axf->Update(&ctx, digest, axf->hashsize); axf->Final(digest, &ctx); } diff --git a/sys/crypto/openssl/ossl_poly1305.h b/sys/crypto/openssl/ossl_poly1305.h new file mode 100644 --- /dev/null +++ b/sys/crypto/openssl/ossl_poly1305.h @@ -0,0 +1,35 @@ +/* + * Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* From include/crypto/poly1305.h */ + +#define POLY1305_BLOCK_SIZE 16 + +typedef struct poly1305_context POLY1305; + +/* From crypto/poly1305/poly1305_local.h */ + +typedef void (*poly1305_blocks_f) (void *ctx, const unsigned char *inp, + size_t len, unsigned int padbit); +typedef void (*poly1305_emit_f) (void *ctx, unsigned char mac[16], + const unsigned int nonce[4]); + +struct poly1305_context { + double opaque[24]; /* large enough to hold internal state, declared + * 'double' to ensure at least 64-bit invariant + * alignment across all platforms and + * configurations */ + unsigned int nonce[4]; + unsigned char data[POLY1305_BLOCK_SIZE]; + size_t num; + struct { + poly1305_blocks_f blocks; + poly1305_emit_f emit; + } func; +}; diff --git a/sys/crypto/openssl/ossl_poly1305.c b/sys/crypto/openssl/ossl_poly1305.c new file mode 100644 --- /dev/null +++ b/sys/crypto/openssl/ossl_poly1305.c @@ -0,0 +1,181 @@ +/* + * Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include + +#include +#include + +#include +#include + +#define POLY1305_ASM + +/* From crypto/poly1305/poly1305.c */ + +/* pick 32-bit unsigned integer in little endian order */ +static unsigned int U8TOU32(const unsigned char *p) +{ + return (((unsigned int)(p[0] & 0xff)) | + ((unsigned int)(p[1] & 0xff) << 8) | + ((unsigned int)(p[2] & 0xff) << 16) | + ((unsigned int)(p[3] & 0xff) << 24)); +} + +/* + * Implementations can be classified by amount of significant bits in + * words making up the multi-precision value, or in other words radix + * or base of numerical representation, e.g. base 2^64, base 2^32, + * base 2^26. Complementary characteristic is how wide is the result of + * multiplication of pair of digits, e.g. it would take 128 bits to + * accommodate multiplication result in base 2^64 case. These are used + * interchangeably. To describe implementation that is. But interface + * is designed to isolate this so that low-level primitives implemented + * in assembly can be self-contained/self-coherent. + */ +int poly1305_init(void *ctx, const unsigned char key[16], void *func); +void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, + unsigned int padbit); +void poly1305_emit(void *ctx, unsigned char mac[16], + const unsigned int nonce[4]); + +static void Poly1305_Init(POLY1305 *ctx, const unsigned char key[32]) +{ + ctx->nonce[0] = U8TOU32(&key[16]); + ctx->nonce[1] = U8TOU32(&key[20]); + ctx->nonce[2] = U8TOU32(&key[24]); + ctx->nonce[3] = U8TOU32(&key[28]); + + /* + * Unlike reference poly1305_init assembly counterpart is expected + * to return a value: non-zero if it initializes ctx->func, and zero + * otherwise. Latter is to simplify assembly in cases when there no + * multiple code paths to switch between. + */ + if (!poly1305_init(ctx->opaque, key, &ctx->func)) { + ctx->func.blocks = poly1305_blocks; + ctx->func.emit = poly1305_emit; + } + + ctx->num = 0; + +} + +#ifdef POLY1305_ASM +/* + * This "eclipses" poly1305_blocks and poly1305_emit, but it's + * conscious choice imposed by -Wshadow compiler warnings. + */ +# define poly1305_blocks (*poly1305_blocks_p) +# define poly1305_emit (*poly1305_emit_p) +#endif + +static void Poly1305_Update(POLY1305 *ctx, const unsigned char *inp, size_t len) +{ +#ifdef POLY1305_ASM + /* + * As documented, poly1305_blocks is never called with input + * longer than single block and padbit argument set to 0. This + * property is fluently used in assembly modules to optimize + * padbit handling on loop boundary. + */ + poly1305_blocks_f poly1305_blocks_p = ctx->func.blocks; +#endif + size_t rem, num; + + if ((num = ctx->num)) { + rem = POLY1305_BLOCK_SIZE - num; + if (len >= rem) { + memcpy(ctx->data + num, inp, rem); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1); + inp += rem; + len -= rem; + } else { + /* Still not enough data to process a block. */ + memcpy(ctx->data + num, inp, len); + ctx->num = num + len; + return; + } + } + + rem = len % POLY1305_BLOCK_SIZE; + len -= rem; + + if (len >= POLY1305_BLOCK_SIZE) { + poly1305_blocks(ctx->opaque, inp, len, 1); + inp += len; + } + + if (rem) + memcpy(ctx->data, inp, rem); + + ctx->num = rem; +} + +static void Poly1305_Final(POLY1305 *ctx, unsigned char mac[16]) +{ +#ifdef POLY1305_ASM + poly1305_blocks_f poly1305_blocks_p = ctx->func.blocks; + poly1305_emit_f poly1305_emit_p = ctx->func.emit; +#endif + size_t num; + + if ((num = ctx->num)) { + ctx->data[num++] = 1; /* pad bit */ + while (num < POLY1305_BLOCK_SIZE) + ctx->data[num++] = 0; + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0); + } + + poly1305_emit(ctx->opaque, mac, ctx->nonce); + + /* zero out the state */ + OPENSSL_cleanse(ctx, sizeof(*ctx)); +} + +static void +ossl_poly1305_init(void *vctx) +{ +} + +static void +ossl_poly1305_setkey(void *vctx, const uint8_t *key, u_int klen) +{ + MPASS(klen == 32); + Poly1305_Init(vctx, key); +} + +static int +ossl_poly1305_update(void *vctx, const void *buf, u_int len) +{ + Poly1305_Update(vctx, buf, len); + return (0); +} + +static void +ossl_poly1305_final(uint8_t *digest, void *vctx) +{ + Poly1305_Final(vctx, digest); +} + +struct auth_hash ossl_hash_poly1305 = { + .type = CRYPTO_POLY1305, + .name = "OpenSSL-Poly1305", + .hashsize = POLY1305_HASH_LEN, + .ctxsize = sizeof(struct poly1305_context), + .blocksize = POLY1305_BLOCK_SIZE, + .Init = ossl_poly1305_init, + .Setkey = ossl_poly1305_setkey, + .Update = ossl_poly1305_update, + .Final = ossl_poly1305_final, +}; + +_Static_assert(sizeof(struct poly1305_context) <= + sizeof(struct ossl_hash_context), "ossl_hash_context too small"); diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -8,24 +8,28 @@ cryptodev_if.h \ device_if.h \ ossl.c \ + ossl_poly1305.c \ ossl_sha1.c \ ossl_sha256.c \ ossl_sha512.c \ ${SRCS.${MACHINE_CPUARCH}} SRCS.aarch64= \ + poly1305-armv8.S \ sha1-armv8.S \ sha256-armv8.S \ sha512-armv8.S \ ossl_aarch64.c SRCS.amd64= \ + poly1305-x86_64.S \ sha1-x86_64.S \ sha256-x86_64.S \ sha512-x86_64.S \ ossl_x86.c SRCS.i386= \ + poly1305-x86.S \ sha1-586.S \ sha256-586.S \ sha512-586.S \