Index: head/sys/dev/cxgbe/adapter.h =================================================================== --- head/sys/dev/cxgbe/adapter.h +++ head/sys/dev/cxgbe/adapter.h @@ -1142,7 +1142,6 @@ void t4_iterate(void (*)(struct adapter *, void *), void *); void t4_init_devnames(struct adapter *); void t4_add_adapter(struct adapter *); -void t4_aes_getdeckey(void *, const void *, unsigned int); int t4_detach_common(device_t); int t4_map_bars_0_and_4(struct adapter *); int t4_map_bar_2(struct adapter *); @@ -1169,6 +1168,15 @@ void cxgbe_media_status(struct ifnet *, struct ifmediareq *); bool t4_os_dump_cimla(struct adapter *, int, bool); void t4_os_dump_devlog(struct adapter *); + +/* t4_keyctx.c */ +struct auth_hash; +union authctx; + +void t4_aes_getdeckey(void *, const void *, unsigned int); +void t4_copy_partial_hash(int, union authctx *, void *); +void t4_init_gmac_hash(const char *, int, char *); +void t4_init_hmac_digest(struct auth_hash *, u_int, char *, int, char *); #ifdef DEV_NETMAP /* t4_netmap.c */ Index: head/sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- head/sys/dev/cxgbe/crypto/t4_crypto.c +++ head/sys/dev/cxgbe/crypto/t4_crypto.c @@ -141,8 +141,7 @@ unsigned int partial_digest_len; unsigned int auth_mode; unsigned int mk_size; - char ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; - char opad[CHCR_HASH_MAX_BLOCK_SIZE_128]; + char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2]; }; struct ccr_session_gmac { @@ -530,10 +529,7 @@ V_SCMD_LAST_FRAG(0) | V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1)); - memcpy(crwr->key_ctx.key, s->hmac.ipad, s->hmac.partial_digest_len); - if (use_opad) - memcpy(crwr->key_ctx.key + iopad_size, s->hmac.opad, - s->hmac.partial_digest_len); + memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len); /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */ kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; @@ -1069,8 +1065,7 @@ } dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); - memcpy(dst, s->hmac.ipad, s->hmac.partial_digest_len); - memcpy(dst + iopad_size, s->hmac.opad, s->hmac.partial_digest_len); + memcpy(dst, s->hmac.pads, iopad_size * 2); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); @@ -2212,44 +2207,6 @@ } static void -ccr_copy_partial_hash(void *dst, int cri_alg, union authctx *auth_ctx) -{ - uint32_t *u32; - uint64_t *u64; - u_int i; - - u32 = (uint32_t *)dst; - u64 = (uint64_t *)dst; - switch (cri_alg) { - case CRYPTO_SHA1: - case CRYPTO_SHA1_HMAC: - for (i = 0; i < SHA1_HASH_LEN / 4; i++) - u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); - break; - case CRYPTO_SHA2_224: - case CRYPTO_SHA2_224_HMAC: - for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) - u32[i] = htobe32(auth_ctx->sha224ctx.state[i]); - break; - case CRYPTO_SHA2_256: - case CRYPTO_SHA2_256_HMAC: - for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) - u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); - break; - case CRYPTO_SHA2_384: - case CRYPTO_SHA2_384_HMAC: - for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) - u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); - break; - case CRYPTO_SHA2_512: - case CRYPTO_SHA2_512_HMAC: - for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) - u64[i] = htobe64(auth_ctx->sha512ctx.state[i]); - break; - } -} - -static void ccr_init_hash_digest(struct ccr_session *s, int cri_alg) { union authctx auth_ctx; @@ -2257,66 +2214,9 @@ axf = s->hmac.auth_hash; axf->Init(&auth_ctx); - ccr_copy_partial_hash(s->hmac.ipad, cri_alg, &auth_ctx); + t4_copy_partial_hash(cri_alg, &auth_ctx, s->hmac.pads); } -static void -ccr_init_hmac_digest(struct ccr_session *s, int cri_alg, char *key, - int klen) -{ - union authctx auth_ctx; - struct auth_hash *axf; - u_int i; - - /* - * If the key is larger than the block size, use the digest of - * the key as the key instead. - */ - axf = s->hmac.auth_hash; - klen /= 8; - if (klen > axf->blocksize) { - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, key, klen); - axf->Final(s->hmac.ipad, &auth_ctx); - klen = axf->hashsize; - } else - memcpy(s->hmac.ipad, key, klen); - - memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); - memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); - - for (i = 0; i < axf->blocksize; i++) { - s->hmac.ipad[i] ^= HMAC_IPAD_VAL; - s->hmac.opad[i] ^= HMAC_OPAD_VAL; - } - - /* - * Hash the raw ipad and opad and store the partial result in - * the same buffer. - */ - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, s->hmac.ipad, axf->blocksize); - ccr_copy_partial_hash(s->hmac.ipad, cri_alg, &auth_ctx); - - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize); - ccr_copy_partial_hash(s->hmac.opad, cri_alg, &auth_ctx); -} - -/* - * Borrowed from AES_GMAC_Setkey(). - */ -static void -ccr_init_gmac_hash(struct ccr_session *s, char *key, int klen) -{ - static char zeroes[GMAC_BLOCK_LEN]; - uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; - int rounds; - - rounds = rijndaelKeySetupEnc(keysched, key, klen); - rijndaelEncrypt(keysched, rounds, zeroes, s->gmac.ghash_h); -} - static int ccr_aes_check_keylen(int alg, int klen) { @@ -2613,7 +2513,8 @@ s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = hash->cri_mlen; - ccr_init_gmac_hash(s, hash->cri_key, hash->cri_klen); + t4_init_gmac_hash(hash->cri_key, hash->cri_klen, + s->gmac.ghash_h); } else if (auth_mode == SCMD_AUTH_MODE_CBCMAC) { if (hash->cri_mlen == 0) s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN; @@ -2629,8 +2530,8 @@ else s->hmac.hash_len = hash->cri_mlen; if (hmac) - ccr_init_hmac_digest(s, hash->cri_alg, hash->cri_key, - hash->cri_klen); + t4_init_hmac_digest(auth_hash, partial_digest_len, + hash->cri_key, hash->cri_klen, s->hmac.pads); else ccr_init_hash_digest(s, hash->cri_alg); } @@ -2694,8 +2595,9 @@ break; case HMAC: if (crd->crd_flags & CRD_F_KEY_EXPLICIT) - ccr_init_hmac_digest(s, crd->crd_alg, crd->crd_key, - crd->crd_klen); + t4_init_hmac_digest(s->hmac.auth_hash, + s->hmac.partial_digest_len, crd->crd_key, + crd->crd_klen, s->hmac.pads); error = ccr_hash(sc, s, crp); if (error == 0) sc->stats_hmac++; @@ -2743,8 +2645,9 @@ if (error) break; if (crda->crd_flags & CRD_F_KEY_EXPLICIT) - ccr_init_hmac_digest(s, crda->crd_alg, crda->crd_key, - crda->crd_klen); + t4_init_hmac_digest(s->hmac.auth_hash, + s->hmac.partial_digest_len, crda->crd_key, + crda->crd_klen, s->hmac.pads); if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { error = ccr_aes_check_keylen(crde->crd_alg, crde->crd_klen); @@ -2771,7 +2674,8 @@ crde = crd->crd_next; } if (crda->crd_flags & CRD_F_KEY_EXPLICIT) - ccr_init_gmac_hash(s, crda->crd_key, crda->crd_klen); + t4_init_gmac_hash(crda->crd_key, crda->crd_klen, + s->gmac.ghash_h); if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { error = ccr_aes_check_keylen(crde->crd_alg, crde->crd_klen); Index: head/sys/dev/cxgbe/crypto/t4_keyctx.c =================================================================== --- head/sys/dev/cxgbe/crypto/t4_keyctx.c +++ head/sys/dev/cxgbe/crypto/t4_keyctx.c @@ -0,0 +1,198 @@ +/*- + * Copyright (c) 2017-2019 Chelsio Communications, Inc. + * All rights reserved. + * Written by: John Baldwin + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include + +#include "common/common.h" +#include "crypto/t4_crypto.h" + +/* + * Crypto operations use a key context to store cipher keys and + * partial hash digests. They can either be passed inline as part of + * a work request using crypto or they can be stored in card RAM. For + * the latter case, work requests must replace the inline key context + * with a request to read the context from card RAM. + * + * The format of a key context: + * + * +-------------------------------+ + * | key context header | + * +-------------------------------+ + * | AES key | ----- For requests with AES + * +-------------------------------+ + * | Hash state | ----- For hash-only requests + * +-------------------------------+ - + * | IPAD (16-byte aligned) | \ + * +-------------------------------+ +---- For requests with HMAC + * | OPAD (16-byte aligned) | / + * +-------------------------------+ - + * | GMAC H | ----- For AES-GCM + * +-------------------------------+ - + */ + +/* + * Generate the initial GMAC hash state for a AES-GCM key. + * + * Borrowed from AES_GMAC_Setkey(). + */ +void +t4_init_gmac_hash(const char *key, int klen, char *ghash) +{ + static char zeroes[GMAC_BLOCK_LEN]; + uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; + int rounds; + + rounds = rijndaelKeySetupEnc(keysched, key, klen); + rijndaelEncrypt(keysched, rounds, zeroes, ghash); +} + +/* Copy out the partial hash state from a software hash implementation. */ +void +t4_copy_partial_hash(int alg, union authctx *auth_ctx, void *dst) +{ + uint32_t *u32; + uint64_t *u64; + u_int i; + + u32 = (uint32_t *)dst; + u64 = (uint64_t *)dst; + switch (alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + for (i = 0; i < SHA1_HASH_LEN / 4; i++) + u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); + break; + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_224_HMAC: + for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) + u32[i] = htobe32(auth_ctx->sha224ctx.state[i]); + break; + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) + u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); + break; + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) + u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); + break; + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) + u64[i] = htobe64(auth_ctx->sha512ctx.state[i]); + break; + } +} + +void +t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len, + char *key, int klen, char *dst) +{ + union authctx auth_ctx; + char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN]; + u_int i; + + /* + * If the key is larger than the block size, use the digest of + * the key as the key instead. + */ + klen /= 8; + if (klen > axf->blocksize) { + axf->Init(&auth_ctx); + axf->Update(&auth_ctx, key, klen); + axf->Final(ipad, &auth_ctx); + klen = axf->hashsize; + } else + memcpy(ipad, key, klen); + + memset(ipad + klen, 0, axf->blocksize - klen); + memcpy(opad, ipad, axf->blocksize); + + for (i = 0; i < axf->blocksize; i++) { + ipad[i] ^= HMAC_IPAD_VAL; + opad[i] ^= HMAC_OPAD_VAL; + } + + /* + * Hash the raw ipad and opad and store the partial results in + * the key context. + */ + axf->Init(&auth_ctx); + axf->Update(&auth_ctx, ipad, axf->blocksize); + t4_copy_partial_hash(axf->type, &auth_ctx, dst); + + dst += roundup2(partial_digest_len, 16); + axf->Init(&auth_ctx); + axf->Update(&auth_ctx, opad, axf->blocksize); + t4_copy_partial_hash(axf->type, &auth_ctx, dst); +} + +/* + * Borrowed from cesa_prep_aes_key(). + * + * NB: The crypto engine wants the words in the decryption key in reverse + * order. + */ +void +t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits) +{ + uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; + uint32_t *dkey; + int i; + + rijndaelKeySetupEnc(ek, enc_key, kbits); + dkey = dec_key; + dkey += (kbits / 8) / 4; + + switch (kbits) { + case 128: + for (i = 0; i < 4; i++) + *--dkey = htobe32(ek[4 * 10 + i]); + break; + case 192: + for (i = 0; i < 2; i++) + *--dkey = htobe32(ek[4 * 11 + 2 + i]); + for (i = 0; i < 4; i++) + *--dkey = htobe32(ek[4 * 12 + i]); + break; + case 256: + for (i = 0; i < 4; i++) + *--dkey = htobe32(ek[4 * 13 + i]); + for (i = 0; i < 4; i++) + *--dkey = htobe32(ek[4 * 14 + i]); + break; + } + MPASS(dkey == dec_key); +} Index: head/sys/dev/cxgbe/t4_main.c =================================================================== --- head/sys/dev/cxgbe/t4_main.c +++ head/sys/dev/cxgbe/t4_main.c @@ -71,7 +71,6 @@ #include #include #endif -#include #ifdef DDB #include #include @@ -10850,44 +10849,6 @@ t4_dump_tcb(device_get_softc(dev), tid); } #endif - -/* - * Borrowed from cesa_prep_aes_key(). - * - * NB: The crypto engine wants the words in the decryption key in reverse - * order. - */ -void -t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits) -{ - uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; - uint32_t *dkey; - int i; - - rijndaelKeySetupEnc(ek, enc_key, kbits); - dkey = dec_key; - dkey += (kbits / 8) / 4; - - switch (kbits) { - case 128: - for (i = 0; i < 4; i++) - *--dkey = htobe32(ek[4 * 10 + i]); - break; - case 192: - for (i = 0; i < 2; i++) - *--dkey = htobe32(ek[4 * 11 + 2 + i]); - for (i = 0; i < 4; i++) - *--dkey = htobe32(ek[4 * 12 + i]); - break; - case 256: - for (i = 0; i < 4; i++) - *--dkey = htobe32(ek[4 * 13 + i]); - for (i = 0; i < 4; i++) - *--dkey = htobe32(ek[4 * 14 + i]); - break; - } - MPASS(dkey == dec_key); -} static struct sx mlu; /* mod load unload */ SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); Index: head/sys/dev/cxgbe/tom/t4_tls.c =================================================================== --- head/sys/dev/cxgbe/tom/t4_tls.c +++ head/sys/dev/cxgbe/tom/t4_tls.c @@ -865,88 +865,7 @@ } #ifdef KERN_TLS -/* XXX: Should share this with ccr(4) eventually. */ static void -init_ktls_gmac_hash(const char *key, int klen, char *ghash) -{ - static char zeroes[GMAC_BLOCK_LEN]; - uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; - int rounds; - - rounds = rijndaelKeySetupEnc(keysched, key, klen); - rijndaelEncrypt(keysched, rounds, zeroes, ghash); -} - -/* XXX: Should share this with ccr(4) eventually. */ -static void -ktls_copy_partial_hash(void *dst, int cri_alg, union authctx *auth_ctx) -{ - uint32_t *u32; - uint64_t *u64; - u_int i; - - u32 = (uint32_t *)dst; - u64 = (uint64_t *)dst; - switch (cri_alg) { - case CRYPTO_SHA1_HMAC: - for (i = 0; i < SHA1_HASH_LEN / 4; i++) - u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); - break; - case CRYPTO_SHA2_256_HMAC: - for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) - u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); - break; - case CRYPTO_SHA2_384_HMAC: - for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) - u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); - break; - } -} - -static void -init_ktls_hmac_digest(struct auth_hash *axf, u_int partial_digest_len, - char *key, int klen, char *dst) -{ - union authctx auth_ctx; - char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN]; - u_int i; - - /* - * If the key is larger than the block size, use the digest of - * the key as the key instead. - */ - klen /= 8; - if (klen > axf->blocksize) { - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, key, klen); - axf->Final(ipad, &auth_ctx); - klen = axf->hashsize; - } else - memcpy(ipad, key, klen); - - memset(ipad + klen, 0, axf->blocksize - klen); - memcpy(opad, ipad, axf->blocksize); - - for (i = 0; i < axf->blocksize; i++) { - ipad[i] ^= HMAC_IPAD_VAL; - opad[i] ^= HMAC_OPAD_VAL; - } - - /* - * Hash the raw ipad and opad and store the partial results in - * the key context. - */ - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, ipad, axf->blocksize); - ktls_copy_partial_hash(dst, axf->type, &auth_ctx); - - dst += roundup2(partial_digest_len, 16); - axf->Init(&auth_ctx); - axf->Update(&auth_ctx, opad, axf->blocksize); - ktls_copy_partial_hash(dst, axf->type, &auth_ctx); -} - -static void init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx) { struct auth_hash *axf; @@ -972,7 +891,7 @@ k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP; k_ctx->tx_key_info_size += GMAC_BLOCK_LEN; memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); - init_ktls_gmac_hash(tls->params.cipher_key, + t4_init_gmac_hash(tls->params.cipher_key, tls->params.cipher_key_len * 8, hash); } else { switch (tls->params.auth_algorithm) { @@ -1000,7 +919,7 @@ k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2; k_ctx->mac_secret_size = mac_key_size; - init_ktls_hmac_digest(axf, mac_key_size, tls->params.auth_key, + t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, tls->params.auth_key_len * 8, hash); } Index: head/sys/modules/cxgbe/if_cxgbe/Makefile =================================================================== --- head/sys/modules/cxgbe/if_cxgbe/Makefile +++ head/sys/modules/cxgbe/if_cxgbe/Makefile @@ -3,7 +3,7 @@ # CXGBE= ${SRCTOP}/sys/dev/cxgbe -.PATH: ${CXGBE} ${CXGBE}/common ${CXGBE}/cudbg +.PATH: ${CXGBE} ${CXGBE}/common ${CXGBE}/crypto ${CXGBE}/cudbg KMOD= if_cxgbe SRCS= bus_if.h @@ -20,6 +20,7 @@ SRCS+= t4_hw.c SRCS+= t4_if.c t4_if.h SRCS+= t4_iov.c +SRCS+= t4_keyctx.c SRCS+= t4_l2t.c SRCS+= t4_main.c SRCS+= t4_mp_ring.c