diff --git a/sys/crypto/aesni/aesni.c b/sys/crypto/aesni/aesni.c index de797abd1af5..eea32532131f 100644 --- a/sys/crypto/aesni/aesni.c +++ b/sys/crypto/aesni/aesni.c @@ -1,940 +1,939 @@ /*- * Copyright (c) 2005-2008 Pawel Jakub Dawidek * Copyright (c) 2010 Konstantin Belousov * Copyright (c) 2014-2021 The FreeBSD Foundation * Copyright (c) 2017 Conrad Meyer * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Portions of this software were developed by Ararat River * Consulting, LLC under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct mtx_padalign *ctx_mtx; static struct fpu_kern_ctx **ctx_fpu; struct aesni_softc { int32_t cid; bool has_aes; bool has_sha; }; #define ACQUIRE_CTX(i, ctx) \ do { \ (i) = PCPU_GET(cpuid); \ mtx_lock(&ctx_mtx[(i)]); \ (ctx) = ctx_fpu[(i)]; \ } while (0) #define RELEASE_CTX(i, ctx) \ do { \ mtx_unlock(&ctx_mtx[(i)]); \ (i) = -1; \ (ctx) = NULL; \ } while (0) static int aesni_cipher_setup(struct aesni_session *ses, const struct crypto_session_params *csp); static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp); static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp, const struct crypto_session_params *csp); static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp, const struct crypto_session_params *csp); MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data"); static void aesni_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "aesni", -1) == NULL && BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0) panic("aesni: could not attach"); } static void detect_cpu_features(bool *has_aes, bool *has_sha) { *has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 && (cpu_feature2 & CPUID2_SSE41) != 0); *has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 && (cpu_feature2 & CPUID2_SSSE3) != 0); } static int aesni_probe(device_t dev) { bool has_aes, has_sha; detect_cpu_features(&has_aes, &has_sha); if (!has_aes && !has_sha) { device_printf(dev, "No AES or SHA support.\n"); return (EINVAL); } else if (has_aes && has_sha) device_set_desc(dev, "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256"); else if (has_aes) device_set_desc(dev, "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS"); else device_set_desc(dev, "SHA1,SHA256"); return (0); } static void aesni_cleanctx(void) { int i; /* XXX - no way to return driverid */ CPU_FOREACH(i) { if (ctx_fpu[i] != NULL) { mtx_destroy(&ctx_mtx[i]); fpu_kern_free_ctx(ctx_fpu[i]); } ctx_fpu[i] = NULL; } free(ctx_mtx, M_AESNI); ctx_mtx = NULL; free(ctx_fpu, M_AESNI); ctx_fpu = NULL; } static int aesni_attach(device_t dev) { struct aesni_softc *sc; int i; sc = device_get_softc(dev); sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); if (sc->cid < 0) { device_printf(dev, "Could not get crypto driver id.\n"); return (ENOMEM); } ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI, M_WAITOK|M_ZERO); ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI, M_WAITOK|M_ZERO); CPU_FOREACH(i) { #ifdef __amd64__ ctx_fpu[i] = fpu_kern_alloc_ctx_domain( pcpu_find(i)->pc_domain, FPU_KERN_NORMAL); #else ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL); #endif mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW); } detect_cpu_features(&sc->has_aes, &sc->has_sha); return (0); } static int aesni_detach(device_t dev) { struct aesni_softc *sc; sc = device_get_softc(dev); crypto_unregister_all(sc->cid); aesni_cleanctx(); return (0); } static bool aesni_auth_supported(struct aesni_softc *sc, const struct crypto_session_params *csp) { if (!sc->has_sha) return (false); switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: break; default: return (false); } return (true); } static bool aesni_cipher_supported(struct aesni_softc *sc, const struct crypto_session_params *csp) { if (!sc->has_aes) return (false); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: CRYPTDEB("invalid CBC/ICM key length"); return (false); } if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: switch (csp->csp_cipher_klen * 8) { case 256: case 512: break; default: CRYPTDEB("invalid XTS key length"); return (false); } if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (true); } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) static int aesni_probesession(device_t dev, const struct crypto_session_params *csp) { struct aesni_softc *sc; sc = device_get_softc(dev); if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_DIGEST: if (!aesni_auth_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_CIPHER: if (!aesni_cipher_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: CRYPTDEB("invalid GCM key length"); return (EINVAL); } if (csp->csp_auth_mlen != 0 && csp->csp_auth_mlen != GMAC_DIGEST_LEN) return (EINVAL); - if (csp->csp_ivlen != AES_GCM_IV_LEN || - !sc->has_aes) + if (!sc->has_aes) return (EINVAL); break; case CRYPTO_AES_CCM_16: switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: CRYPTDEB("invalid CCM key length"); return (EINVAL); } if (!sc->has_aes) return (EINVAL); break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!aesni_auth_supported(sc, csp) || !aesni_cipher_supported(sc, csp)) return (EINVAL); break; default: return (EINVAL); } return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } static int aesni_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct aesni_softc *sc; struct aesni_session *ses; int error; sc = device_get_softc(dev); ses = crypto_get_driver_session(cses); switch (csp->csp_mode) { case CSP_MODE_DIGEST: case CSP_MODE_CIPHER: case CSP_MODE_AEAD: case CSP_MODE_ETA: break; default: return (EINVAL); } error = aesni_cipher_setup(ses, csp); if (error != 0) { CRYPTDEB("setup failed"); return (error); } return (0); } static int aesni_process(device_t dev, struct cryptop *crp, int hint __unused) { struct aesni_session *ses; int error; ses = crypto_get_driver_session(crp->crp_session); error = aesni_cipher_process(ses, crp); crp->crp_etype = error; crypto_done(crp); return (0); } static uint8_t * aesni_cipher_alloc(struct cryptop *crp, int start, int length, bool *allocated) { uint8_t *addr; addr = crypto_contiguous_subsegment(crp, start, length); if (addr != NULL) { *allocated = false; return (addr); } addr = malloc(length, M_AESNI, M_NOWAIT); if (addr != NULL) { *allocated = true; crypto_copydata(crp, start, length, addr); } else *allocated = false; return (addr); } static device_method_t aesni_methods[] = { DEVMETHOD(device_identify, aesni_identify), DEVMETHOD(device_probe, aesni_probe), DEVMETHOD(device_attach, aesni_attach), DEVMETHOD(device_detach, aesni_detach), DEVMETHOD(cryptodev_probesession, aesni_probesession), DEVMETHOD(cryptodev_newsession, aesni_newsession), DEVMETHOD(cryptodev_process, aesni_process), DEVMETHOD_END }; static driver_t aesni_driver = { "aesni", aesni_methods, sizeof(struct aesni_softc), }; static devclass_t aesni_devclass; DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0); MODULE_VERSION(aesni, 1); MODULE_DEPEND(aesni, crypto, 1, 1, 1); static int intel_sha1_update(void *vctx, const void *vdata, u_int datalen) { struct sha1_ctxt *ctx = vctx; const char *data = vdata; size_t gaplen; size_t gapstart; size_t off; size_t copysiz; u_int blocks; off = 0; /* Do any aligned blocks without redundant copying. */ if (datalen >= 64 && ctx->count % 64 == 0) { blocks = datalen / 64; ctx->c.b64[0] += blocks * 64 * 8; intel_sha1_step(ctx->h.b32, data + off, blocks); off += blocks * 64; } while (off < datalen) { gapstart = ctx->count % 64; gaplen = 64 - gapstart; copysiz = (gaplen < datalen - off) ? gaplen : datalen - off; bcopy(&data[off], &ctx->m.b8[gapstart], copysiz); ctx->count += copysiz; ctx->count %= 64; ctx->c.b64[0] += copysiz * 8; if (ctx->count % 64 == 0) intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1); off += copysiz; } return (0); } static void SHA1_Init_fn(void *ctx) { sha1_init(ctx); } static void SHA1_Finalize_fn(void *digest, void *ctx) { sha1_result(ctx, digest); } static int intel_sha256_update(void *vctx, const void *vdata, u_int len) { SHA256_CTX *ctx = vctx; uint64_t bitlen; uint32_t r; u_int blocks; const unsigned char *src = vdata; /* Number of bytes left in the buffer from previous updates */ r = (ctx->count >> 3) & 0x3f; /* Convert the length into a number of bits */ bitlen = len << 3; /* Update number of bits */ ctx->count += bitlen; /* Handle the case where we don't need to perform any transforms */ if (len < 64 - r) { memcpy(&ctx->buf[r], src, len); return (0); } /* Finish the current block */ memcpy(&ctx->buf[r], src, 64 - r); intel_sha256_step(ctx->state, ctx->buf, 1); src += 64 - r; len -= 64 - r; /* Perform complete blocks */ if (len >= 64) { blocks = len / 64; intel_sha256_step(ctx->state, src, blocks); src += blocks * 64; len -= blocks * 64; } /* Copy left over data into buffer */ memcpy(ctx->buf, src, len); return (0); } static void SHA224_Init_fn(void *ctx) { SHA224_Init(ctx); } static void SHA224_Finalize_fn(void *digest, void *ctx) { SHA224_Final(digest, ctx); } static void SHA256_Init_fn(void *ctx) { SHA256_Init(ctx); } static void SHA256_Finalize_fn(void *digest, void *ctx) { SHA256_Final(digest, ctx); } static int aesni_authprepare(struct aesni_session *ses, int klen) { if (klen > SHA1_BLOCK_LEN) return (EINVAL); if ((ses->hmac && klen == 0) || (!ses->hmac && klen != 0)) return (EINVAL); return (0); } static int aesni_cipher_setup(struct aesni_session *ses, const struct crypto_session_params *csp) { struct fpu_kern_ctx *ctx; uint8_t *schedbase; int kt, ctxidx, error; schedbase = (uint8_t *)roundup2((uintptr_t)ses->schedules, AES_SCHED_ALIGN); ses->enc_schedule = schedbase; ses->dec_schedule = schedbase + AES_SCHED_LEN; ses->xts_schedule = schedbase + AES_SCHED_LEN * 2; switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: ses->hmac = true; /* FALLTHROUGH */ case CRYPTO_SHA1: ses->hash_len = SHA1_HASH_LEN; ses->hash_init = SHA1_Init_fn; ses->hash_update = intel_sha1_update; ses->hash_finalize = SHA1_Finalize_fn; break; case CRYPTO_SHA2_224_HMAC: ses->hmac = true; /* FALLTHROUGH */ case CRYPTO_SHA2_224: ses->hash_len = SHA2_224_HASH_LEN; ses->hash_init = SHA224_Init_fn; ses->hash_update = intel_sha256_update; ses->hash_finalize = SHA224_Finalize_fn; break; case CRYPTO_SHA2_256_HMAC: ses->hmac = true; /* FALLTHROUGH */ case CRYPTO_SHA2_256: ses->hash_len = SHA2_256_HASH_LEN; ses->hash_init = SHA256_Init_fn; ses->hash_update = intel_sha256_update; ses->hash_finalize = SHA256_Finalize_fn; break; } if (ses->hash_len != 0) { if (csp->csp_auth_mlen == 0) ses->mlen = ses->hash_len; else ses->mlen = csp->csp_auth_mlen; error = aesni_authprepare(ses, csp->csp_auth_klen); if (error != 0) return (error); } else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) { if (csp->csp_auth_mlen == 0) ses->mlen = AES_CBC_MAC_HASH_LEN; else ses->mlen = csp->csp_auth_mlen; } kt = is_fpu_kern_thread(0) || (csp->csp_cipher_alg == 0); if (!kt) { ACQUIRE_CTX(ctxidx, ctx); fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); } error = 0; if (csp->csp_cipher_key != NULL) aesni_cipher_setup_common(ses, csp, csp->csp_cipher_key, csp->csp_cipher_klen); if (!kt) { fpu_kern_leave(curthread, ctx); RELEASE_CTX(ctxidx, ctx); } return (error); } static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp) { const struct crypto_session_params *csp; struct fpu_kern_ctx *ctx; int error, ctxidx; bool kt; csp = crypto_get_params(crp->crp_session); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CCM_16: if (crp->crp_payload_length > ccm_max_payload_length(csp)) return (EMSGSIZE); /* FALLTHROUGH */ case CRYPTO_AES_ICM: case CRYPTO_AES_NIST_GCM_16: if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); break; case CRYPTO_AES_CBC: case CRYPTO_AES_XTS: /* CBC & XTS can only handle full blocks for now */ if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); break; } ctx = NULL; ctxidx = 0; error = 0; kt = is_fpu_kern_thread(0); if (!kt) { ACQUIRE_CTX(ctxidx, ctx); fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); } /* Do work */ if (csp->csp_mode == CSP_MODE_ETA) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { error = aesni_cipher_crypt(ses, crp, csp); if (error == 0) error = aesni_cipher_mac(ses, crp, csp); } else { error = aesni_cipher_mac(ses, crp, csp); if (error == 0) error = aesni_cipher_crypt(ses, crp, csp); } } else if (csp->csp_mode == CSP_MODE_DIGEST) error = aesni_cipher_mac(ses, crp, csp); else error = aesni_cipher_crypt(ses, crp, csp); if (!kt) { fpu_kern_leave(curthread, ctx); RELEASE_CTX(ctxidx, ctx); } return (error); } static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp, const struct crypto_session_params *csp) { uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; uint8_t *authbuf, *buf, *outbuf; int error; bool encflag, allocated, authallocated, outallocated, outcopy; if (crp->crp_payload_length == 0) { buf = NULL; allocated = false; } else { buf = aesni_cipher_alloc(crp, crp->crp_payload_start, crp->crp_payload_length, &allocated); if (buf == NULL) return (ENOMEM); } outallocated = false; authallocated = false; authbuf = NULL; if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 || csp->csp_cipher_alg == CRYPTO_AES_CCM_16) { if (crp->crp_aad_length == 0) { authbuf = NULL; } else if (crp->crp_aad != NULL) { authbuf = crp->crp_aad; } else { authbuf = aesni_cipher_alloc(crp, crp->crp_aad_start, crp->crp_aad_length, &authallocated); if (authbuf == NULL) { error = ENOMEM; goto out; } } } if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && crp->crp_payload_length > 0) { outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf, crp->crp_payload_output_start, crp->crp_payload_length); if (outbuf == NULL) { outcopy = true; if (allocated) outbuf = buf; else { outbuf = malloc(crp->crp_payload_length, M_AESNI, M_NOWAIT); if (outbuf == NULL) { error = ENOMEM; goto out; } outallocated = true; } } else outcopy = false; } else { outbuf = buf; outcopy = allocated; } error = 0; encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); if (crp->crp_cipher_key != NULL) aesni_cipher_setup_common(ses, csp, crp->crp_cipher_key, csp->csp_cipher_klen); crypto_read_iv(crp, iv); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (encflag) aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, crp->crp_payload_length, buf, outbuf, iv); else { if (buf != outbuf) memcpy(outbuf, buf, crp->crp_payload_length); aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, crp->crp_payload_length, outbuf, iv); } break; case CRYPTO_AES_ICM: /* encryption & decryption are the same */ aesni_encrypt_icm(ses->rounds, ses->enc_schedule, crp->crp_payload_length, buf, outbuf, iv); break; case CRYPTO_AES_XTS: if (encflag) aesni_encrypt_xts(ses->rounds, ses->enc_schedule, ses->xts_schedule, crp->crp_payload_length, buf, outbuf, iv); else aesni_decrypt_xts(ses->rounds, ses->dec_schedule, ses->xts_schedule, crp->crp_payload_length, buf, outbuf, iv); break; case CRYPTO_AES_NIST_GCM_16: if (encflag) { memset(tag, 0, sizeof(tag)); AES_GCM_encrypt(buf, outbuf, authbuf, iv, tag, crp->crp_payload_length, crp->crp_aad_length, csp->csp_ivlen, ses->enc_schedule, ses->rounds); crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), tag); } else { crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), tag); if (!AES_GCM_decrypt(buf, outbuf, authbuf, iv, tag, crp->crp_payload_length, crp->crp_aad_length, csp->csp_ivlen, ses->enc_schedule, ses->rounds)) error = EBADMSG; } break; case CRYPTO_AES_CCM_16: if (encflag) { memset(tag, 0, sizeof(tag)); AES_CCM_encrypt(buf, outbuf, authbuf, iv, tag, crp->crp_payload_length, crp->crp_aad_length, csp->csp_ivlen, ses->mlen, ses->enc_schedule, ses->rounds); crypto_copyback(crp, crp->crp_digest_start, ses->mlen, tag); } else { crypto_copydata(crp, crp->crp_digest_start, ses->mlen, tag); if (!AES_CCM_decrypt(buf, outbuf, authbuf, iv, tag, crp->crp_payload_length, crp->crp_aad_length, csp->csp_ivlen, ses->mlen, ses->enc_schedule, ses->rounds)) error = EBADMSG; } break; } if (outcopy && error == 0) crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ? crp->crp_payload_output_start : crp->crp_payload_start, crp->crp_payload_length, outbuf); out: if (allocated) zfree(buf, M_AESNI); if (authallocated) zfree(authbuf, M_AESNI); if (outallocated) zfree(outbuf, M_AESNI); explicit_bzero(iv, sizeof(iv)); explicit_bzero(tag, sizeof(tag)); return (error); } static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp, const struct crypto_session_params *csp) { union { struct SHA256Context sha2 __aligned(16); struct sha1_ctxt sha1 __aligned(16); } sctx; uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)]; const uint8_t *key; int i, keylen; if (crp->crp_auth_key != NULL) key = crp->crp_auth_key; else key = csp->csp_auth_key; keylen = csp->csp_auth_klen; if (ses->hmac) { uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16); /* Inner hash: (K ^ IPAD) || data */ ses->hash_init(&sctx); for (i = 0; i < keylen; i++) hmac_key[i] = key[i] ^ HMAC_IPAD_VAL; for (i = keylen; i < sizeof(hmac_key); i++) hmac_key[i] = 0 ^ HMAC_IPAD_VAL; ses->hash_update(&sctx, hmac_key, sizeof(hmac_key)); if (crp->crp_aad != NULL) ses->hash_update(&sctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, ses->hash_update, &sctx); if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) crypto_apply_buf(&crp->crp_obuf, crp->crp_payload_output_start, crp->crp_payload_length, ses->hash_update, &sctx); else crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, ses->hash_update, &sctx); if (csp->csp_flags & CSP_F_ESN) ses->hash_update(&sctx, crp->crp_esn, 4); ses->hash_finalize(res, &sctx); /* Outer hash: (K ^ OPAD) || inner hash */ ses->hash_init(&sctx); for (i = 0; i < keylen; i++) hmac_key[i] = key[i] ^ HMAC_OPAD_VAL; for (i = keylen; i < sizeof(hmac_key); i++) hmac_key[i] = 0 ^ HMAC_OPAD_VAL; ses->hash_update(&sctx, hmac_key, sizeof(hmac_key)); ses->hash_update(&sctx, res, ses->hash_len); ses->hash_finalize(res, &sctx); explicit_bzero(hmac_key, sizeof(hmac_key)); } else { ses->hash_init(&sctx); if (crp->crp_aad != NULL) ses->hash_update(&sctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, ses->hash_update, &sctx); if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) crypto_apply_buf(&crp->crp_obuf, crp->crp_payload_output_start, crp->crp_payload_length, ses->hash_update, &sctx); else crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, ses->hash_update, &sctx); ses->hash_finalize(res, &sctx); } if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { uint32_t res2[SHA2_256_HASH_LEN / sizeof(uint32_t)]; crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2); if (timingsafe_bcmp(res, res2, ses->mlen) != 0) return (EBADMSG); explicit_bzero(res2, sizeof(res2)); } else crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res); explicit_bzero(res, sizeof(res)); return (0); } diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c index 077667a4bb2a..18b0870f380b 100644 --- a/sys/crypto/armv8/armv8_crypto.c +++ b/sys/crypto/armv8/armv8_crypto.c @@ -1,510 +1,508 @@ /*- * Copyright (c) 2005-2008 Pawel Jakub Dawidek * Copyright (c) 2010 Konstantin Belousov * Copyright (c) 2014,2016 The FreeBSD Foundation * Copyright (c) 2020 Ampere Computing * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This is based on the aesni code. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct armv8_crypto_softc { int dieing; int32_t cid; struct rwlock lock; bool has_pmul; }; static struct mtx *ctx_mtx; static struct fpu_kern_ctx **ctx_vfp; #define AQUIRE_CTX(i, ctx) \ do { \ (i) = PCPU_GET(cpuid); \ mtx_lock(&ctx_mtx[(i)]); \ (ctx) = ctx_vfp[(i)]; \ } while (0) #define RELEASE_CTX(i, ctx) \ do { \ mtx_unlock(&ctx_mtx[(i)]); \ (i) = -1; \ (ctx) = NULL; \ } while (0) static int armv8_crypto_cipher_process(struct armv8_crypto_session *, struct cryptop *); MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); static void armv8_crypto_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "armv8crypto", -1) == NULL && BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) panic("ARMv8 crypto: could not attach"); } static int armv8_crypto_probe(device_t dev) { uint64_t reg; int ret = ENXIO; reg = READ_SPECIALREG(id_aa64isar0_el1); switch (ID_AA64ISAR0_AES_VAL(reg)) { case ID_AA64ISAR0_AES_BASE: ret = 0; device_set_desc(dev, "AES-CBC,AES-XTS"); break; case ID_AA64ISAR0_AES_PMULL: ret = 0; device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM"); break; default: break; case ID_AA64ISAR0_AES_NONE: device_printf(dev, "CPU lacks AES instructions\n"); break; } /* TODO: Check more fields as we support more features */ return (ret); } static int armv8_crypto_attach(device_t dev) { struct armv8_crypto_softc *sc; uint64_t reg; int i; sc = device_get_softc(dev); sc->dieing = 0; reg = READ_SPECIALREG(id_aa64isar0_el1); if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL) sc->has_pmul = true; sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); if (sc->cid < 0) { device_printf(dev, "Could not get crypto driver id.\n"); return (ENOMEM); } rw_init(&sc->lock, "armv8crypto"); ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, M_WAITOK|M_ZERO); ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, M_WAITOK|M_ZERO); CPU_FOREACH(i) { ctx_vfp[i] = fpu_kern_alloc_ctx(0); mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); } return (0); } static int armv8_crypto_detach(device_t dev) { struct armv8_crypto_softc *sc; int i; sc = device_get_softc(dev); rw_wlock(&sc->lock); sc->dieing = 1; rw_wunlock(&sc->lock); crypto_unregister_all(sc->cid); rw_destroy(&sc->lock); CPU_FOREACH(i) { if (ctx_vfp[i] != NULL) { mtx_destroy(&ctx_mtx[i]); fpu_kern_free_ctx(ctx_vfp[i]); } ctx_vfp[i] = NULL; } free(ctx_mtx, M_ARMV8_CRYPTO); ctx_mtx = NULL; free(ctx_vfp, M_ARMV8_CRYPTO); ctx_vfp = NULL; return (0); } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD) static int armv8_crypto_probesession(device_t dev, const struct crypto_session_params *csp) { struct armv8_crypto_softc *sc; sc = device_get_softc(dev); if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: if (!sc->has_pmul) return (EINVAL); - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); if (csp->csp_auth_mlen != 0 && csp->csp_auth_mlen != GMAC_DIGEST_LEN) return (EINVAL); switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: return (EINVAL); } break; default: return (EINVAL); } break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (EINVAL); switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: return (EINVAL); } break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (EINVAL); switch (csp->csp_cipher_klen * 8) { case 256: case 512: break; default: return (EINVAL); } break; default: return (EINVAL); } break; default: return (EINVAL); } return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } static int armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, const struct crypto_session_params *csp, const uint8_t *key, int keylen) { __uint128_val_t H; struct fpu_kern_ctx *ctx; int kt, i; if (csp->csp_cipher_alg == CRYPTO_AES_XTS) keylen /= 2; switch (keylen * 8) { case 128: case 192: case 256: break; default: return (EINVAL); } kt = is_fpu_kern_thread(0); if (!kt) { AQUIRE_CTX(i, ctx); fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); } aes_v8_set_encrypt_key(key, keylen * 8, &ses->enc_schedule); if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) || (csp->csp_cipher_alg == CRYPTO_AES_CBC)) aes_v8_set_decrypt_key(key, keylen * 8, &ses->dec_schedule); if (csp->csp_cipher_alg == CRYPTO_AES_XTS) aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule); if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { memset(H.c, 0, sizeof(H.c)); aes_v8_encrypt(H.c, H.c, &ses->enc_schedule); H.u[0] = bswap64(H.u[0]); H.u[1] = bswap64(H.u[1]); gcm_init_v8(ses->Htable, H.u); } if (!kt) { fpu_kern_leave(curthread, ctx); RELEASE_CTX(i, ctx); } return (0); } static int armv8_crypto_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct armv8_crypto_softc *sc; struct armv8_crypto_session *ses; int error; sc = device_get_softc(dev); rw_wlock(&sc->lock); if (sc->dieing) { rw_wunlock(&sc->lock); return (EINVAL); } ses = crypto_get_driver_session(cses); error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, csp->csp_cipher_klen); rw_wunlock(&sc->lock); return (error); } static int armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) { struct armv8_crypto_session *ses; ses = crypto_get_driver_session(crp->crp_session); crp->crp_etype = armv8_crypto_cipher_process(ses, crp); crypto_done(crp); return (0); } static uint8_t * armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated) { uint8_t *addr; addr = crypto_contiguous_subsegment(crp, start, length); if (addr != NULL) { *allocated = 0; return (addr); } addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); if (addr != NULL) { *allocated = 1; crypto_copydata(crp, start, length, addr); } else *allocated = 0; return (addr); } static int armv8_crypto_cipher_process(struct armv8_crypto_session *ses, struct cryptop *crp) { struct crypto_buffer_cursor fromc, toc; const struct crypto_session_params *csp; struct fpu_kern_ctx *ctx; uint8_t *authbuf; uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; int authallocated, i; int encflag; int kt; int error; csp = crypto_get_params(crp->crp_session); encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); authallocated = 0; authbuf = NULL; kt = 1; if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { if (crp->crp_aad != NULL) authbuf = crp->crp_aad; else authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start, crp->crp_aad_length, &authallocated); if (authbuf == NULL) { error = ENOMEM; goto out; } } crypto_cursor_init(&fromc, &crp->crp_buf); crypto_cursor_advance(&fromc, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&toc, &crp->crp_obuf); crypto_cursor_advance(&toc, crp->crp_payload_output_start); } else { crypto_cursor_copy(&fromc, &toc); } kt = is_fpu_kern_thread(0); if (!kt) { AQUIRE_CTX(i, ctx); fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); } if (crp->crp_cipher_key != NULL) { armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, csp->csp_cipher_klen); } crypto_read_iv(crp, iv); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { error = EINVAL; goto out; } if (encflag) armv8_aes_encrypt_cbc(&ses->enc_schedule, crp->crp_payload_length, &fromc, &toc, iv); else armv8_aes_decrypt_cbc(&ses->dec_schedule, crp->crp_payload_length, &fromc, &toc, iv); break; case CRYPTO_AES_XTS: if (encflag) armv8_aes_encrypt_xts(&ses->enc_schedule, &ses->xts_schedule.aes_key, crp->crp_payload_length, &fromc, &toc, iv); else armv8_aes_decrypt_xts(&ses->dec_schedule, &ses->xts_schedule.aes_key, crp->crp_payload_length, &fromc, &toc, iv); break; case CRYPTO_AES_NIST_GCM_16: if (encflag) { memset(tag, 0, sizeof(tag)); armv8_aes_encrypt_gcm(&ses->enc_schedule, crp->crp_payload_length, &fromc, &toc, crp->crp_aad_length, authbuf, tag, iv, ses->Htable); crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), tag); } else { crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), tag); error = armv8_aes_decrypt_gcm(&ses->enc_schedule, crp->crp_payload_length, &fromc, &toc, crp->crp_aad_length, authbuf, tag, iv, ses->Htable); if (error != 0) goto out; } break; } error = 0; out: if (!kt) { fpu_kern_leave(curthread, ctx); RELEASE_CTX(i, ctx); } if (authallocated) zfree(authbuf, M_ARMV8_CRYPTO); explicit_bzero(iv, sizeof(iv)); explicit_bzero(tag, sizeof(tag)); return (error); } static device_method_t armv8_crypto_methods[] = { DEVMETHOD(device_identify, armv8_crypto_identify), DEVMETHOD(device_probe, armv8_crypto_probe), DEVMETHOD(device_attach, armv8_crypto_attach), DEVMETHOD(device_detach, armv8_crypto_detach), DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession), DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), DEVMETHOD(cryptodev_process, armv8_crypto_process), DEVMETHOD_END, }; static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, sizeof(struct armv8_crypto_softc)); static devclass_t armv8_crypto_devclass; DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass, 0, 0); diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c index 7cc38b14f3fd..2b059dcee2ca 100644 --- a/sys/crypto/ccp/ccp.c +++ b/sys/crypto/ccp/ccp.c @@ -1,797 +1,792 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 Chelsio Communications, Inc. * Copyright (c) 2017 Conrad Meyer * All rights reserved. * Largely borrowed from ccr(4), Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #include #include #include #include #include "cryptodev_if.h" #include "ccp.h" #include "ccp_hardware.h" MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto"); /* * Need a global softc available for garbage random_source API, which lacks any * context pointer. It's also handy for debugging. */ struct ccp_softc *g_ccp_softc; bool g_debug_print = false; SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0, "Set to enable debugging log messages"); static struct pciid { uint32_t devid; const char *desc; } ccp_ids[] = { { 0x14561022, "AMD CCP-5a" }, { 0x14681022, "AMD CCP-5b" }, { 0x15df1022, "AMD CCP-5a" }, }; static struct random_source random_ccp = { .rs_ident = "AMD CCP TRNG", .rs_source = RANDOM_PURE_CCP, .rs_read = random_ccp_read, }; /* * ccp_populate_sglist() generates a scatter/gather list that covers the entire * crypto operation buffer. */ static int ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) { int error; sglist_reset(sg); switch (cb->cb_type) { case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_SINGLE_MBUF: error = sglist_append_single_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; case CRYPTO_BUF_VMPAGE: error = sglist_append_vmpages(sg, cb->cb_vm_page, cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; } return (error); } static int ccp_probe(device_t dev) { struct pciid *ip; uint32_t id; id = pci_get_devid(dev); for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) { if (id == ip->devid) { device_set_desc(dev, ip->desc); return (0); } } return (ENXIO); } static void ccp_initialize_queues(struct ccp_softc *sc) { struct ccp_queue *qp; size_t i; for (i = 0; i < nitems(sc->queues); i++) { qp = &sc->queues[i]; qp->cq_softc = sc; qp->cq_qindex = i; mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF); /* XXX - arbitrarily chosen sizes */ qp->cq_sg_crp = sglist_alloc(32, M_WAITOK); /* Two more SGEs than sg_crp to accommodate ipad. */ qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK); qp->cq_sg_dst = sglist_alloc(2, M_WAITOK); } } static void ccp_free_queues(struct ccp_softc *sc) { struct ccp_queue *qp; size_t i; for (i = 0; i < nitems(sc->queues); i++) { qp = &sc->queues[i]; mtx_destroy(&qp->cq_lock); sglist_free(qp->cq_sg_crp); sglist_free(qp->cq_sg_ulptx); sglist_free(qp->cq_sg_dst); } } static int ccp_attach(device_t dev) { struct ccp_softc *sc; int error; sc = device_get_softc(dev); sc->dev = dev; sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session), CRYPTOCAP_F_HARDWARE); if (sc->cid < 0) { device_printf(dev, "could not get crypto driver id\n"); return (ENXIO); } error = ccp_hw_attach(dev); if (error != 0) return (error); mtx_init(&sc->lock, "ccp", NULL, MTX_DEF); ccp_initialize_queues(sc); if (g_ccp_softc == NULL) { g_ccp_softc = sc; if ((sc->hw_features & VERSION_CAP_TRNG) != 0) random_source_register(&random_ccp); } return (0); } static int ccp_detach(device_t dev) { struct ccp_softc *sc; sc = device_get_softc(dev); mtx_lock(&sc->lock); sc->detaching = true; mtx_unlock(&sc->lock); crypto_unregister_all(sc->cid); if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0) random_source_deregister(&random_ccp); ccp_hw_detach(dev); ccp_free_queues(sc); if (g_ccp_softc == sc) g_ccp_softc = NULL; mtx_destroy(&sc->lock); return (0); } static void ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen) { union authctx auth_ctx; struct auth_hash *axf; u_int i; /* * If the key is larger than the block size, use the digest of * the key as the key instead. */ axf = s->hmac.auth_hash; if (klen > axf->blocksize) { axf->Init(&auth_ctx); axf->Update(&auth_ctx, key, klen); axf->Final(s->hmac.ipad, &auth_ctx); explicit_bzero(&auth_ctx, sizeof(auth_ctx)); klen = axf->hashsize; } else memcpy(s->hmac.ipad, key, klen); memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); for (i = 0; i < axf->blocksize; i++) { s->hmac.ipad[i] ^= HMAC_IPAD_VAL; s->hmac.opad[i] ^= HMAC_OPAD_VAL; } } static bool ccp_aes_check_keylen(int alg, int klen) { switch (klen * 8) { case 128: case 192: if (alg == CRYPTO_AES_XTS) return (false); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) return (false); break; default: return (false); } return (true); } static void ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen) { unsigned kbits; if (alg == CRYPTO_AES_XTS) kbits = (klen / 2) * 8; else kbits = klen * 8; switch (kbits) { case 128: s->blkcipher.cipher_type = CCP_AES_TYPE_128; break; case 192: s->blkcipher.cipher_type = CCP_AES_TYPE_192; break; case 256: s->blkcipher.cipher_type = CCP_AES_TYPE_256; break; default: panic("should not get here"); } s->blkcipher.key_len = klen; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); } static bool ccp_auth_supported(struct ccp_softc *sc, const struct crypto_session_params *csp) { if ((sc->hw_features & VERSION_CAP_SHA) == 0) return (false); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: if (csp->csp_auth_key == NULL) return (false); break; default: return (false); } return (true); } static bool ccp_cipher_supported(struct ccp_softc *sc, const struct crypto_session_params *csp) { if ((sc->hw_features & VERSION_CAP_AES) == 0) return (false); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (ccp_aes_check_keylen(csp->csp_cipher_alg, csp->csp_cipher_klen)); } static int ccp_probesession(device_t dev, const struct crypto_session_params *csp) { struct ccp_softc *sc; if (csp->csp_flags != 0) return (EINVAL); sc = device_get_softc(dev); switch (csp->csp_mode) { case CSP_MODE_DIGEST: if (!ccp_auth_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_CIPHER: if (!ccp_cipher_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); - if (csp->csp_auth_mlen < 0 || - csp->csp_auth_mlen > AES_GMAC_HASH_LEN) - return (EINVAL); if ((sc->hw_features & VERSION_CAP_AES) == 0) return (EINVAL); break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!ccp_auth_supported(sc, csp) || !ccp_cipher_supported(sc, csp)) return (EINVAL); break; default: return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } static int ccp_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ccp_softc *sc; struct ccp_session *s; struct auth_hash *auth_hash; enum ccp_aes_mode cipher_mode; unsigned auth_mode; unsigned q; /* XXX reconcile auth_mode with use by ccp_sha */ switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: auth_hash = &auth_hash_hmac_sha1; auth_mode = SHA1; break; case CRYPTO_SHA2_256_HMAC: auth_hash = &auth_hash_hmac_sha2_256; auth_mode = SHA2_256; break; case CRYPTO_SHA2_384_HMAC: auth_hash = &auth_hash_hmac_sha2_384; auth_mode = SHA2_384; break; case CRYPTO_SHA2_512_HMAC: auth_hash = &auth_hash_hmac_sha2_512; auth_mode = SHA2_512; break; default: auth_hash = NULL; auth_mode = 0; break; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: cipher_mode = CCP_AES_MODE_CBC; break; case CRYPTO_AES_ICM: cipher_mode = CCP_AES_MODE_CTR; break; case CRYPTO_AES_NIST_GCM_16: cipher_mode = CCP_AES_MODE_GCTR; break; case CRYPTO_AES_XTS: cipher_mode = CCP_AES_MODE_XTS; break; default: cipher_mode = CCP_AES_MODE_ECB; break; } sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); return (ENXIO); } s = crypto_get_driver_session(cses); /* Just grab the first usable queue for now. */ for (q = 0; q < nitems(sc->queues); q++) if ((sc->valid_queues & (1 << q)) != 0) break; if (q == nitems(sc->queues)) { mtx_unlock(&sc->lock); return (ENXIO); } s->queue = q; switch (csp->csp_mode) { case CSP_MODE_AEAD: s->mode = GCM; break; case CSP_MODE_ETA: s->mode = AUTHENC; break; case CSP_MODE_DIGEST: s->mode = HMAC; break; case CSP_MODE_CIPHER: s->mode = BLKCIPHER; break; } if (s->mode == GCM) { if (csp->csp_auth_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = csp->csp_auth_mlen; } else if (auth_hash != NULL) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; if (csp->csp_auth_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else s->hmac.hash_len = csp->csp_auth_mlen; ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen); } if (cipher_mode != CCP_AES_MODE_ECB) { s->blkcipher.cipher_mode = cipher_mode; if (csp->csp_cipher_key != NULL) ccp_aes_setkey(s, csp->csp_cipher_alg, csp->csp_cipher_key, csp->csp_cipher_klen); } s->active = true; mtx_unlock(&sc->lock); return (0); } static void ccp_freesession(device_t dev, crypto_session_t cses) { struct ccp_session *s; s = crypto_get_driver_session(cses); if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); s->active = false; } static int ccp_process(device_t dev, struct cryptop *crp, int hint) { const struct crypto_session_params *csp; struct ccp_softc *sc; struct ccp_queue *qp; struct ccp_session *s; int error; bool qpheld; qpheld = false; qp = NULL; csp = crypto_get_params(crp->crp_session); s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); mtx_lock(&sc->lock); qp = &sc->queues[s->queue]; mtx_unlock(&sc->lock); error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT); if (error != 0) goto out; qpheld = true; error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf); if (error != 0) goto out; if (crp->crp_auth_key != NULL) { KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC")); ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen); } if (crp->crp_cipher_key != NULL) ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key, csp->csp_cipher_klen); switch (s->mode) { case HMAC: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_hmac(qp, s, crp); break; case BLKCIPHER: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_blkcipher(qp, s, crp); break; case AUTHENC: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_authenc(qp, s, crp); break; case GCM: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_gcm(qp, s, crp); break; } if (error == 0) s->pending++; out: if (qpheld) { if (error != 0) { /* * Squash EAGAIN so callers don't uselessly and * expensively retry if the ring was full. */ if (error == EAGAIN) error = ENOMEM; ccp_queue_abort(qp); } else ccp_queue_release(qp); } if (error != 0) { DPRINTF(dev, "%s: early error:%d\n", __func__, error); crp->crp_etype = error; crypto_done(crp); } return (0); } static device_method_t ccp_methods[] = { DEVMETHOD(device_probe, ccp_probe), DEVMETHOD(device_attach, ccp_attach), DEVMETHOD(device_detach, ccp_detach), DEVMETHOD(cryptodev_probesession, ccp_probesession), DEVMETHOD(cryptodev_newsession, ccp_newsession), DEVMETHOD(cryptodev_freesession, ccp_freesession), DEVMETHOD(cryptodev_process, ccp_process), DEVMETHOD_END }; static driver_t ccp_driver = { "ccp", ccp_methods, sizeof(struct ccp_softc) }; static devclass_t ccp_devclass; DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL); MODULE_VERSION(ccp, 1); MODULE_DEPEND(ccp, crypto, 1, 1, 1); MODULE_DEPEND(ccp, random_device, 1, 1, 1); #if 0 /* There are enough known issues that we shouldn't load automatically */ MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, nitems(ccp_ids)); #endif static int ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags) { struct ccp_softc *sc; mtx_assert(&qp->cq_lock, MA_OWNED); sc = qp->cq_softc; if (n < 1 || n >= (1 << sc->ring_size_order)) return (EINVAL); while (true) { if (ccp_queue_get_ring_space(qp) >= n) return (0); if ((mflags & M_WAITOK) == 0) return (EAGAIN); qp->cq_waiting = true; msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0); } } int ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags) { int error; mtx_lock(&qp->cq_lock); qp->cq_acq_tail = qp->cq_tail; error = ccp_queue_reserve_space(qp, n, mflags); if (error != 0) mtx_unlock(&qp->cq_lock); return (error); } void ccp_queue_release(struct ccp_queue *qp) { mtx_assert(&qp->cq_lock, MA_OWNED); if (qp->cq_tail != qp->cq_acq_tail) { wmb(); ccp_queue_write_tail(qp); } mtx_unlock(&qp->cq_lock); } void ccp_queue_abort(struct ccp_queue *qp) { unsigned i; mtx_assert(&qp->cq_lock, MA_OWNED); /* Wipe out any descriptors associated with this aborted txn. */ for (i = qp->cq_acq_tail; i != qp->cq_tail; i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) { memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i])); } qp->cq_tail = qp->cq_acq_tail; mtx_unlock(&qp->cq_lock); } #ifdef DDB #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) static void db_show_ccp_sc(struct ccp_softc *sc) { db_printf("ccp softc at %p\n", sc); db_printf(" cid: %d\n", (int)sc->cid); db_printf(" lock: "); db_show_lock(&sc->lock); db_printf(" detaching: %d\n", (int)sc->detaching); db_printf(" ring_size_order: %u\n", sc->ring_size_order); db_printf(" hw_version: %d\n", (int)sc->hw_version); db_printf(" hw_features: %b\n", (int)sc->hw_features, "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA" "\11SHA\0103DES\07AES"); db_printf(" hw status:\n"); db_ccp_show_hw(sc); } static void db_show_ccp_qp(struct ccp_queue *qp) { db_printf(" lock: "); db_show_lock(&qp->cq_lock); db_printf(" cq_qindex: %u\n", qp->cq_qindex); db_printf(" cq_softc: %p\n", qp->cq_softc); db_printf(" head: %u\n", qp->cq_head); db_printf(" tail: %u\n", qp->cq_tail); db_printf(" acq_tail: %u\n", qp->cq_acq_tail); db_printf(" desc_ring: %p\n", qp->desc_ring); db_printf(" completions_ring: %p\n", qp->completions_ring); db_printf(" descriptors (phys): 0x%jx\n", (uintmax_t)qp->desc_ring_bus_addr); db_printf(" hw status:\n"); db_ccp_show_queue_hw(qp); } DB_SHOW_COMMAND(ccp, db_show_ccp) { struct ccp_softc *sc; unsigned unit, qindex; if (!have_addr) goto usage; unit = (unsigned)addr; sc = devclass_get_softc(ccp_devclass, unit); if (sc == NULL) { db_printf("No such device ccp%u\n", unit); goto usage; } if (count == -1) { db_show_ccp_sc(sc); return; } qindex = (unsigned)count; if (qindex >= nitems(sc->queues)) { db_printf("No such queue %u\n", qindex); goto usage; } db_show_ccp_qp(&sc->queues[qindex]); return; usage: db_printf("usage: show ccp [,]\n"); return; } #endif /* DDB */ diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c index 7c233897dff6..325aba60a742 100644 --- a/sys/dev/cxgbe/crypto/t4_crypto.c +++ b/sys/dev/cxgbe/crypto/t4_crypto.c @@ -1,3054 +1,3048 @@ /*- * Copyright (c) 2017 Chelsio Communications, Inc. * Copyright (c) 2021 The FreeBSD Foundation * All rights reserved. * Written by: John Baldwin * * Portions of this software were developed by Ararat River * Consulting, LLC under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "common/common.h" #include "crypto/t4_crypto.h" /* * Requests consist of: * * +-------------------------------+ * | struct fw_crypto_lookaside_wr | * +-------------------------------+ * | struct ulp_txpkt | * +-------------------------------+ * | struct ulptx_idata | * +-------------------------------+ * | struct cpl_tx_sec_pdu | * +-------------------------------+ * | struct cpl_tls_tx_scmd_fmt | * +-------------------------------+ * | key context header | * +-------------------------------+ * | AES key | ----- For requests with AES * +-------------------------------+ * | Hash state | ----- For hash-only requests * +-------------------------------+ - * | IPAD (16-byte aligned) | \ * +-------------------------------+ +---- For requests with HMAC * | OPAD (16-byte aligned) | / * +-------------------------------+ - * | GMAC H | ----- For AES-GCM * +-------------------------------+ - * | struct cpl_rx_phys_dsgl | \ * +-------------------------------+ +---- Destination buffer for * | PHYS_DSGL entries | / non-hash-only requests * +-------------------------------+ - * | 16 dummy bytes | ----- Only for HMAC/hash-only requests * +-------------------------------+ * | IV | ----- If immediate IV * +-------------------------------+ * | Payload | ----- If immediate Payload * +-------------------------------+ - * | struct ulptx_sgl | \ * +-------------------------------+ +---- If payload via SGL * | SGL entries | / * +-------------------------------+ - * * Note that the key context must be padded to ensure 16-byte alignment. * For HMAC requests, the key consists of the partial hash of the IPAD * followed by the partial hash of the OPAD. * * Replies consist of: * * +-------------------------------+ * | struct cpl_fw6_pld | * +-------------------------------+ * | hash digest | ----- For HMAC request with * +-------------------------------+ 'hash_size' set in work request * * A 32-bit big-endian error status word is supplied in the last 4 * bytes of data[0] in the CPL_FW6_PLD message. bit 0 indicates a * "MAC" error and bit 1 indicates a "PAD" error. * * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message * in the request is returned in data[1] of the CPL_FW6_PLD message. * * For block cipher replies, the updated IV is supplied in data[2] and * data[3] of the CPL_FW6_PLD message. * * For hash replies where the work request set 'hash_size' to request * a copy of the hash in the reply, the hash digest is supplied * immediately following the CPL_FW6_PLD message. */ /* * The crypto engine supports a maximum AAD size of 511 bytes. */ #define MAX_AAD_LEN 511 /* * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG * entries. While the CPL includes a 16-bit length field, the T6 can * sometimes hang if an error occurs while processing a request with a * single DSGL entry larger than 2k. */ #define MAX_RX_PHYS_DSGL_SGE 32 #define DSGL_SGE_MAXLEN 2048 /* * The adapter only supports requests with a total input or output * length of 64k-1 or smaller. Longer requests either result in hung * requests or incorrect results. */ #define MAX_REQUEST_SIZE 65535 static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto"); struct ccr_session_hmac { struct auth_hash *auth_hash; int hash_len; unsigned int partial_digest_len; unsigned int auth_mode; unsigned int mk_size; char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2]; }; struct ccr_session_gmac { int hash_len; char ghash_h[GMAC_BLOCK_LEN]; }; struct ccr_session_ccm_mac { int hash_len; }; struct ccr_session_blkcipher { unsigned int cipher_mode; unsigned int key_len; unsigned int iv_len; __be32 key_ctx_hdr; char enckey[CHCR_AES_MAX_KEY_LEN]; char deckey[CHCR_AES_MAX_KEY_LEN]; }; struct ccr_port { struct sge_wrq *txq; struct sge_rxq *rxq; int rx_channel_id; int tx_channel_id; u_int active_sessions; counter_u64_t stats_queued; counter_u64_t stats_completed; }; struct ccr_session { #ifdef INVARIANTS int pending; #endif enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; struct ccr_port *port; union { struct ccr_session_hmac hmac; struct ccr_session_gmac gmac; struct ccr_session_ccm_mac ccm_mac; }; struct ccr_session_blkcipher blkcipher; struct mtx lock; /* * Pre-allocate S/G lists used when preparing a work request. * 'sg_input' contains an sglist describing the entire input * buffer for a 'struct cryptop'. 'sg_output' contains an * sglist describing the entire output buffer. 'sg_ulptx' is * used to describe the data the engine should DMA as input * via ULPTX_SGL. 'sg_dsgl' is used to describe the * destination that cipher text and a tag should be written * to. */ struct sglist *sg_input; struct sglist *sg_output; struct sglist *sg_ulptx; struct sglist *sg_dsgl; }; struct ccr_softc { struct adapter *adapter; device_t dev; uint32_t cid; struct mtx lock; bool detaching; struct ccr_port ports[MAX_NPORTS]; u_int port_mask; int first_rxq_id; /* * Pre-allocate a dummy output buffer for the IV and AAD for * AEAD requests. */ char *iv_aad_buf; struct sglist *sg_iv_aad; /* Statistics. */ counter_u64_t stats_blkcipher_encrypt; counter_u64_t stats_blkcipher_decrypt; counter_u64_t stats_hash; counter_u64_t stats_hmac; counter_u64_t stats_eta_encrypt; counter_u64_t stats_eta_decrypt; counter_u64_t stats_gcm_encrypt; counter_u64_t stats_gcm_decrypt; counter_u64_t stats_ccm_encrypt; counter_u64_t stats_ccm_decrypt; counter_u64_t stats_wr_nomem; counter_u64_t stats_inflight; counter_u64_t stats_mac_error; counter_u64_t stats_pad_error; counter_u64_t stats_sglist_error; counter_u64_t stats_process_error; counter_u64_t stats_sw_fallback; struct sysctl_ctx_list ctx; }; /* * Crypto requests involve two kind of scatter/gather lists. * * Non-hash-only requests require a PHYS_DSGL that describes the * location to store the results of the encryption or decryption * operation. This SGL uses a different format (PHYS_DSGL) and should * exclude the skip bytes at the start of the data as well as any AAD * or IV. For authenticated encryption requests it should include the * destination of the hash or tag. * * The input payload may either be supplied inline as immediate data, * or via a standard ULP_TX SGL. This SGL should include AAD, * ciphertext, and the hash or tag for authenticated decryption * requests. * * These scatter/gather lists can describe different subsets of the * buffers described by the crypto operation. ccr_populate_sglist() * generates a scatter/gather list that covers an entire crypto * operation buffer that is then used to construct the other * scatter/gather lists. */ static int ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) { int error; sglist_reset(sg); switch (cb->cb_type) { case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_SINGLE_MBUF: error = sglist_append_single_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; case CRYPTO_BUF_VMPAGE: error = sglist_append_vmpages(sg, cb->cb_vm_page, cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; } return (error); } /* * Segments in 'sg' larger than 'maxsegsize' are counted as multiple * segments. */ static int ccr_count_sgl(struct sglist *sg, int maxsegsize) { int i, nsegs; nsegs = 0; for (i = 0; i < sg->sg_nseg; i++) nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize); return (nsegs); } /* These functions deal with PHYS_DSGL for the reply buffer. */ static inline int ccr_phys_dsgl_len(int nsegs) { int len; len = (nsegs / 8) * sizeof(struct phys_sge_pairs); if ((nsegs % 8) != 0) { len += sizeof(uint16_t) * 8; len += roundup2(nsegs % 8, 2) * sizeof(uint64_t); } return (len); } static void ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs) { struct sglist *sg; struct cpl_rx_phys_dsgl *cpl; struct phys_sge_pairs *sgl; vm_paddr_t paddr; size_t seglen; u_int i, j; sg = s->sg_dsgl; cpl = dst; cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | V_CPL_RX_PHYS_DSGL_ISRDMA(0)); cpl->pcirlxorder_to_noofsgentr = htobe32( V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) | V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) | V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) | V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs)); cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; cpl->rss_hdr_int.qid = htobe16(s->port->rxq->iq.abs_id); cpl->rss_hdr_int.hash_val = 0; cpl->rss_hdr_int.channel = s->port->rx_channel_id; sgl = (struct phys_sge_pairs *)(cpl + 1); j = 0; for (i = 0; i < sg->sg_nseg; i++) { seglen = sg->sg_segs[i].ss_len; paddr = sg->sg_segs[i].ss_paddr; do { sgl->addr[j] = htobe64(paddr); if (seglen > DSGL_SGE_MAXLEN) { sgl->len[j] = htobe16(DSGL_SGE_MAXLEN); paddr += DSGL_SGE_MAXLEN; seglen -= DSGL_SGE_MAXLEN; } else { sgl->len[j] = htobe16(seglen); seglen = 0; } j++; if (j == 8) { sgl++; j = 0; } } while (seglen != 0); } MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs); } /* These functions deal with the ULPTX_SGL for input payload. */ static inline int ccr_ulptx_sgl_len(int nsegs) { u_int n; nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); return (roundup2(n, 16)); } static void ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs) { struct ulptx_sgl *usgl; struct sglist *sg; struct sglist_seg *ss; int i; sg = s->sg_ulptx; MPASS(nsegs == sg->sg_nseg); ss = &sg->sg_segs[0]; usgl = dst; usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); usgl->len0 = htobe32(ss->ss_len); usgl->addr0 = htobe64(ss->ss_paddr); ss++; for (i = 0; i < sg->sg_nseg - 1; i++) { usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr); ss++; } } static bool ccr_use_imm_data(u_int transhdr_len, u_int input_len) { if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN) return (false); if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) > SGE_MAX_WR_LEN) return (false); return (true); } static void ccr_populate_wreq(struct ccr_softc *sc, struct ccr_session *s, struct chcr_wr *crwr, u_int kctx_len, u_int wr_len, u_int imm_len, u_int sgl_len, u_int hash_size, struct cryptop *crp) { u_int cctx_size, idata_len; cctx_size = sizeof(struct _key_ctx) + kctx_len; crwr->wreq.op_to_cctx_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) | V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) | V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4)); crwr->wreq.len16_pkd = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16)); crwr->wreq.session_id = 0; crwr->wreq.rx_chid_to_rx_q_id = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(s->port->rx_channel_id) | V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) | V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) | V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) | V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) | V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) | /* unused in firmware */ V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(s->port->rxq->iq.abs_id)); crwr->wreq.key_addr = 0; crwr->wreq.pld_size_hash_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) | V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size)); crwr->wreq.cookie = htobe64((uintptr_t)crp); crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DATAMODIFY(0) | V_ULP_TXPKT_CHANNELID(s->port->tx_channel_id) | V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(sc->first_rxq_id) | V_ULP_TXPKT_RO(1)); crwr->ulptx.len = htobe32( ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16)); crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | V_ULP_TX_SC_MORE(sgl_len != 0 ? 1 : 0)); idata_len = wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len; if (imm_len % 16 != 0) idata_len -= 16 - imm_len % 16; crwr->sc_imm.len = htobe32(idata_len); } static int ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { struct chcr_wr *crwr; struct wrqe *wr; struct auth_hash *axf; char *dst; u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len; u_int hmac_ctrl, imm_len, iopad_size; int error, sgl_nsegs, sgl_len, use_opad; /* Reject requests with too large of an input buffer. */ if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); axf = s->hmac.auth_hash; if (s->mode == HMAC) { use_opad = 1; hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; } else { use_opad = 0; hmac_ctrl = SCMD_HMAC_CTRL_NOP; } /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the context includes the aligned IPAD and * OPAD. */ kctx_len = iopad_size; if (use_opad) kctx_len += iopad_size; hash_size_in_response = axf->hashsize; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); if (crp->crp_payload_length == 0) { imm_len = axf->blocksize; sgl_nsegs = 0; sgl_len = 0; } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) { imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, hash_size_in_response, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(0)); crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ? axf->blocksize : crp->crp_payload_length); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_LAST_FRAG(0) | V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1)); memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len); /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */ kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_OPAD_PRESENT(use_opad) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) | V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1)); dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES; if (crp->crp_payload_length == 0) { dst[0] = 0x80; if (s->mode == HMAC) *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = htobe64(axf->blocksize << 3); } else if (imm_len != 0) crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { uint8_t hash[HASH_MAX_LEN]; if (error) return (error); if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len, hash); if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0) return (EBADMSG); } else crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len, (cpl + 1)); return (0); } static int ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int imm_len, iv_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); /* Reject requests with too large of an input buffer. */ if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; sglist_reset(s->sg_dsgl); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* The 'key' must be 128-bit aligned. */ kctx_len = roundup2(s->blkcipher.key_len, 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* For AES-XTS we send a 16-byte IV in the work request. */ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) { imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); crypto_read_iv(crp, iv); /* Zero the remainder of the IV for AES-XTS. */ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. */ return (error); } /* * 'hashsize' is the length of a full digest. 'authsize' is the * requested digest length for this operation which may be less * than 'hashsize'. */ static int ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize) { if (authsize == 10) return (SCMD_HMAC_CTRL_TRUNC_RFC4366); if (authsize == 12) return (SCMD_HMAC_CTRL_IPSEC_96BIT); if (authsize == hashsize / 2) return (SCMD_HMAC_CTRL_DIV2); return (SCMD_HMAC_CTRL_NO_TRUNC); } static int ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; struct auth_hash *axf; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len, iopad_size, iv_len; u_int aad_start, aad_stop; u_int auth_insert; u_int cipher_start, cipher_stop; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; /* * If there is a need in the future, requests with an empty * payload could be supported as HMAC-only requests. */ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); /* For AES-XTS we send a 16-byte IV in the work request. */ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; if (crp->crp_aad_length + iv_len > MAX_AAD_LEN) return (EINVAL); axf = s->hmac.auth_hash; hash_size_in_response = s->hmac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The output buffer consists of the cipher text followed by * the hash when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + crp->crp_aad_length + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the key context consists of the key followed * by the IPAD and OPAD. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = crp->crp_aad_length + crp->crp_payload_length; /* * The firmware hangs if sent a request which is a * bit smaller than MAX_REQUEST_SIZE. In particular, the * firmware appears to require 512 - 16 bytes of spare room * along with the size of the hash even if the hash isn't * included in the input buffer. */ if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) > MAX_REQUEST_SIZE) return (EFBIG); if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } /* Any AAD comes after the IV. */ if (crp->crp_aad_length != 0) { aad_start = iv_len + 1; aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); crypto_read_iv(crp, iv); /* Zero the remainder of the IV for AES-XTS. */ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->hmac.pads, iopad_size * 2); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. */ return (error); } static int ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len; u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0) return (EINVAL); /* * The crypto engine doesn't handle GCM requests with an empty * payload, so handle those in software instead. */ if (crp->crp_payload_length == 0) return (EMSGSIZE); if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->gmac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; iv_len = AES_BLOCK_LEN; /* * GCM requests should always provide an explicit IV. */ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + crp->crp_aad_length + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* * The 'key' part of the key context consists of the key followed * by the Galois hash key. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = crp->crp_aad_length + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) return (EFBIG); if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } if (crp->crp_aad_length != 0) { aad_start = iv_len + 1; aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); crypto_read_iv(crp, iv); *(uint32_t *)&iv[12] = htobe32(1); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); /* * NB: cipherstop is explicitly set to 0. On encrypt it * should normally be set to 0 anyway. However, for decrypt * the cipher ends before the tag in the ETA case (and * authstop is set to stop before the tag), but for GCM the * cipher still runs to the end of the buffer. Not sure if * this is intentional or a firmware quirk, but it is required * for working tag validation with GCM decryption. */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * Note that the hardware should always verify the GMAC hash. */ return (error); } /* * Handle a GCM request that is not supported by the crypto engine by * performing the operation in software. Derived from swcr_authenc(). */ static void ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp) { struct auth_hash *axf; struct enc_xform *exf; void *auth_ctx, *kschedule; char block[GMAC_BLOCK_LEN]; char digest[GMAC_DIGEST_LEN]; int error, i, len; auth_ctx = NULL; kschedule = NULL; /* Initialize the MAC. */ switch (s->blkcipher.key_len) { case 16: axf = &auth_hash_nist_gmac_aes_128; break; case 24: axf = &auth_hash_nist_gmac_aes_192; break; case 32: axf = &auth_hash_nist_gmac_aes_256; break; default: error = EINVAL; goto out; } auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); if (auth_ctx == NULL) { error = ENOMEM; goto out; } axf->Init(auth_ctx); axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); /* Initialize the cipher. */ exf = &enc_xform_aes_nist_gcm; kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); if (kschedule == NULL) { error = ENOMEM; goto out; } error = exf->setkey(kschedule, s->blkcipher.enckey, s->blkcipher.key_len); if (error) goto out; if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { error = EINVAL; goto out; } axf->Reinit(auth_ctx, crp->crp_iv, AES_GCM_IV_LEN); /* MAC the AAD. */ if (crp->crp_aad != NULL) { len = rounddown(crp->crp_aad_length, sizeof(block)); if (len != 0) axf->Update(auth_ctx, crp->crp_aad, len); if (crp->crp_aad_length != len) { memset(block, 0, sizeof(block)); memcpy(block, (char *)crp->crp_aad + len, crp->crp_aad_length - len); axf->Update(auth_ctx, block, sizeof(block)); } } else { for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { len = imin(crp->crp_aad_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_aad_start + i, len, block); bzero(block + len, sizeof(block) - len); axf->Update(auth_ctx, block, sizeof(block)); } } exf->reinit(kschedule, crp->crp_iv, AES_GCM_IV_LEN); /* Do encryption with MAC */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt(kschedule, block, block); axf->Update(auth_ctx, block, len); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } else { axf->Update(auth_ctx, block, len); } } /* Length block. */ bzero(block, sizeof(block)); ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8); ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8); axf->Update(auth_ctx, block, sizeof(block)); /* Finalize MAC. */ axf->Final(digest, auth_ctx); /* Inject or validate tag. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), digest); error = 0; } else { char digest2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } } else error = EBADMSG; explicit_bzero(digest2, sizeof(digest2)); } out: zfree(kschedule, M_CCR); zfree(auth_ctx, M_CCR); explicit_bzero(block, sizeof(block)); explicit_bzero(digest, sizeof(digest)); crp->crp_etype = error; crypto_done(crp); } static int ccr_ccm_hmac_ctrl(unsigned int authsize) { switch (authsize) { case 4: return (SCMD_HMAC_CTRL_PL1); case 6: return (SCMD_HMAC_CTRL_PL2); case 8: return (SCMD_HMAC_CTRL_DIV2); case 10: return (SCMD_HMAC_CTRL_TRUNC_RFC4366); case 12: return (SCMD_HMAC_CTRL_IPSEC_96BIT); case 14: return (SCMD_HMAC_CTRL_PL3); case 16: return (SCMD_HMAC_CTRL_NO_TRUNC); default: __assert_unreachable(); } } static void generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response, const char *iv, char *b0) { u_int i, payload_len, L; /* NB: L is already set in the first byte of the IV. */ memcpy(b0, iv, CCM_B0_SIZE); L = iv[0] + 1; /* Set length of hash in bits 3 - 5. */ b0[0] |= (((hash_size_in_response - 2) / 2) << 3); /* Store the payload length as a big-endian value. */ payload_len = crp->crp_payload_length; for (i = 0; i < L; i++) { b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len; payload_len >>= 8; } /* * If there is AAD in the request, set bit 6 in the flags * field and store the AAD length as a big-endian value at the * start of block 1. This only assumes a 16-bit AAD length * since T6 doesn't support large AAD sizes. */ if (crp->crp_aad_length != 0) { b0[0] |= (1 << 6); *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length); } } static int ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; const struct crypto_session_params *csp; struct ulptx_idata *idata; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; u_int aad_len, b0_len, hash_size_in_response, imm_len; u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; csp = crypto_get_params(crp->crp_session); if (s->blkcipher.key_len == 0) return (EINVAL); /* * The crypto engine doesn't handle CCM requests with an empty * payload, so handle those in software instead. */ if (crp->crp_payload_length == 0) return (EMSGSIZE); /* The length has to fit within the length field in block 0. */ if (crp->crp_payload_length > ccm_max_payload_length(csp)) return (EMSGSIZE); /* * CCM always includes block 0 in the AAD before AAD from the * request. */ b0_len = CCM_B0_SIZE; if (crp->crp_aad_length != 0) b0_len += CCM_AAD_FIELD_SIZE; aad_len = b0_len + crp->crp_aad_length; /* * CCM requests should always provide an explicit IV (really * the nonce). */ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* * The IV in the work request is 16 bytes and not just the * nonce. */ iv_len = AES_BLOCK_LEN; if (iv_len + aad_len > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->ccm_mac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + aad_len + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + aad_len + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + aad_len); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* * The 'key' part of the key context consists of two copies of * the AES key. */ kctx_len = roundup2(s->blkcipher.key_len, 16) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, AAD (including block * 0), and then the cipher/plain text. For decryption * requests the hash is appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = aad_len + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) return (EFBIG); if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { /* Block 0 is passed as immediate data. */ imm_len = b0_len; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } aad_start = iv_len + 1; aad_stop = aad_start + aad_len - 1; cipher_start = aad_stop + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); /* * Read the nonce from the request. Use the nonce to generate * the full IV with the counter set to 0. */ memset(iv, 0, iv_len); iv[0] = (15 - csp->csp_ivlen) - 1; crypto_read_iv(crp, iv + 1); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); /* * NB: cipherstop is explicitly set to 0. See comments above * in ccr_gcm(). */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_ccm_hmac_ctrl(hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 0 : 1) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CCM) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_CBCMAC) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16), s->blkcipher.enckey, s->blkcipher.key_len); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; generate_ccm_b0(crp, hash_size_in_response, iv, dst); if (sgl_nsegs == 0) { dst += b0_len; if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else { dst += CCM_B0_SIZE; if (b0_len > CCM_B0_SIZE) { /* * If there is AAD, insert padding including a * ULP_TX_SC_NOOP so that the ULP_TX_SC_DSGL * is 16-byte aligned. */ KASSERT(b0_len - CCM_B0_SIZE == CCM_AAD_FIELD_SIZE, ("b0_len mismatch")); memset(dst + CCM_AAD_FIELD_SIZE, 0, 8 - CCM_AAD_FIELD_SIZE); idata = (void *)(dst + 8); idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); idata->len = htobe32(0); dst = (void *)(idata + 1); } ccr_write_ulptx_sgl(s, dst, sgl_nsegs); } /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * Note that the hardware should always verify the CBC MAC * hash. */ return (error); } /* * Handle a CCM request that is not supported by the crypto engine by * performing the operation in software. Derived from swcr_ccm(). */ static void build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, u_int data_length, u_int tag_length, uint8_t *b0) { uint8_t *bp; uint8_t flags, L; KASSERT(nonce_length >= 7 && nonce_length <= 13, ("nonce_length must be between 7 and 13 bytes")); /* * Need to determine the L field value. This is the number of * bytes needed to specify the length of the message; the length * is whatever is left in the 16 bytes after specifying flags and * the nonce. */ L = 15 - nonce_length; flags = ((aad_length > 0) << 6) + (((tag_length - 2) / 2) << 3) + L - 1; /* * Now we need to set up the first block, which has flags, nonce, * and the message length. */ b0[0] = flags; memcpy(b0 + 1, nonce, nonce_length); bp = b0 + 1 + nonce_length; /* Need to copy L' [aka L-1] bytes of data_length */ for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { *dst = data_length; data_length >>= 8; } } /* NB: OCF only supports AAD lengths < 2^32. */ static int build_ccm_aad_length(u_int aad_length, uint8_t *blk) { if (aad_length < ((1 << 16) - (1 << 8))) { be16enc(blk, aad_length); return (sizeof(uint16_t)); } else { blk[0] = 0xff; blk[1] = 0xfe; be32enc(blk + 2, aad_length); return (2 + sizeof(uint32_t)); } } static void ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp) { const struct crypto_session_params *csp; struct auth_hash *axf; struct enc_xform *exf; union authctx *auth_ctx; void *kschedule; char block[CCM_CBC_BLOCK_LEN]; char tag[AES_CBC_MAC_HASH_LEN]; u_int taglen; int error, i, len; auth_ctx = NULL; kschedule = NULL; taglen = s->ccm_mac.hash_len; csp = crypto_get_params(crp->crp_session); if (crp->crp_payload_length > ccm_max_payload_length(csp)) { error = EMSGSIZE; goto out; } /* Initialize the MAC. */ switch (s->blkcipher.key_len) { case 16: axf = &auth_hash_ccm_cbc_mac_128; break; case 24: axf = &auth_hash_ccm_cbc_mac_192; break; case 32: axf = &auth_hash_ccm_cbc_mac_256; break; default: error = EINVAL; goto out; } auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); if (auth_ctx == NULL) { error = ENOMEM; goto out; } axf->Init(auth_ctx); axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); /* Initialize the cipher. */ exf = &enc_xform_ccm; kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); if (kschedule == NULL) { error = ENOMEM; goto out; } error = exf->setkey(kschedule, s->blkcipher.enckey, s->blkcipher.key_len); if (error) goto out; if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { error = EINVAL; goto out; } axf->Reinit(auth_ctx, crp->crp_iv, csp->csp_ivlen); /* Supply MAC with b0. */ build_ccm_b0(crp->crp_iv, csp->csp_ivlen, crp->crp_aad_length, crp->crp_payload_length, taglen, block); axf->Update(auth_ctx, block, CCM_CBC_BLOCK_LEN); /* MAC the AAD. */ if (crp->crp_aad_length != 0) { len = build_ccm_aad_length(crp->crp_aad_length, block); axf->Update(auth_ctx, block, len); if (crp->crp_aad != NULL) axf->Update(auth_ctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, auth_ctx); /* Pad the AAD (including length field) to a full block. */ len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; if (len != 0) { len = CCM_CBC_BLOCK_LEN - len; memset(block, 0, CCM_CBC_BLOCK_LEN); axf->Update(auth_ctx, block, len); } } exf->reinit(kschedule, crp->crp_iv, csp->csp_ivlen); /* Do encryption/decryption with MAC */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { axf->Update(auth_ctx, block, len); exf->encrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } else { exf->decrypt(kschedule, block, block); axf->Update(auth_ctx, block, len); } } /* Finalize MAC. */ axf->Final(tag, auth_ctx); /* Inject or validate tag. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { crypto_copyback(crp, crp->crp_digest_start, taglen, tag); error = 0; } else { char tag2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, taglen, tag2); if (timingsafe_bcmp(tag, tag2, taglen) == 0) { error = 0; /* Tag matches, decrypt data. */ exf->reinit(kschedule, crp->crp_iv, csp->csp_ivlen); for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } } else error = EBADMSG; explicit_bzero(tag2, sizeof(tag2)); } out: zfree(kschedule, M_CCR); zfree(auth_ctx, M_CCR); explicit_bzero(block, sizeof(block)); explicit_bzero(tag, sizeof(tag)); crp->crp_etype = error; crypto_done(crp); } static void ccr_identify(driver_t *driver, device_t parent) { struct adapter *sc; sc = device_get_softc(parent); if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE && device_find_child(parent, "ccr", -1) == NULL) device_add_child(parent, "ccr", -1); } static int ccr_probe(device_t dev) { device_set_desc(dev, "Chelsio Crypto Accelerator"); return (BUS_PROBE_DEFAULT); } static void ccr_sysctls(struct ccr_softc *sc) { struct sysctl_ctx_list *ctx = &sc->ctx; struct sysctl_oid *oid, *port_oid; struct sysctl_oid_list *children; char buf[16]; int i; /* * dev.ccr.X. */ oid = device_get_sysctl_tree(sc->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "port_mask", CTLFLAG_RW, &sc->port_mask, 0, "Mask of enabled ports"); /* * dev.ccr.X.stats. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, &sc->stats_hash, "Hash requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, &sc->stats_hmac, "HMAC requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, &sc->stats_blkcipher_encrypt, "Cipher encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, &sc->stats_blkcipher_decrypt, "Cipher decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, &sc->stats_eta_encrypt, "Combined AES+HMAC encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, &sc->stats_eta_decrypt, "Combined AES+HMAC decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, &sc->stats_gcm_encrypt, "AES-GCM encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, &sc->stats_gcm_decrypt, "AES-GCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD, &sc->stats_ccm_encrypt, "AES-CCM encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD, &sc->stats_ccm_decrypt, "AES-CCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, &sc->stats_wr_nomem, "Work request memory allocation failures"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, &sc->stats_inflight, "Requests currently pending"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, &sc->stats_mac_error, "MAC errors"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, &sc->stats_pad_error, "Padding errors"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, &sc->stats_sglist_error, "Requests for which DMA mapping failed"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, &sc->stats_process_error, "Requests failed during queueing"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, &sc->stats_sw_fallback, "Requests processed by falling back to software"); /* * dev.ccr.X.stats.port */ port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics"); for (i = 0; i < nitems(sc->ports); i++) { if (sc->ports[i].rxq == NULL) continue; /* * dev.ccr.X.stats.port.Y */ snprintf(buf, sizeof(buf), "%d", i); oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(port_oid), OID_AUTO, buf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, buf); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "active_sessions", CTLFLAG_RD, &sc->ports[i].active_sessions, 0, "Count of active sessions"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "queued", CTLFLAG_RD, &sc->ports[i].stats_queued, "Requests queued"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "completed", CTLFLAG_RD, &sc->ports[i].stats_completed, "Requests completed"); } } static void ccr_init_port(struct ccr_softc *sc, int port) { struct port_info *pi; pi = sc->adapter->port[port]; sc->ports[port].txq = &sc->adapter->sge.ctrlq[port]; sc->ports[port].rxq = &sc->adapter->sge.rxq[pi->vi->first_rxq]; sc->ports[port].rx_channel_id = pi->rx_c_chan; sc->ports[port].tx_channel_id = pi->tx_chan; sc->ports[port].stats_queued = counter_u64_alloc(M_WAITOK); sc->ports[port].stats_completed = counter_u64_alloc(M_WAITOK); _Static_assert(sizeof(sc->port_mask) * NBBY >= MAX_NPORTS - 1, "Too many ports to fit in port_mask"); /* * Completions for crypto requests on port 1 can sometimes * return a stale cookie value due to a firmware bug. Disable * requests on port 1 by default on affected firmware. */ if (sc->adapter->params.fw_vers >= FW_VERSION32(1, 25, 4, 0) || port == 0) sc->port_mask |= 1u << port; } static int ccr_attach(device_t dev) { struct ccr_softc *sc; int32_t cid; int i; sc = device_get_softc(dev); sc->dev = dev; sysctl_ctx_init(&sc->ctx); sc->adapter = device_get_softc(device_get_parent(dev)); for_each_port(sc->adapter, i) { ccr_init_port(sc, i); } cid = crypto_get_driverid(dev, sizeof(struct ccr_session), CRYPTOCAP_F_HARDWARE); if (cid < 0) { device_printf(dev, "could not get crypto driver id\n"); return (ENXIO); } sc->cid = cid; sc->adapter->ccr_softc = sc; /* * The FID must be the first RXQ for port 0 regardless of * which port is used to service the request. */ sc->first_rxq_id = sc->adapter->sge.rxq[0].iq.abs_id; mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_hash = counter_u64_alloc(M_WAITOK); sc->stats_hmac = counter_u64_alloc(M_WAITOK); sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK); sc->stats_inflight = counter_u64_alloc(M_WAITOK); sc->stats_mac_error = counter_u64_alloc(M_WAITOK); sc->stats_pad_error = counter_u64_alloc(M_WAITOK); sc->stats_sglist_error = counter_u64_alloc(M_WAITOK); sc->stats_process_error = counter_u64_alloc(M_WAITOK); sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK); ccr_sysctls(sc); return (0); } static void ccr_free_port(struct ccr_softc *sc, int port) { counter_u64_free(sc->ports[port].stats_queued); counter_u64_free(sc->ports[port].stats_completed); } static int ccr_detach(device_t dev) { struct ccr_softc *sc; int i; sc = device_get_softc(dev); mtx_lock(&sc->lock); sc->detaching = true; mtx_unlock(&sc->lock); crypto_unregister_all(sc->cid); sysctl_ctx_free(&sc->ctx); mtx_destroy(&sc->lock); counter_u64_free(sc->stats_blkcipher_encrypt); counter_u64_free(sc->stats_blkcipher_decrypt); counter_u64_free(sc->stats_hash); counter_u64_free(sc->stats_hmac); counter_u64_free(sc->stats_eta_encrypt); counter_u64_free(sc->stats_eta_decrypt); counter_u64_free(sc->stats_gcm_encrypt); counter_u64_free(sc->stats_gcm_decrypt); counter_u64_free(sc->stats_ccm_encrypt); counter_u64_free(sc->stats_ccm_decrypt); counter_u64_free(sc->stats_wr_nomem); counter_u64_free(sc->stats_inflight); counter_u64_free(sc->stats_mac_error); counter_u64_free(sc->stats_pad_error); counter_u64_free(sc->stats_sglist_error); counter_u64_free(sc->stats_process_error); counter_u64_free(sc->stats_sw_fallback); for_each_port(sc->adapter, i) { ccr_free_port(sc, i); } sglist_free(sc->sg_iv_aad); free(sc->iv_aad_buf, M_CCR); sc->adapter->ccr_softc = NULL; return (0); } static void ccr_init_hash_digest(struct ccr_session *s) { union authctx auth_ctx; struct auth_hash *axf; axf = s->hmac.auth_hash; axf->Init(&auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads); } static bool ccr_aes_check_keylen(int alg, int klen) { switch (klen * 8) { case 128: case 192: if (alg == CRYPTO_AES_XTS) return (false); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) return (false); break; default: return (false); } return (true); } static void ccr_aes_setkey(struct ccr_session *s, const void *key, int klen) { unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size; unsigned int opad_present; if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) kbits = (klen / 2) * 8; else kbits = klen * 8; switch (kbits) { case 128: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; break; case 192: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; break; case 256: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; break; default: panic("should not get here"); } s->blkcipher.key_len = klen; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: case SCMD_CIPH_MODE_AES_XTS: t4_aes_getdeckey(s->blkcipher.deckey, key, kbits); break; } kctx_len = roundup2(s->blkcipher.key_len, 16); switch (s->mode) { case ETA: mk_size = s->hmac.mk_size; opad_present = 1; iopad_size = roundup2(s->hmac.partial_digest_len, 16); kctx_len += iopad_size * 2; break; case GCM: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; opad_present = 0; kctx_len += GMAC_BLOCK_LEN; break; case CCM: switch (kbits) { case 128: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; break; case 192: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; break; case 256: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; break; default: panic("should not get here"); } opad_present = 0; kctx_len *= 2; break; default: mk_size = CHCR_KEYCTX_NO_KEY; opad_present = 0; break; } kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) | V_KEY_CONTEXT_OPAD_PRESENT(opad_present) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) | V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1)); } static bool ccr_auth_supported(const struct crypto_session_params *csp) { switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: break; default: return (false); } return (true); } static bool ccr_cipher_supported(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (ccr_aes_check_keylen(csp->csp_cipher_alg, csp->csp_cipher_klen)); } static int ccr_cipher_mode(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: return (SCMD_CIPH_MODE_AES_CBC); case CRYPTO_AES_ICM: return (SCMD_CIPH_MODE_AES_CTR); case CRYPTO_AES_NIST_GCM_16: return (SCMD_CIPH_MODE_AES_GCM); case CRYPTO_AES_XTS: return (SCMD_CIPH_MODE_AES_XTS); case CRYPTO_AES_CCM_16: return (SCMD_CIPH_MODE_AES_CCM); default: return (SCMD_CIPH_MODE_NOP); } } static int ccr_probesession(device_t dev, const struct crypto_session_params *csp) { unsigned int cipher_mode; if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_DIGEST: if (!ccr_auth_supported(csp)) return (EINVAL); break; case CSP_MODE_CIPHER: if (!ccr_cipher_supported(csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); - if (csp->csp_auth_mlen < 0 || - csp->csp_auth_mlen > AES_GMAC_HASH_LEN) - return (EINVAL); - break; case CRYPTO_AES_CCM_16: break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp)) return (EINVAL); break; default: return (EINVAL); } if (csp->csp_cipher_klen != 0) { cipher_mode = ccr_cipher_mode(csp); if (cipher_mode == SCMD_CIPH_MODE_NOP) return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } /* * Select an available port with the lowest number of active sessions. */ static struct ccr_port * ccr_choose_port(struct ccr_softc *sc) { struct ccr_port *best, *p; int i; mtx_assert(&sc->lock, MA_OWNED); best = NULL; for (i = 0; i < nitems(sc->ports); i++) { p = &sc->ports[i]; /* Ignore non-existent ports. */ if (p->rxq == NULL) continue; /* * XXX: Ignore ports whose queues aren't initialized. * This is racy as the rxq can be destroyed by the * associated VI detaching. Eventually ccr should use * dedicated queues. */ if (p->rxq->iq.adapter == NULL || p->txq->adapter == NULL) continue; if ((sc->port_mask & (1u << i)) == 0) continue; if (best == NULL || p->active_sessions < best->active_sessions) best = p; } return (best); } static void ccr_delete_session(struct ccr_session *s) { sglist_free(s->sg_input); sglist_free(s->sg_output); sglist_free(s->sg_ulptx); sglist_free(s->sg_dsgl); mtx_destroy(&s->lock); } static int ccr_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ccr_softc *sc; struct ccr_session *s; struct auth_hash *auth_hash; unsigned int auth_mode, cipher_mode, mk_size; unsigned int partial_digest_len; switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: auth_hash = &auth_hash_hmac_sha1; auth_mode = SCMD_AUTH_MODE_SHA1; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; partial_digest_len = SHA1_HASH_LEN; break; case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: auth_hash = &auth_hash_hmac_sha2_224; auth_mode = SCMD_AUTH_MODE_SHA224; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: auth_hash = &auth_hash_hmac_sha2_256; auth_mode = SCMD_AUTH_MODE_SHA256; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: auth_hash = &auth_hash_hmac_sha2_384; auth_mode = SCMD_AUTH_MODE_SHA512_384; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: auth_hash = &auth_hash_hmac_sha2_512; auth_mode = SCMD_AUTH_MODE_SHA512_512; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; default: auth_hash = NULL; auth_mode = SCMD_AUTH_MODE_NOP; mk_size = 0; partial_digest_len = 0; break; } cipher_mode = ccr_cipher_mode(csp); #ifdef INVARIANTS switch (csp->csp_mode) { case CSP_MODE_CIPHER: if (cipher_mode == SCMD_CIPH_MODE_NOP || cipher_mode == SCMD_CIPH_MODE_AES_GCM || cipher_mode == SCMD_CIPH_MODE_AES_CCM) panic("invalid cipher algo"); break; case CSP_MODE_DIGEST: if (auth_mode == SCMD_AUTH_MODE_NOP) panic("invalid auth algo"); break; case CSP_MODE_AEAD: if (cipher_mode != SCMD_CIPH_MODE_AES_GCM && cipher_mode != SCMD_CIPH_MODE_AES_CCM) panic("invalid aead cipher algo"); if (auth_mode != SCMD_AUTH_MODE_NOP) panic("invalid aead auth aglo"); break; case CSP_MODE_ETA: if (cipher_mode == SCMD_CIPH_MODE_NOP || cipher_mode == SCMD_CIPH_MODE_AES_GCM || cipher_mode == SCMD_CIPH_MODE_AES_CCM) panic("invalid cipher algo"); if (auth_mode == SCMD_AUTH_MODE_NOP) panic("invalid auth algo"); break; default: panic("invalid csp mode"); } #endif s = crypto_get_driver_session(cses); mtx_init(&s->lock, "ccr session", NULL, MTX_DEF); s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT); if (s->sg_input == NULL || s->sg_output == NULL || s->sg_ulptx == NULL || s->sg_dsgl == NULL) { ccr_delete_session(s); return (ENOMEM); } sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); ccr_delete_session(s); return (ENXIO); } s->port = ccr_choose_port(sc); if (s->port == NULL) { mtx_unlock(&sc->lock); ccr_delete_session(s); return (ENXIO); } switch (csp->csp_mode) { case CSP_MODE_AEAD: if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) s->mode = CCM; else s->mode = GCM; break; case CSP_MODE_ETA: s->mode = ETA; break; case CSP_MODE_DIGEST: if (csp->csp_auth_klen != 0) s->mode = HMAC; else s->mode = HASH; break; case CSP_MODE_CIPHER: s->mode = BLKCIPHER; break; } if (s->mode == GCM) { if (csp->csp_auth_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = csp->csp_auth_mlen; t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen, s->gmac.ghash_h); } else if (s->mode == CCM) { if (csp->csp_auth_mlen == 0) s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN; else s->ccm_mac.hash_len = csp->csp_auth_mlen; } else if (auth_mode != SCMD_AUTH_MODE_NOP) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; s->hmac.mk_size = mk_size; s->hmac.partial_digest_len = partial_digest_len; if (csp->csp_auth_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else s->hmac.hash_len = csp->csp_auth_mlen; if (csp->csp_auth_key != NULL) t4_init_hmac_digest(auth_hash, partial_digest_len, csp->csp_auth_key, csp->csp_auth_klen, s->hmac.pads); else ccr_init_hash_digest(s); } if (cipher_mode != SCMD_CIPH_MODE_NOP) { s->blkcipher.cipher_mode = cipher_mode; s->blkcipher.iv_len = csp->csp_ivlen; if (csp->csp_cipher_key != NULL) ccr_aes_setkey(s, csp->csp_cipher_key, csp->csp_cipher_klen); } s->port->active_sessions++; mtx_unlock(&sc->lock); return (0); } static void ccr_freesession(device_t dev, crypto_session_t cses) { struct ccr_softc *sc; struct ccr_session *s; sc = device_get_softc(dev); s = crypto_get_driver_session(cses); #ifdef INVARIANTS if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); #endif mtx_lock(&sc->lock); s->port->active_sessions--; mtx_unlock(&sc->lock); ccr_delete_session(s); } static int ccr_process(device_t dev, struct cryptop *crp, int hint) { const struct crypto_session_params *csp; struct ccr_softc *sc; struct ccr_session *s; int error; csp = crypto_get_params(crp->crp_session); s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); mtx_lock(&s->lock); error = ccr_populate_sglist(s->sg_input, &crp->crp_buf); if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf); if (error) { counter_u64_add(sc->stats_sglist_error, 1); goto out; } switch (s->mode) { case HASH: error = ccr_hash(sc, s, crp); if (error == 0) counter_u64_add(sc->stats_hash, 1); break; case HMAC: if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, s->hmac.partial_digest_len, crp->crp_auth_key, csp->csp_auth_klen, s->hmac.pads); error = ccr_hash(sc, s, crp); if (error == 0) counter_u64_add(sc->stats_hmac, 1); break; case BLKCIPHER: if (crp->crp_cipher_key != NULL) ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); error = ccr_blkcipher(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_blkcipher_encrypt, 1); else counter_u64_add(sc->stats_blkcipher_decrypt, 1); } break; case ETA: if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, s->hmac.partial_digest_len, crp->crp_auth_key, csp->csp_auth_klen, s->hmac.pads); if (crp->crp_cipher_key != NULL) ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); error = ccr_eta(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_eta_encrypt, 1); else counter_u64_add(sc->stats_eta_decrypt, 1); } break; case GCM: if (crp->crp_cipher_key != NULL) { t4_init_gmac_hash(crp->crp_cipher_key, csp->csp_cipher_klen, s->gmac.ghash_h); ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); } if (crp->crp_payload_length == 0) { mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } error = ccr_gcm(sc, s, crp); if (error == EMSGSIZE || error == EFBIG) { counter_u64_add(sc->stats_sw_fallback, 1); mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_gcm_encrypt, 1); else counter_u64_add(sc->stats_gcm_decrypt, 1); } break; case CCM: if (crp->crp_cipher_key != NULL) { ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); } error = ccr_ccm(sc, s, crp); if (error == EMSGSIZE || error == EFBIG) { counter_u64_add(sc->stats_sw_fallback, 1); mtx_unlock(&s->lock); ccr_ccm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_ccm_encrypt, 1); else counter_u64_add(sc->stats_ccm_decrypt, 1); } break; } if (error == 0) { #ifdef INVARIANTS s->pending++; #endif counter_u64_add(sc->stats_inflight, 1); counter_u64_add(s->port->stats_queued, 1); } else counter_u64_add(sc->stats_process_error, 1); out: mtx_unlock(&s->lock); if (error) { crp->crp_etype = error; crypto_done(crp); } return (0); } static int do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct ccr_softc *sc = iq->adapter->ccr_softc; struct ccr_session *s; const struct cpl_fw6_pld *cpl; struct cryptop *crp; uint32_t status; int error; if (m != NULL) cpl = mtod(m, const void *); else cpl = (const void *)(rss + 1); crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]); s = crypto_get_driver_session(crp->crp_session); status = be64toh(cpl->data[0]); if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status)) error = EBADMSG; else error = 0; #ifdef INVARIANTS mtx_lock(&s->lock); s->pending--; mtx_unlock(&s->lock); #endif counter_u64_add(sc->stats_inflight, -1); counter_u64_add(s->port->stats_completed, 1); switch (s->mode) { case HASH: case HMAC: error = ccr_hash_done(sc, s, crp, cpl, error); break; case BLKCIPHER: error = ccr_blkcipher_done(sc, s, crp, cpl, error); break; case ETA: error = ccr_eta_done(sc, s, crp, cpl, error); break; case GCM: error = ccr_gcm_done(sc, s, crp, cpl, error); break; case CCM: error = ccr_ccm_done(sc, s, crp, cpl, error); break; } if (error == EBADMSG) { if (CHK_MAC_ERR_BIT(status)) counter_u64_add(sc->stats_mac_error, 1); if (CHK_PAD_ERR_BIT(status)) counter_u64_add(sc->stats_pad_error, 1); } crp->crp_etype = error; crypto_done(crp); m_freem(m); return (0); } static int ccr_modevent(module_t mod, int cmd, void *arg) { switch (cmd) { case MOD_LOAD: t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld); return (0); case MOD_UNLOAD: t4_register_cpl_handler(CPL_FW6_PLD, NULL); return (0); default: return (EOPNOTSUPP); } } static device_method_t ccr_methods[] = { DEVMETHOD(device_identify, ccr_identify), DEVMETHOD(device_probe, ccr_probe), DEVMETHOD(device_attach, ccr_attach), DEVMETHOD(device_detach, ccr_detach), DEVMETHOD(cryptodev_probesession, ccr_probesession), DEVMETHOD(cryptodev_newsession, ccr_newsession), DEVMETHOD(cryptodev_freesession, ccr_freesession), DEVMETHOD(cryptodev_process, ccr_process), DEVMETHOD_END }; static driver_t ccr_driver = { "ccr", ccr_methods, sizeof(struct ccr_softc) }; static devclass_t ccr_devclass; DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL); MODULE_VERSION(ccr, 1); MODULE_DEPEND(ccr, crypto, 1, 1, 1); MODULE_DEPEND(ccr, t6nex, 1, 1, 1); diff --git a/sys/dev/qat/qat.c b/sys/dev/qat/qat.c index b5d3f4d9629a..dddfe4da1083 100644 --- a/sys/dev/qat/qat.c +++ b/sys/dev/qat/qat.c @@ -1,2298 +1,2296 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $"); #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include #include #include "qatreg.h" #include "qatvar.h" #include "qat_aevar.h" extern struct qat_hw qat_hw_c2xxx; extern struct qat_hw qat_hw_c3xxx; extern struct qat_hw qat_hw_c62x; extern struct qat_hw qat_hw_d15xx; extern struct qat_hw qat_hw_dh895xcc; #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18 #define PCI_PRODUCT_INTEL_C3K_QAT 0x19e2 #define PCI_PRODUCT_INTEL_C3K_QAT_VF 0x19e3 #define PCI_PRODUCT_INTEL_C620_QAT 0x37c8 #define PCI_PRODUCT_INTEL_C620_QAT_VF 0x37c9 #define PCI_PRODUCT_INTEL_XEOND_QAT 0x6f54 #define PCI_PRODUCT_INTEL_XEOND_QAT_VF 0x6f55 #define PCI_PRODUCT_INTEL_DH895XCC_QAT 0x0435 #define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF 0x0443 static const struct qat_product { uint16_t qatp_vendor; uint16_t qatp_product; const char *qatp_name; enum qat_chip_type qatp_chip; const struct qat_hw *qatp_hw; } qat_products[] = { { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS, "Intel C2000 QuickAssist PF", QAT_CHIP_C2XXX, &qat_hw_c2xxx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C3K_QAT, "Intel C3000 QuickAssist PF", QAT_CHIP_C3XXX, &qat_hw_c3xxx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C620_QAT, "Intel C620/Xeon D-2100 QuickAssist PF", QAT_CHIP_C62X, &qat_hw_c62x }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XEOND_QAT, "Intel Xeon D-1500 QuickAssist PF", QAT_CHIP_D15XX, &qat_hw_d15xx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH895XCC_QAT, "Intel 8950 QuickAssist PCIe Adapter PF", QAT_CHIP_DH895XCC, &qat_hw_dh895xcc }, { 0, 0, NULL, 0, NULL }, }; /* Hash Algorithm specific structure */ /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = { 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0 }; /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 }; /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = { 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 }; /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 }; static const struct qat_sym_hash_alg_info sha1_info = { .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE, .qshai_init_state = sha1_initial_state, .qshai_sah = &auth_hash_hmac_sha1, .qshai_state_offset = 0, .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha256_info = { .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE, .qshai_init_state = sha256_initial_state, .qshai_sah = &auth_hash_hmac_sha2_256, .qshai_state_offset = offsetof(SHA256_CTX, state), .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha384_info = { .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE, .qshai_init_state = sha384_initial_state, .qshai_sah = &auth_hash_hmac_sha2_384, .qshai_state_offset = offsetof(SHA384_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info sha512_info = { .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE, .qshai_init_state = sha512_initial_state, .qshai_sah = &auth_hash_hmac_sha2_512, .qshai_state_offset = offsetof(SHA512_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info aes_gcm_info = { .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE, .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE, .qshai_sah = &auth_hash_nist_gmac_aes_128, }; /* Hash QAT specific structures */ static const struct qat_sym_hash_qat_info sha1_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA1, .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE, .qshqi_state1_len = HW_SHA1_STATE1_SZ, .qshqi_state2_len = HW_SHA1_STATE2_SZ, }; static const struct qat_sym_hash_qat_info sha256_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA256, .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE, .qshqi_state1_len = HW_SHA256_STATE1_SZ, .qshqi_state2_len = HW_SHA256_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha384_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA384, .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE, .qshqi_state1_len = HW_SHA384_STATE1_SZ, .qshqi_state2_len = HW_SHA384_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha512_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA512, .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE, .qshqi_state1_len = HW_SHA512_STATE1_SZ, .qshqi_state2_len = HW_SHA512_STATE2_SZ }; static const struct qat_sym_hash_qat_info aes_gcm_config = { .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128, .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ, .qshqi_state2_len = HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ, }; static const struct qat_sym_hash_def qat_sym_hash_defs[] = { [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config }, [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config }, [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config }, [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config }, [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config }, }; static const struct qat_product *qat_lookup(device_t); static int qat_probe(device_t); static int qat_attach(device_t); static int qat_init(device_t); static int qat_start(device_t); static int qat_detach(device_t); static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp); static void qat_freesession(device_t dev, crypto_session_t cses); static int qat_setup_msix_intr(struct qat_softc *); static void qat_etr_init(struct qat_softc *); static void qat_etr_deinit(struct qat_softc *); static void qat_etr_bank_init(struct qat_softc *, int); static void qat_etr_bank_deinit(struct qat_softc *sc, int); static void qat_etr_ap_bank_init(struct qat_softc *); static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int); static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *, uint32_t, int); static void qat_etr_ap_bank_setup_ring(struct qat_softc *, struct qat_ring *); static int qat_etr_verify_ring_size(uint32_t, uint32_t); static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *, struct qat_ring *); static void qat_etr_bank_intr(void *); static void qat_arb_update(struct qat_softc *, struct qat_bank *); static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie( struct qat_crypto_bank *); static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *, struct qat_sym_cookie *); static int qat_crypto_setup_ring(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_bank_init(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_init(struct qat_softc *); static void qat_crypto_deinit(struct qat_softc *); static int qat_crypto_start(struct qat_softc *); static void qat_crypto_stop(struct qat_softc *); static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *); static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver"); static const struct qat_product * qat_lookup(device_t dev) { const struct qat_product *qatp; for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) { if (pci_get_vendor(dev) == qatp->qatp_vendor && pci_get_device(dev) == qatp->qatp_product) return qatp; } return NULL; } static int qat_probe(device_t dev) { const struct qat_product *prod; prod = qat_lookup(dev); if (prod != NULL) { device_set_desc(dev, prod->qatp_name); return BUS_PROBE_DEFAULT; } return ENXIO; } static int qat_attach(device_t dev) { struct qat_softc *sc = device_get_softc(dev); const struct qat_product *qatp; int bar, count, error, i; sc->sc_dev = dev; sc->sc_rev = pci_get_revid(dev); sc->sc_crypto.qcy_cid = -1; qatp = qat_lookup(dev); memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw)); /* Determine active accelerators and engines */ sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc); sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc); sc->sc_accel_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) { if (sc->sc_accel_mask & (1 << i)) sc->sc_accel_num++; } sc->sc_ae_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) { if (sc->sc_ae_mask & (1 << i)) sc->sc_ae_num++; } if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) { device_printf(sc->sc_dev, "couldn't find acceleration"); goto fail; } MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL); MPASS(sc->sc_ae_num <= MAX_NUM_AE); /* Determine SKU and capabilities */ sc->sc_sku = sc->sc_hw.qhw_get_sku(sc); sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc); sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc); i = 0; if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) { MPASS(sc->sc_hw.qhw_sram_bar_id == 0); uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4); /* Skip SRAM BAR */ i = (fusectl & FUSECTL_MASK) ? 1 : 0; } for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) { uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4); if (val == 0 || !PCI_BAR_MEM(val)) continue; sc->sc_rid[i] = PCIR_BAR(bar); sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid[i], RF_ACTIVE); if (sc->sc_res[i] == NULL) { device_printf(dev, "couldn't map BAR %d\n", bar); goto fail; } sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]); sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]); i++; if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) bar++; } pci_enable_busmaster(dev); count = sc->sc_hw.qhw_num_banks + 1; if (pci_msix_count(dev) < count) { device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n", pci_msix_count(dev), count); goto fail; } error = pci_alloc_msix(dev, &count); if (error != 0) { device_printf(dev, "failed to allocate MSI-X vectors\n"); goto fail; } error = qat_init(dev); if (error == 0) return 0; fail: qat_detach(dev); return ENXIO; } static int qat_init(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; qat_etr_init(sc); if (sc->sc_hw.qhw_init_admin_comms != NULL && (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize admin comms: %d\n", error); return error; } if (sc->sc_hw.qhw_init_arb != NULL && (error = sc->sc_hw.qhw_init_arb(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize hw arbiter: %d\n", error); return error; } error = qat_ae_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize Acceleration Engine: %d\n", error); return error; } error = qat_aefw_load(sc); if (error) { device_printf(sc->sc_dev, "Could not load firmware: %d\n", error); return error; } error = qat_setup_msix_intr(sc); if (error) { device_printf(sc->sc_dev, "Could not setup interrupts: %d\n", error); return error; } sc->sc_hw.qhw_enable_intr(sc); error = qat_crypto_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize service: %d\n", error); return error; } if (sc->sc_hw.qhw_enable_error_correction != NULL) sc->sc_hw.qhw_enable_error_correction(sc); if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL && (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize watchdog timer: %d\n", error); return error; } error = qat_start(dev); if (error) { device_printf(sc->sc_dev, "Could not start: %d\n", error); return error; } return 0; } static int qat_start(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; error = qat_ae_start(sc); if (error) return error; if (sc->sc_hw.qhw_send_admin_init != NULL && (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) { return error; } error = qat_crypto_start(sc); if (error) return error; return 0; } static int qat_detach(device_t dev) { struct qat_softc *sc; int bar, i; sc = device_get_softc(dev); qat_crypto_stop(sc); qat_crypto_deinit(sc); qat_aefw_unload(sc); if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i]; if (qb->qb_ih_cookie != NULL) (void)bus_teardown_intr(dev, qb->qb_ih, qb->qb_ih_cookie); if (qb->qb_ih != NULL) (void)bus_release_resource(dev, SYS_RES_IRQ, i + 1, qb->qb_ih); } } if (sc->sc_ih_cookie != NULL) { (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie); sc->sc_ih_cookie = NULL; } if (sc->sc_ih != NULL) { (void)bus_release_resource(dev, SYS_RES_IRQ, sc->sc_hw.qhw_num_banks + 1, sc->sc_ih); sc->sc_ih = NULL; } pci_release_msi(dev); qat_etr_deinit(sc); for (bar = 0; bar < MAX_BARS; bar++) { if (sc->sc_res[bar] != NULL) { (void)bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid[bar], sc->sc_res[bar]); sc->sc_res[bar] = NULL; } } return 0; } void * qat_alloc_mem(size_t size) { return (malloc(size, M_QAT, M_WAITOK | M_ZERO)); } void qat_free_mem(void *ptr) { free(ptr, M_QAT); } static void qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct qat_dmamem *qdm; if (error != 0) return; KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); qdm = arg; qdm->qdm_dma_seg = segs[0]; } int qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm, int nseg, bus_size_t size, bus_size_t alignment) { int error; KASSERT(qdm->qdm_dma_vaddr == NULL, ("%s: DMA memory descriptor in use", __func__)); error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), alignment, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ nseg, /* nsegments */ size, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &qdm->qdm_dma_tag); if (error != 0) return error; error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &qdm->qdm_dma_map); if (error != 0) { device_printf(sc->sc_dev, "couldn't allocate dmamem, error = %d\n", error); goto fail_0; } error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map, qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm, BUS_DMA_NOWAIT); if (error) { device_printf(sc->sc_dev, "couldn't load dmamem map, error = %d\n", error); goto fail_1; } return 0; fail_1: bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); fail_0: bus_dma_tag_destroy(qdm->qdm_dma_tag); return error; } void qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm) { if (qdm->qdm_dma_tag != NULL) { bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map); bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); bus_dma_tag_destroy(qdm->qdm_dma_tag); explicit_bzero(qdm, sizeof(*qdm)); } } static int qat_setup_msix_intr(struct qat_softc *sc) { device_t dev; int error, i, rid; dev = sc->sc_dev; for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i - 1]; rid = i; qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (qb->qb_ih == NULL) { device_printf(dev, "failed to allocate bank intr resource\n"); return ENXIO; } error = bus_setup_intr(dev, qb->qb_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb, &qb->qb_ih_cookie); if (error != 0) { device_printf(dev, "failed to set up bank intr\n"); return error; } error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus); if (error != 0) device_printf(dev, "failed to bind intr %d\n", i); } rid = i; sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_ih == NULL) return ENXIO; error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie); return error; } static void qat_etr_init(struct qat_softc *sc) { int i; sc->sc_etr_banks = qat_alloc_mem( sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks); for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_init(sc, i); if (sc->sc_hw.qhw_num_ap_banks) { sc->sc_etr_ap_banks = qat_alloc_mem( sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks); qat_etr_ap_bank_init(sc); } } static void qat_etr_deinit(struct qat_softc *sc) { int i; if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_deinit(sc, i); qat_free_mem(sc->sc_etr_banks); sc->sc_etr_banks = NULL; } if (sc->sc_etr_ap_banks != NULL) { qat_free_mem(sc->sc_etr_ap_banks); sc->sc_etr_ap_banks = NULL; } } static void qat_etr_bank_init(struct qat_softc *sc, int bank) { struct qat_bank *qb = &sc->sc_etr_banks[bank]; int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap; MPASS(bank < sc->sc_hw.qhw_num_banks); mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF); qb->qb_sc = sc; qb->qb_bank = bank; qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT; /* Clean CSRs for all rings within the bank */ for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { struct qat_ring *qr = &qb->qb_et_rings[i]; qat_etr_bank_ring_write_4(sc, bank, i, ETR_RING_CONFIG, 0); qat_etr_bank_ring_base_write_8(sc, bank, i, 0); if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t)); } else if (sc->sc_hw.qhw_tx_rings_mask & (1 << (i - tx_rx_gap))) { /* Share inflight counter with rx and tx */ qr->qr_inflight = qb->qb_et_rings[i - tx_rx_gap].qr_inflight; } } if (sc->sc_hw.qhw_init_etr_intr != NULL) { sc->sc_hw.qhw_init_etr_intr(sc, bank); } else { /* common code in qat 1.7 */ qat_etr_bank_write_4(sc, bank, ETR_INT_REG, ETR_INT_REG_CLEAR_MASK); for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank / ETR_RINGS_PER_INT_SRCSEL; i++) { qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL + (i * ETR_INT_SRCSEL_NEXT_OFFSET), ETR_INT_SRCSEL_MASK); } } } static void qat_etr_bank_deinit(struct qat_softc *sc, int bank) { struct qat_bank *qb; struct qat_ring *qr; int i; qb = &sc->sc_etr_banks[bank]; for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr = &qb->qb_et_rings[i]; qat_free_mem(qr->qr_inflight); } } } static void qat_etr_ap_bank_init(struct qat_softc *sc) { int ap_bank; for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) { struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank]; qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK, ETR_AP_NF_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK, ETR_AP_NE_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0); memset(qab, 0, sizeof(*qab)); } } static void qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask) { if (set_mask) *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); else *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); } static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest, uint32_t ring, int set_dest) { uint32_t ae_mask; uint8_t mailbox, ae, nae; uint8_t *dest = (uint8_t *)ap_dest; mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring); nae = 0; ae_mask = sc->sc_ae_mask; for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) { if ((ae_mask & (1 << ae)) == 0) continue; if (set_dest) { dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) | __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) | ETR_AP_DEST_ENABLE; } else { dest[nae] = 0; } nae++; if (nae == ETR_MAX_AE_PER_MAILBOX) break; } } static void qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr) { struct qat_ap_bank *qab; int ap_bank; if (sc->sc_hw.qhw_num_ap_banks == 0) return; ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring); MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks); qab = &sc->sc_etr_ap_banks[ap_bank]; if (qr->qr_cb == NULL) { qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1); if (!qab->qab_ne_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, qab->qab_ne_dest); } } else { qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1); if (!qab->qab_nf_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, qab->qab_nf_dest); } } } static int qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs) { int i = QAT_MIN_RING_SIZE; for (; i <= QAT_MAX_RING_SIZE; i++) if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i)) return i; return QAT_DEFAULT_RING_SIZE; } int qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring, uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg, const char *name, struct qat_ring **rqr) { struct qat_bank *qb; struct qat_ring *qr = NULL; int error; uint32_t ring_size_bytes, ring_config; uint64_t ring_base; uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512; uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0; MPASS(bank < sc->sc_hw.qhw_num_banks); /* Allocate a ring from specified bank */ qb = &sc->sc_etr_banks[bank]; if (ring >= sc->sc_hw.qhw_num_rings_per_bank) return EINVAL; if (qb->qb_allocated_rings & (1 << ring)) return ENOENT; qr = &qb->qb_et_rings[ring]; qb->qb_allocated_rings |= 1 << ring; /* Initialize allocated ring */ qr->qr_ring = ring; qr->qr_bank = bank; qr->qr_name = name; qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring; qr->qr_ring_mask = (1 << ring); qr->qr_cb = cb; qr->qr_cb_arg = cb_arg; /* Setup the shadow variables */ qr->qr_head = 0; qr->qr_tail = 0; qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size); qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs); /* * To make sure that ring is alligned to ring size allocate * at least 4k and then tell the user it is smaller. */ ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size); ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes); error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes, ring_size_bytes); if (error) return error; qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr; qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr; memset(qr->qr_ring_vaddr, QAT_RING_PATTERN, qr->qr_dma.qdm_dma_seg.ds_len); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (cb == NULL) { ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size); } else { ring_config = ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne); } qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config); ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size); qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base); if (sc->sc_hw.qhw_init_arb != NULL) qat_arb_update(sc, qb); mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF); qat_etr_ap_bank_setup_ring(sc, qr); if (cb != NULL) { uint32_t intr_mask; qb->qb_intr_mask |= qr->qr_ring_mask; intr_mask = qb->qb_intr_mask; qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask); qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); } *rqr = qr; return 0; } static inline u_int qat_modulo(u_int data, u_int shift) { u_int div = data >> shift; u_int mult = div << shift; return data - mult; } int qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg) { uint32_t inflight; uint32_t *addr; mtx_lock(&qr->qr_ring_mtx); inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1; if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) { atomic_subtract_32(qr->qr_inflight, 1); qr->qr_need_wakeup = true; mtx_unlock(&qr->qr_ring_mtx); counter_u64_add(sc->sc_ring_full_restarts, 1); return ERESTART; } addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail); memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size)); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE); qr->qr_tail = qat_modulo(qr->qr_tail + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_TAIL_OFFSET, qr->qr_tail); mtx_unlock(&qr->qr_ring_mtx); return 0; } static int qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb, struct qat_ring *qr) { uint32_t *msg, nmsg = 0; int handled = 0; bool blocked = false; mtx_lock(&qr->qr_ring_mtx); msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) { atomic_subtract_32(qr->qr_inflight, 1); if (qr->qr_cb != NULL) { mtx_unlock(&qr->qr_ring_mtx); handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg); mtx_lock(&qr->qr_ring_mtx); } atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG); qr->qr_head = qat_modulo(qr->qr_head + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); nmsg++; msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); } bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (nmsg > 0) { qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_HEAD_OFFSET, qr->qr_head); if (qr->qr_need_wakeup) { blocked = true; qr->qr_need_wakeup = false; } } mtx_unlock(&qr->qr_ring_mtx); if (blocked) crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ); return handled; } static void qat_etr_bank_intr(void *arg) { struct qat_bank *qb = arg; struct qat_softc *sc = qb->qb_sc; uint32_t estat; int i; mtx_lock(&qb->qb_bank_mtx); qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0); /* Now handle all the responses */ estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT); estat &= qb->qb_intr_mask; qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); mtx_unlock(&qb->qb_bank_mtx); while ((i = ffs(estat)) != 0) { struct qat_ring *qr = &qb->qb_et_rings[--i]; estat &= ~(1 << i); (void)qat_etr_ring_intr(sc, qb, qr); } } void qat_arb_update(struct qat_softc *sc, struct qat_bank *qb) { qat_arb_ringsrvarben_write_4(sc, qb->qb_bank, qb->qb_allocated_rings & 0xff); } static struct qat_sym_cookie * qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb) { struct qat_sym_cookie *qsc; mtx_lock(&qcb->qcb_bank_mtx); if (qcb->qcb_symck_free_count == 0) { mtx_unlock(&qcb->qcb_bank_mtx); return NULL; } qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count]; mtx_unlock(&qcb->qcb_bank_mtx); return qsc; } static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb, struct qat_sym_cookie *qsc) { explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN); explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN); mtx_lock(&qcb->qcb_bank_mtx); qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc; mtx_unlock(&qcb->qcb_bank_mtx); } void qat_memcpy_htobe64(void *dst, const void *src, size_t len) { uint64_t *dst0 = dst; const uint64_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe64(*(src0 + i)); } void qat_memcpy_htobe32(void *dst, const void *src, size_t len) { uint32_t *dst0 = dst; const uint32_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe32(*(src0 + i)); } void qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte) { switch (wordbyte) { case 4: qat_memcpy_htobe32(dst, src, len); break; case 8: qat_memcpy_htobe64(dst, src, len); break; default: panic("invalid word size %u", wordbyte); } } void qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state) { uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; char zeros[AES_BLOCK_LEN]; int rounds; memset(zeros, 0, sizeof(zeros)); rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); rijndaelEncrypt(ks, rounds, zeros, state); explicit_bzero(ks, sizeof(ks)); } void qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state1, uint8_t *state2) { union authctx ctx; const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah; uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset; uint32_t state_size = hash_def->qshd_alg->qshai_state_size; uint32_t state_word = hash_def->qshd_alg->qshai_state_word; hmac_init_ipad(sah, key, klen, &ctx); qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size, state_word); hmac_init_opad(sah, key, klen, &ctx); qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size, state_word); explicit_bzero(&ctx, sizeof(ctx)); } static enum hw_cipher_algo qat_aes_cipher_algo(int klen) { switch (klen) { case HW_AES_128_KEY_SZ: return HW_CIPHER_ALGO_AES128; case HW_AES_192_KEY_SZ: return HW_CIPHER_ALGO_AES192; case HW_AES_256_KEY_SZ: return HW_CIPHER_ALGO_AES256; default: panic("invalid key length %d", klen); } } uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc, const struct qat_session *qs) { enum hw_cipher_algo algo; enum hw_cipher_dir dir; enum hw_cipher_convert key_convert; enum hw_cipher_mode mode; dir = desc->qcd_cipher_dir; key_convert = HW_CIPHER_NO_CONVERT; mode = qs->qs_cipher_mode; switch (mode) { case HW_CIPHER_CBC_MODE: case HW_CIPHER_XTS_MODE: algo = qs->qs_cipher_algo; /* * AES decrypt key needs to be reversed. * Instead of reversing the key at session registration, * it is instead reversed on-the-fly by setting the KEY_CONVERT * bit here. */ if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT) key_convert = HW_CIPHER_KEY_CONVERT; break; case HW_CIPHER_CTR_MODE: algo = qs->qs_cipher_algo; dir = HW_CIPHER_ENCRYPT; break; default: panic("unhandled cipher mode %d", mode); break; } return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir); } uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct qat_sym_hash_def **hash_def) { enum qat_sym_hash_algorithm algo; switch (qs->qs_auth_algo) { case HW_AUTH_ALGO_SHA1: algo = QAT_SYM_HASH_SHA1; break; case HW_AUTH_ALGO_SHA256: algo = QAT_SYM_HASH_SHA256; break; case HW_AUTH_ALGO_SHA384: algo = QAT_SYM_HASH_SHA384; break; case HW_AUTH_ALGO_SHA512: algo = QAT_SYM_HASH_SHA512; break; case HW_AUTH_ALGO_GALOIS_128: algo = QAT_SYM_HASH_AES_GCM; break; default: panic("unhandled auth algorithm %d", qs->qs_auth_algo); break; } *hash_def = &qat_sym_hash_defs[algo]; return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode, (*hash_def)->qshd_qat->qshqi_algo_enc, (*hash_def)->qshd_alg->qshai_digest_len); } struct qat_crypto_load_cb_arg { struct qat_session *qs; struct qat_sym_cookie *qsc; struct cryptop *crp; int error; }; static int qat_crypto_populate_buf_list(struct buffer_list_desc *buffers, bus_dma_segment_t *segs, int niseg, int noseg, int skip) { struct flat_buffer_desc *flatbuf; bus_addr_t addr; bus_size_t len; int iseg, oseg; for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG; iseg++) { addr = segs[iseg].ds_addr; len = segs[iseg].ds_len; if (skip > 0) { if (skip < len) { addr += skip; len -= skip; skip = 0; } else { skip -= len; continue; } } flatbuf = &buffers->flat_bufs[oseg++]; flatbuf->data_len_in_bytes = (uint32_t)len; flatbuf->phy_buffer = (uint64_t)addr; } buffers->num_buffers = oseg; return iseg < niseg ? E2BIG : 0; } static void qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct qat_crypto_load_cb_arg *arg; struct qat_sym_cookie *qsc; arg = _arg; if (error != 0) { arg->error = error; return; } qsc = arg->qsc; arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, nseg, 0, 0); } static void qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct cryptop *crp; struct qat_crypto_load_cb_arg *arg; struct qat_session *qs; struct qat_sym_cookie *qsc; int noseg, skip; arg = _arg; if (error != 0) { arg->error = error; return; } crp = arg->crp; qs = arg->qs; qsc = arg->qsc; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { /* AAD was handled in qat_crypto_load(). */ skip = crp->crp_payload_start; noseg = 0; } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) { skip = crp->crp_aad_start; noseg = 0; } else { skip = crp->crp_payload_start; noseg = crp->crp_aad == NULL ? 0 : qsc->qsc_buf_list.num_buffers; } arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, nseg, noseg, skip); } static void qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct buffer_list_desc *ibufs, *obufs; struct flat_buffer_desc *ibuf, *obuf; struct cryptop *crp; struct qat_crypto_load_cb_arg *arg; struct qat_session *qs; struct qat_sym_cookie *qsc; int buflen, osegs, tocopy; arg = _arg; if (error != 0) { arg->error = error; return; } crp = arg->crp; qs = arg->qs; qsc = arg->qsc; /* * The payload must start at the same offset in the output SG list as in * the input SG list. Copy over SG entries from the input corresponding * to the AAD buffer. */ osegs = 0; if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 && crp->crp_aad_length > 0) { tocopy = crp->crp_aad == NULL ? crp->crp_payload_start - crp->crp_aad_start : crp->crp_aad_length; ibufs = &qsc->qsc_buf_list; obufs = &qsc->qsc_obuf_list; for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) { ibuf = &ibufs->flat_bufs[osegs]; obuf = &obufs->flat_bufs[osegs]; obuf->phy_buffer = ibuf->phy_buffer; buflen = imin(ibuf->data_len_in_bytes, tocopy); obuf->data_len_in_bytes = buflen; tocopy -= buflen; } } arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs, nseg, osegs, crp->crp_payload_output_start); } static int qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc, struct qat_crypto_desc const *desc, struct cryptop *crp) { struct qat_crypto_load_cb_arg arg; int error; crypto_read_iv(crp, qsc->qsc_iv_buf); arg.crp = crp; arg.qs = qs; arg.qsc = qsc; arg.error = 0; error = 0; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 && crp->crp_aad_length > 0) { /* * The firmware expects AAD to be in a contiguous buffer and * padded to a multiple of 16 bytes. To satisfy these * constraints we bounce the AAD into a per-request buffer. * There is a small limit on the AAD size so this is not too * onerous. */ memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX); if (crp->crp_aad == NULL) { crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, qsc->qsc_gcm_aad); } else { memcpy(qsc->qsc_gcm_aad, crp->crp_aad, crp->crp_aad_length); } } else if (crp->crp_aad != NULL) { error = bus_dmamap_load( qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, crp->crp_aad, crp->crp_aad_length, qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } if (error == 0) { error = bus_dmamap_load_crp_buffer( qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, &crp->crp_buf, qat_crypto_load_buf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) { error = bus_dmamap_load_crp_buffer( qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg, BUS_DMA_NOWAIT); if (error == 0) error = arg.error; } return error; } static inline struct qat_crypto_bank * qat_crypto_select_bank(struct qat_crypto *qcy) { u_int cpuid = PCPU_GET(cpuid); return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks]; } static int qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb) { char *name; int bank, curname, error, i, j; bank = qcb->qcb_bank; curname = 0; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size, NULL, NULL, name, &qcb->qcb_sym_tx); if (error) return error; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size, qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx); if (error) return error; for (i = 0; i < QAT_NSYMCOOKIE; i++) { struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i]; struct qat_sym_cookie *qsc; error = qat_alloc_dmamem(sc, qdm, 1, sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN); if (error) return error; qsc = qdm->qdm_dma_vaddr; qsc->qsc_self_dmamap = qdm->qdm_dma_map; qsc->qsc_self_dma_tag = qdm->qdm_dma_tag; qsc->qsc_bulk_req_params_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_bulk_cookie.qsbc_req_params_buf); qsc->qsc_buffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_buf_list); qsc->qsc_obuffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_obuf_list); qsc->qsc_obuffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_obuf_list); qsc->qsc_iv_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_iv_buf); qsc->qsc_auth_res_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_auth_res); qsc->qsc_gcm_aad_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_gcm_aad); qsc->qsc_content_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_content_desc); qcb->qcb_symck_free[i] = qsc; qcb->qcb_symck_free_count++; for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ QAT_MAXLEN, /* maxsize */ QAT_MAXSEG, /* nsegments */ QAT_MAXLEN, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &qsc->qsc_dma[j].qsd_dma_tag); if (error != 0) return error; error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag, BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap); if (error != 0) return error; } } return 0; } static int qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb) { mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF); return qat_crypto_setup_ring(sc, qcb); } static void qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb) { struct qat_dmamem *qdm; struct qat_sym_cookie *qsc; int i, j; for (i = 0; i < QAT_NSYMCOOKIE; i++) { qdm = &qcb->qcb_symck_dmamems[i]; qsc = qcb->qcb_symck_free[i]; for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag, qsc->qsc_dma[j].qsd_dmamap); bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag); } qat_free_dmamem(sc, qdm); } qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma); qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma); mtx_destroy(&qcb->qcb_bank_mtx); } static int qat_crypto_init(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; int bank, error, num_banks; qcy->qcy_sc = sc; if (sc->sc_hw.qhw_init_arb != NULL) num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks); else num_banks = sc->sc_ae_num; qcy->qcy_num_banks = num_banks; qcy->qcy_banks = qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks); for (bank = 0; bank < num_banks; bank++) { struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank]; qcb->qcb_bank = bank; error = qat_crypto_bank_init(sc, qcb); if (error) return error; } mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF); ctx = device_get_sysctl_ctx(sc->sc_dev); oid = device_get_sysctl_tree(sc->sc_dev); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts", CTLFLAG_RD, &sc->sc_gcm_aad_restarts, "GCM requests deferred due to AAD size change"); sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates", CTLFLAG_RD, &sc->sc_gcm_aad_updates, "GCM requests that required session state update"); sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full", CTLFLAG_RD, &sc->sc_ring_full_restarts, "Requests deferred due to in-flight max reached"); sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures", CTLFLAG_RD, &sc->sc_sym_alloc_failures, "Request allocation failures"); return 0; } static void qat_crypto_deinit(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct qat_crypto_bank *qcb; int bank; counter_u64_free(sc->sc_sym_alloc_failures); counter_u64_free(sc->sc_ring_full_restarts); counter_u64_free(sc->sc_gcm_aad_updates); counter_u64_free(sc->sc_gcm_aad_restarts); if (qcy->qcy_banks != NULL) { for (bank = 0; bank < qcy->qcy_num_banks; bank++) { qcb = &qcy->qcy_banks[bank]; qat_crypto_bank_deinit(sc, qcb); } qat_free_mem(qcy->qcy_banks); mtx_destroy(&qcy->qcy_crypto_mtx); } } static int qat_crypto_start(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; qcy->qcy_cid = crypto_get_driverid(sc->sc_dev, sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE); if (qcy->qcy_cid < 0) { device_printf(sc->sc_dev, "could not get opencrypto driver id\n"); return ENOENT; } return 0; } static void qat_crypto_stop(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; if (qcy->qcy_cid >= 0) (void)crypto_unregister_all(qcy->qcy_cid); } static void qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i) { bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap); } static int qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg) { char icv[QAT_SYM_HASH_BUFFER_LEN]; struct qat_crypto_bank *qcb = arg; struct qat_crypto *qcy; struct qat_session *qs; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; struct cryptop *crp; int error; uint16_t auth_sz; bool blocked; qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset); qsbc = &qsc->qsc_bulk_cookie; qcy = qsbc->qsbc_crypto; qs = qsbc->qsbc_session; crp = qsbc->qsbc_cb_tag; bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); if (crp->crp_aad != NULL) qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF); qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF); error = 0; if ((auth_sz = qs->qs_auth_mlen) != 0) { if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { crypto_copydata(crp, crp->crp_digest_start, auth_sz, icv); if (timingsafe_bcmp(icv, qsc->qsc_auth_res, auth_sz) != 0) { error = EBADMSG; } } else { crypto_copyback(crp, crp->crp_digest_start, auth_sz, qsc->qsc_auth_res); } } qat_crypto_free_sym_cookie(qcb, qsc); blocked = false; mtx_lock(&qs->qs_session_mtx); MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE); qs->qs_inflight--; if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) { blocked = true; qs->qs_need_wakeup = false; } mtx_unlock(&qs->qs_session_mtx); crp->crp_etype = error; crypto_done(crp); if (blocked) crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ); return 1; } static int qat_probesession(device_t dev, const struct crypto_session_params *csp) { if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) return EINVAL; if (csp->csp_cipher_alg == CRYPTO_AES_XTS && qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) { /* * AES-XTS is not supported by the NanoQAT. */ return EINVAL; } switch (csp->csp_mode) { case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_DIGEST: switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: break; case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_ETA: switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; default: return EINVAL; } break; default: return EINVAL; } return CRYPTODEV_PROBE_HARDWARE; } static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct qat_crypto *qcy; struct qat_dmamem *qdm; struct qat_session *qs; struct qat_softc *sc; struct qat_crypto_desc *ddesc, *edesc; int error, slices; sc = device_get_softc(dev); qs = crypto_get_driver_session(cses); qcy = &sc->sc_crypto; qdm = &qs->qs_desc_mem; error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG, sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN); if (error != 0) return error; mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF); qs->qs_aad_length = -1; qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr; qs->qs_enc_desc = edesc = ddesc + 1; ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr; ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr + sizeof(struct qat_crypto_desc); edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); qs->qs_status = QAT_SESSION_STATUS_ACTIVE; qs->qs_inflight = 0; qs->qs_cipher_key = csp->csp_cipher_key; qs->qs_cipher_klen = csp->csp_cipher_klen; qs->qs_auth_key = csp->csp_auth_key; qs->qs_auth_klen = csp->csp_auth_klen; switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CBC_MODE; break; case CRYPTO_AES_ICM: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; break; case CRYPTO_AES_XTS: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen / 2); qs->qs_cipher_mode = HW_CIPHER_XTS_MODE; break; case CRYPTO_AES_NIST_GCM_16: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; break; case 0: break; default: panic("%s: unhandled cipher algorithm %d", __func__, csp->csp_cipher_alg); } switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA1: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_256_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_256: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_384_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_384: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_512_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_512: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_AES_NIST_GMAC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; qs->qs_cipher_key = qs->qs_auth_key; qs->qs_cipher_klen = qs->qs_auth_klen; break; case 0: break; default: panic("%s: unhandled auth algorithm %d", __func__, csp->csp_auth_alg); } slices = 0; switch (csp->csp_mode) { case CSP_MODE_AEAD: case CSP_MODE_ETA: /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; break; case CSP_MODE_CIPHER: /* decrypt */ ddesc->qcd_slices[0] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER; /* encrypt */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER; slices = 1; break; case CSP_MODE_DIGEST: if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; } else { ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_cmd_id = FW_LA_CMD_AUTH; edesc->qcd_slices[0] = FW_SLICE_AUTH; edesc->qcd_cmd_id = FW_LA_CMD_AUTH; slices = 1; } break; default: panic("%s: unhandled crypto algorithm %d, %d", __func__, csp->csp_cipher_alg, csp->csp_auth_alg); } ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc); qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc); if (csp->csp_auth_mlen != 0) qs->qs_auth_mlen = csp->csp_auth_mlen; else qs->qs_auth_mlen = edesc->qcd_auth_sz; /* Compute the GMAC by specifying a null cipher payload. */ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH; return 0; } static void qat_crypto_clear_desc(struct qat_crypto_desc *desc) { explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc)); explicit_bzero(desc->qcd_hash_state_prefix_buf, sizeof(desc->qcd_hash_state_prefix_buf)); explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache)); } static void qat_freesession(device_t dev, crypto_session_t cses) { struct qat_session *qs; qs = crypto_get_driver_session(cses); KASSERT(qs->qs_inflight == 0, ("%s: session %p has requests in flight", __func__, qs)); qat_crypto_clear_desc(qs->qs_enc_desc); qat_crypto_clear_desc(qs->qs_dec_desc); qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem); mtx_destroy(&qs->qs_session_mtx); } static int qat_process(device_t dev, struct cryptop *crp, int hint) { struct qat_crypto *qcy; struct qat_crypto_bank *qcb; struct qat_crypto_desc const *desc; struct qat_session *qs; struct qat_softc *sc; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; int error; sc = device_get_softc(dev); qcy = &sc->sc_crypto; qs = crypto_get_driver_session(crp->crp_session); qsc = NULL; if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) { error = E2BIG; goto fail1; } mtx_lock(&qs->qs_session_mtx); if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) { error = E2BIG; mtx_unlock(&qs->qs_session_mtx); goto fail1; } /* * The firmware interface for GCM annoyingly requires the AAD * size to be stored in the session's content descriptor, which * is not really meant to be updated after session * initialization. For IPSec the AAD size is fixed so this is * not much of a problem in practice, but we have to catch AAD * size updates here so that the device code can safely update * the session's recorded AAD size. */ if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) { if (qs->qs_inflight == 0) { if (qs->qs_aad_length != -1) { counter_u64_add(sc->sc_gcm_aad_updates, 1); } qs->qs_aad_length = crp->crp_aad_length; } else { qs->qs_need_wakeup = true; mtx_unlock(&qs->qs_session_mtx); counter_u64_add(sc->sc_gcm_aad_restarts, 1); error = ERESTART; goto fail1; } } } qs->qs_inflight++; mtx_unlock(&qs->qs_session_mtx); qcb = qat_crypto_select_bank(qcy); qsc = qat_crypto_alloc_sym_cookie(qcb); if (qsc == NULL) { counter_u64_add(sc->sc_sym_alloc_failures, 1); error = ENOBUFS; goto fail2; } if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) desc = qs->qs_enc_desc; else desc = qs->qs_dec_desc; error = qat_crypto_load(qs, qsc, desc, crp); if (error != 0) goto fail2; qsbc = &qsc->qsc_bulk_cookie; qsbc->qsbc_crypto = qcy; qsbc->qsbc_session = qs; qsbc->qsbc_cb_tag = crp; sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp); if (crp->crp_aad != NULL) { bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); } bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); } bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); error = qat_etr_put_msg(sc, qcb->qcb_sym_tx, (uint32_t *)qsbc->qsbc_msg); if (error) goto fail2; return 0; fail2: if (qsc) qat_crypto_free_sym_cookie(qcb, qsc); mtx_lock(&qs->qs_session_mtx); qs->qs_inflight--; mtx_unlock(&qs->qs_session_mtx); fail1: crp->crp_etype = error; crypto_done(crp); return 0; } static device_method_t qat_methods[] = { /* Device interface */ DEVMETHOD(device_probe, qat_probe), DEVMETHOD(device_attach, qat_attach), DEVMETHOD(device_detach, qat_detach), /* Cryptodev interface */ DEVMETHOD(cryptodev_probesession, qat_probesession), DEVMETHOD(cryptodev_newsession, qat_newsession), DEVMETHOD(cryptodev_freesession, qat_freesession), DEVMETHOD(cryptodev_process, qat_process), DEVMETHOD_END }; static devclass_t qat_devclass; static driver_t qat_driver = { .name = "qat", .methods = qat_methods, .size = sizeof(struct qat_softc), }; DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0); MODULE_VERSION(qat, 1); MODULE_DEPEND(qat, crypto, 1, 1, 1); MODULE_DEPEND(qat, pci, 1, 1, 1); diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c index 242f1bcfb90a..16978632a1be 100644 --- a/sys/dev/safexcel/safexcel.c +++ b/sys/dev/safexcel/safexcel.c @@ -1,2618 +1,2615 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020, 2021 Rubicon Communications, LLC (Netgate) * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ararat River * Consulting, LLC under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "safexcel_reg.h" #include "safexcel_var.h" /* * We only support the EIP97 for now. */ static struct ofw_compat_data safexcel_compat[] = { { "inside-secure,safexcel-eip97ies", (uintptr_t)97 }, { "inside-secure,safexcel-eip97", (uintptr_t)97 }, { NULL, 0 } }; const struct safexcel_reg_offsets eip97_regs_offset = { .hia_aic = SAFEXCEL_EIP97_HIA_AIC_BASE, .hia_aic_g = SAFEXCEL_EIP97_HIA_AIC_G_BASE, .hia_aic_r = SAFEXCEL_EIP97_HIA_AIC_R_BASE, .hia_aic_xdr = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE, .hia_dfe = SAFEXCEL_EIP97_HIA_DFE_BASE, .hia_dfe_thr = SAFEXCEL_EIP97_HIA_DFE_THR_BASE, .hia_dse = SAFEXCEL_EIP97_HIA_DSE_BASE, .hia_dse_thr = SAFEXCEL_EIP97_HIA_DSE_THR_BASE, .hia_gen_cfg = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE, .pe = SAFEXCEL_EIP97_PE_BASE, }; const struct safexcel_reg_offsets eip197_regs_offset = { .hia_aic = SAFEXCEL_EIP197_HIA_AIC_BASE, .hia_aic_g = SAFEXCEL_EIP197_HIA_AIC_G_BASE, .hia_aic_r = SAFEXCEL_EIP197_HIA_AIC_R_BASE, .hia_aic_xdr = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE, .hia_dfe = SAFEXCEL_EIP197_HIA_DFE_BASE, .hia_dfe_thr = SAFEXCEL_EIP197_HIA_DFE_THR_BASE, .hia_dse = SAFEXCEL_EIP197_HIA_DSE_BASE, .hia_dse_thr = SAFEXCEL_EIP197_HIA_DSE_THR_BASE, .hia_gen_cfg = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE, .pe = SAFEXCEL_EIP197_PE_BASE, }; static struct safexcel_request * safexcel_next_request(struct safexcel_ring *ring) { int i; i = ring->cdr.read; KASSERT(i >= 0 && i < SAFEXCEL_RING_SIZE, ("%s: out of bounds request index %d", __func__, i)); return (&ring->requests[i]); } static struct safexcel_cmd_descr * safexcel_cmd_descr_next(struct safexcel_cmd_descr_ring *ring) { struct safexcel_cmd_descr *cdesc; if (ring->write == ring->read) return (NULL); cdesc = &ring->desc[ring->read]; ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; return (cdesc); } static struct safexcel_res_descr * safexcel_res_descr_next(struct safexcel_res_descr_ring *ring) { struct safexcel_res_descr *rdesc; if (ring->write == ring->read) return (NULL); rdesc = &ring->desc[ring->read]; ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; return (rdesc); } static struct safexcel_request * safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring) { int i; mtx_assert(&ring->mtx, MA_OWNED); i = ring->cdr.write; if ((i + 1) % SAFEXCEL_RING_SIZE == ring->cdr.read) return (NULL); return (&ring->requests[i]); } static void safexcel_free_request(struct safexcel_ring *ring, struct safexcel_request *req) { struct safexcel_context_record *ctx; mtx_assert(&ring->mtx, MA_OWNED); if (req->dmap_loaded) { bus_dmamap_unload(ring->data_dtag, req->dmap); req->dmap_loaded = false; } ctx = (struct safexcel_context_record *)req->ctx.vaddr; explicit_bzero(ctx->data, sizeof(ctx->data)); explicit_bzero(req->iv, sizeof(req->iv)); } static void safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) { TAILQ_HEAD(, cryptop) cq; struct cryptop *crp, *tmp; struct safexcel_cmd_descr *cdesc; struct safexcel_res_descr *rdesc; struct safexcel_request *req; struct safexcel_ring *ring; uint32_t blocked, error, i, nrdescs, nreqs; blocked = 0; ring = &sc->sc_ring[ringidx]; nreqs = SAFEXCEL_READ(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT); nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET; nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK; if (nreqs == 0) { SAFEXCEL_DPRINTF(sc, 1, "zero pending requests on ring %d\n", ringidx); mtx_lock(&ring->mtx); goto out; } TAILQ_INIT(&cq); ring = &sc->sc_ring[ringidx]; bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); nrdescs = 0; for (i = 0; i < nreqs; i++) { req = safexcel_next_request(ring); bus_dmamap_sync(req->ctx.tag, req->ctx.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->data_dtag, req->dmap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (req->cdescs-- > 0) { cdesc = safexcel_cmd_descr_next(&ring->cdr); KASSERT(cdesc != NULL, ("%s: missing control descriptor", __func__)); if (req->cdescs == 0) KASSERT(cdesc->last_seg, ("%s: chain is not terminated", __func__)); } nrdescs += req->rdescs; while (req->rdescs-- > 0) { rdesc = safexcel_res_descr_next(&ring->rdr); error = rdesc->result_data.error_code; if (error != 0) { if (error == SAFEXCEL_RESULT_ERR_AUTH_FAILED && req->crp->crp_etype == 0) { req->crp->crp_etype = EBADMSG; } else { SAFEXCEL_DPRINTF(sc, 1, "error code %#x\n", error); req->crp->crp_etype = EIO; } } } TAILQ_INSERT_TAIL(&cq, req->crp, crp_next); } mtx_lock(&ring->mtx); if (nreqs != 0) { KASSERT(ring->queued >= nreqs, ("%s: request count underflow, %d queued %d completed", __func__, ring->queued, nreqs)); ring->queued -= nreqs; SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); blocked = ring->blocked; ring->blocked = 0; } out: if (ring->queued != 0) { SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | imin(ring->queued, 16)); } mtx_unlock(&ring->mtx); if (blocked) crypto_unblock(sc->sc_cid, blocked); TAILQ_FOREACH_SAFE(crp, &cq, crp_next, tmp) crypto_done(crp); } static void safexcel_ring_intr(void *arg) { struct safexcel_softc *sc; struct safexcel_intr_handle *ih; uint32_t status, stat; int ring; bool rdrpending; ih = arg; sc = ih->sc; ring = ih->ring; status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring)); /* CDR interrupts */ if (status & SAFEXCEL_CDR_IRQ(ring)) { stat = SAFEXCEL_READ(sc, SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, stat & SAFEXCEL_CDR_INTR_MASK); } /* RDR interrupts */ rdrpending = false; if (status & SAFEXCEL_RDR_IRQ(ring)) { stat = SAFEXCEL_READ(sc, SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); if ((stat & SAFEXCEL_xDR_ERR) == 0) rdrpending = true; SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, stat & SAFEXCEL_RDR_INTR_MASK); } SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring), status); if (rdrpending) safexcel_rdr_intr(sc, ring); } static int safexcel_configure(struct safexcel_softc *sc) { uint32_t i, mask, pemask, reg; if (sc->sc_type == 197) { sc->sc_offsets = eip197_regs_offset; pemask = SAFEXCEL_N_PES_MASK; } else { sc->sc_offsets = eip97_regs_offset; pemask = EIP97_N_PES_MASK; } /* Scan for valid ring interrupt controllers. */ for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) { reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_VERSION(i)); if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE) break; } sc->sc_config.aic_rings = i; if (sc->sc_config.aic_rings == 0) return (-1); reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS); /* Check for 64bit addressing. */ if ((reg & SAFEXCEL_OPT_ADDR_64) == 0) return (-1); /* Check alignment constraints (which we do not support). */ if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >> SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0) return (-1); sc->sc_config.hdw = (reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET; mask = (1 << sc->sc_config.hdw) - 1; sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; /* Limit the number of rings to the number of the AIC Rings. */ sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; sc->sc_config.cd_size = sizeof(struct safexcel_cmd_descr) / sizeof(uint32_t); sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; sc->sc_config.rd_size = sizeof(struct safexcel_res_descr) / sizeof(uint32_t); sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; sc->sc_config.atok_offset = (SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) & ~mask; return (0); } static void safexcel_init_hia_bus_access(struct safexcel_softc *sc) { uint32_t version, val; /* Determine endianness and configure byte swap. */ version = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION); val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) { val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL, val); } /* Configure wr/rd cache values. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL, SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS)); } static void safexcel_disable_global_interrupts(struct safexcel_softc *sc) { /* Disable and clear pending interrupts. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, SAFEXCEL_AIC_G_ACK_ALL_MASK); } /* * Configure the data fetch engine. This component parses command descriptors * and sets up DMA transfers from host memory to the corresponding processing * engine. */ static void safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe) { /* Reset all DFE threads. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_RESET_PE); /* Deassert the DFE reset. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0); /* DMA transfer size to use. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe), SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG | SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) | SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) | SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) | SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) | SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS)); /* Configure the PE DMA transfer thresholds. */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe), SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | SAFEXCEL_PE_IN_xBUF_THRES_MAX(9)); SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe), SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | SAFEXCEL_PE_IN_xBUF_THRES_MAX(7)); } /* * Configure the data store engine. This component parses result descriptors * and sets up DMA transfers from the processing engine to host memory. */ static int safexcel_configure_dse(struct safexcel_softc *sc, int pe) { uint32_t val; int count; /* Disable and reset all DSE threads. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_RESET_PE); /* Wait for a second for threads to go idle. */ for (count = 0;;) { val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe)); if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) == SAFEXCEL_DSE_THR_RDR_ID_MASK) break; if (count++ > 10000) { device_printf(sc->sc_dev, "DSE reset timeout\n"); return (-1); } DELAY(100); } /* Exit the reset state. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0); /* DMA transfer size to use */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe), SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG | SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) | SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) | SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) | SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE); /* Configure the procesing engine thresholds */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe), SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) | SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8)); return (0); } static void safexcel_hw_prepare_rings(struct safexcel_softc *sc) { int i; for (i = 0; i < sc->sc_config.rings; i++) { /* * Command descriptors. */ /* Clear interrupts for this ring. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK); /* Disable external triggering. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * sizeof(uint32_t)); /* * Result descriptors. */ /* Disable external triggering. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); /* Ring size. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * sizeof(uint32_t)); } } static void safexcel_hw_setup_rings(struct safexcel_softc *sc) { struct safexcel_ring *ring; uint32_t cd_size_rnd, mask, rd_size_rnd, val; int i; mask = (1 << sc->sc_config.hdw) - 1; cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; val = (sizeof(struct safexcel_res_descr) - sizeof(struct safexcel_res_data)) / sizeof(uint32_t); rd_size_rnd = (val + mask) >> sc->sc_config.hdw; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; /* * Command descriptors. */ /* Ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP | (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | sc->sc_config.cd_size); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << SAFEXCEL_xDR_xD_FETCH_THRESH) | (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); /* Configure DMA tx control. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS)); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_CDR_INTR_MASK); /* * Result descriptors. */ /* Ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, SAFEXCEL_xDR_DESC_MODE_64BIT | (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | sc->sc_config.rd_size); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << SAFEXCEL_xDR_xD_FETCH_THRESH) | (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); /* Configure DMA tx control. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) | SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_RDR_INTR_MASK); /* Enable ring interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i), SAFEXCEL_RDR_IRQ(i)); } } /* Reset the command and result descriptor rings. */ static void safexcel_hw_reset_rings(struct safexcel_softc *sc) { int i; for (i = 0; i < sc->sc_config.rings; i++) { /* * Result descriptor ring operations. */ /* Reset ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_RDR_INTR_MASK); /* Disable ring interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), SAFEXCEL_RDR_IRQ(i)); /* * Command descriptor ring operations. */ /* Reset ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_CDR_INTR_MASK); } } static void safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe) { int i, ring_mask; for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { ring_mask <<= 1; ring_mask |= 1; } /* Enable command descriptor rings. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); /* Enable result descriptor rings. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); /* Clear any HIA interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, SAFEXCEL_AIC_G_ACK_HIA_MASK); } static void safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, struct safexcel_request *req, int hint) { int ringidx, ncdesc, nrdesc; bool busy; mtx_assert(&ring->mtx, MA_OWNED); if ((hint & CRYPTO_HINT_MORE) != 0) { ring->pending++; ring->pending_cdesc += req->cdescs; ring->pending_rdesc += req->rdescs; return; } ringidx = req->ringidx; busy = ring->queued != 0; ncdesc = ring->pending_cdesc + req->cdescs; nrdesc = ring->pending_rdesc + req->rdescs; ring->queued += ring->pending + 1; if (!busy) { SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | ring->queued); } SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, nrdesc * sc->sc_config.rd_offset * sizeof(uint32_t)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, ncdesc * sc->sc_config.cd_offset * sizeof(uint32_t)); ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; } static void safexcel_init_rings(struct safexcel_softc *sc) { struct safexcel_cmd_descr *cdesc; struct safexcel_ring *ring; uint64_t atok; int i, j; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; snprintf(ring->lockname, sizeof(ring->lockname), "safexcel_ring%d", i); mtx_init(&ring->mtx, ring->lockname, NULL, MTX_DEF); ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; ring->queued = 0; ring->cdr.read = ring->cdr.write = 0; ring->rdr.read = ring->rdr.write = 0; for (j = 0; j < SAFEXCEL_RING_SIZE; j++) { cdesc = &ring->cdr.desc[j]; atok = ring->dma_atok.paddr + sc->sc_config.atok_offset * j; cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); } } } static void safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct safexcel_dma_mem *sdm; if (error != 0) return; KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); sdm = arg; sdm->paddr = segs->ds_addr; } static int safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm, bus_size_t size) { int error; KASSERT(sdm->vaddr == NULL, ("%s: DMA memory descriptor in use.", __func__)); error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ PAGE_SIZE, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ size, 1, /* maxsize, nsegments */ size, BUS_DMA_COHERENT, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sdm->tag); /* dmat */ if (error != 0) { device_printf(sc->sc_dev, "failed to allocate busdma tag, error %d\n", error); goto err1; } error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate DMA safe memory, error %d\n", error); goto err2; } error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "cannot get address of the DMA memory, error %d\n", error); goto err3; } return (0); err3: bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); err2: bus_dma_tag_destroy(sdm->tag); err1: sdm->vaddr = NULL; return (error); } static void safexcel_dma_free_mem(struct safexcel_dma_mem *sdm) { bus_dmamap_unload(sdm->tag, sdm->map); bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); bus_dma_tag_destroy(sdm->tag); } static void safexcel_dma_free_rings(struct safexcel_softc *sc) { struct safexcel_ring *ring; int i; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; safexcel_dma_free_mem(&ring->cdr.dma); safexcel_dma_free_mem(&ring->dma_atok); safexcel_dma_free_mem(&ring->rdr.dma); bus_dma_tag_destroy(ring->data_dtag); mtx_destroy(&ring->mtx); } } static int safexcel_dma_init(struct safexcel_softc *sc) { struct safexcel_ring *ring; bus_size_t size; int error, i; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; error = bus_dma_tag_create( bus_get_dma_tag(sc->sc_dev),/* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ SAFEXCEL_MAX_REQUEST_SIZE, /* maxsize */ SAFEXCEL_MAX_FRAGMENTS, /* nsegments */ SAFEXCEL_MAX_REQUEST_SIZE, /* maxsegsz */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &ring->data_dtag); /* dmat */ if (error != 0) { device_printf(sc->sc_dev, "bus_dma_tag_create main failed; error %d\n", error); return (error); } size = sizeof(uint32_t) * sc->sc_config.cd_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate CDR DMA memory, error %d\n", error); goto err; } ring->cdr.desc = (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; /* Allocate additional CDR token memory. */ size = (bus_size_t)sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate atoken DMA memory, error %d\n", error); goto err; } size = sizeof(uint32_t) * sc->sc_config.rd_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); if (error) { device_printf(sc->sc_dev, "failed to allocate RDR DMA memory, error %d\n", error); goto err; } ring->rdr.desc = (struct safexcel_res_descr *)ring->rdr.dma.vaddr; } return (0); err: safexcel_dma_free_rings(sc); return (error); } static void safexcel_deinit_hw(struct safexcel_softc *sc) { safexcel_hw_reset_rings(sc); safexcel_dma_free_rings(sc); } static int safexcel_init_hw(struct safexcel_softc *sc) { int pe; /* 23.3.7 Initialization */ if (safexcel_configure(sc) != 0) return (EINVAL); if (safexcel_dma_init(sc) != 0) return (ENOMEM); safexcel_init_rings(sc); safexcel_init_hia_bus_access(sc); /* 23.3.7.2 Disable EIP-97 global Interrupts */ safexcel_disable_global_interrupts(sc); for (pe = 0; pe < sc->sc_config.pes; pe++) { /* 23.3.7.3 Configure Data Fetch Engine */ safexcel_configure_dfe_engine(sc, pe); /* 23.3.7.4 Configure Data Store Engine */ if (safexcel_configure_dse(sc, pe)) { safexcel_deinit_hw(sc); return (-1); } /* 23.3.7.5 1. Protocol enables */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe), 0xffffffff); SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe), 0xffffffff); } safexcel_hw_prepare_rings(sc); /* 23.3.7.5 Configure the Processing Engine(s). */ for (pe = 0; pe < sc->sc_config.pes; pe++) safexcel_enable_pe_engine(sc, pe); safexcel_hw_setup_rings(sc); return (0); } static int safexcel_setup_dev_interrupts(struct safexcel_softc *sc) { int error, i, j; for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { sc->sc_ih[i].sc = sc; sc->sc_ih[i].ring = i; if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr, &sc->sc_ih[i], &sc->sc_ih[i].handle)) { device_printf(sc->sc_dev, "couldn't setup interrupt %d\n", i); goto err; } error = bus_bind_intr(sc->sc_dev, sc->sc_intr[i], i % mp_ncpus); if (error != 0) device_printf(sc->sc_dev, "failed to bind ring %d\n", error); } return (0); err: for (j = 0; j < i; j++) bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], sc->sc_ih[j].handle); return (ENXIO); } static void safexcel_teardown_dev_interrupts(struct safexcel_softc *sc) { int i; for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], sc->sc_ih[i].handle); } static int safexcel_alloc_dev_resources(struct safexcel_softc *sc) { char name[16]; device_t dev; phandle_t node; int error, i, rid; dev = sc->sc_dev; node = ofw_bus_get_node(dev); rid = 0; sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_res == NULL) { device_printf(dev, "couldn't allocate memory resources\n"); return (ENXIO); } for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) { (void)snprintf(name, sizeof(name), "ring%d", i); error = ofw_bus_find_string_index(node, "interrupt-names", name, &rid); if (error != 0) break; sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_intr[i] == NULL) { error = ENXIO; goto out; } } if (i == 0) { device_printf(dev, "couldn't allocate interrupt resources\n"); error = ENXIO; goto out; } return (0); out: for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), sc->sc_res); return (error); } static void safexcel_free_dev_resources(struct safexcel_softc *sc) { int i; for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) bus_release_resource(sc->sc_dev, SYS_RES_IRQ, rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); if (sc->sc_res != NULL) bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), sc->sc_res); } static int safexcel_probe(device_t dev) { struct safexcel_softc *sc; if (!ofw_bus_status_okay(dev)) return (ENXIO); sc = device_get_softc(dev); sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; if (sc->sc_type == 0) return (ENXIO); device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); return (BUS_PROBE_DEFAULT); } static int safexcel_attach(device_t dev) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; struct safexcel_softc *sc; struct safexcel_request *req; struct safexcel_ring *ring; int i, j, ringidx; sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_cid = -1; if (safexcel_alloc_dev_resources(sc)) goto err; if (safexcel_setup_dev_interrupts(sc)) goto err1; if (safexcel_init_hw(sc)) goto err2; for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { ring = &sc->sc_ring[ringidx]; ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); for (i = 0; i < SAFEXCEL_RING_SIZE; i++) { req = &ring->requests[i]; req->sc = sc; req->ringidx = ringidx; if (bus_dmamap_create(ring->data_dtag, BUS_DMA_COHERENT, &req->dmap) != 0) { for (j = 0; j < i; j++) bus_dmamap_destroy(ring->data_dtag, ring->requests[j].dmap); goto err2; } if (safexcel_dma_alloc_mem(sc, &req->ctx, sizeof(struct safexcel_context_record)) != 0) { for (j = 0; j < i; j++) { bus_dmamap_destroy(ring->data_dtag, ring->requests[j].dmap); safexcel_dma_free_mem( &ring->requests[j].ctx); } goto err2; } } } ctx = device_get_sysctl_ctx(dev); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, "Debug message verbosity"); oid = device_get_sysctl_tree(sc->sc_dev); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); sc->sc_req_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "req_alloc_failures", CTLFLAG_RD, &sc->sc_req_alloc_failures, "Number of request allocation failures"); sc->sc_cdesc_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cdesc_alloc_failures", CTLFLAG_RD, &sc->sc_cdesc_alloc_failures, "Number of command descriptor ring overflows"); sc->sc_rdesc_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "rdesc_alloc_failures", CTLFLAG_RD, &sc->sc_rdesc_alloc_failures, "Number of result descriptor ring overflows"); sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), CRYPTOCAP_F_HARDWARE); if (sc->sc_cid < 0) goto err2; return (0); err2: safexcel_teardown_dev_interrupts(sc); err1: safexcel_free_dev_resources(sc); err: return (ENXIO); } static int safexcel_detach(device_t dev) { struct safexcel_ring *ring; struct safexcel_softc *sc; int i, ringidx; sc = device_get_softc(dev); if (sc->sc_cid >= 0) crypto_unregister_all(sc->sc_cid); counter_u64_free(sc->sc_req_alloc_failures); counter_u64_free(sc->sc_cdesc_alloc_failures); counter_u64_free(sc->sc_rdesc_alloc_failures); for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { ring = &sc->sc_ring[ringidx]; for (i = 0; i < SAFEXCEL_RING_SIZE; i++) { bus_dmamap_destroy(ring->data_dtag, ring->requests[i].dmap); safexcel_dma_free_mem(&ring->requests[i].ctx); } sglist_free(ring->cmd_data); sglist_free(ring->res_data); } safexcel_deinit_hw(sc); safexcel_teardown_dev_interrupts(sc); safexcel_free_dev_resources(sc); return (0); } /* * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted * using the cipher key. */ static void safexcel_setkey_ghash(const uint8_t *key, int klen, uint32_t *hashkey) { uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; uint8_t zeros[AES_BLOCK_LEN]; int i, rounds; memset(zeros, 0, sizeof(zeros)); rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)hashkey); for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) hashkey[i] = htobe32(hashkey[i]); explicit_bzero(ks, sizeof(ks)); } /* * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 * in the hardware implementation. K1 is the cipher key and comes last in the * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC * is not implemented so K2 and K3 are fixed. */ static void safexcel_setkey_xcbcmac(const uint8_t *key, int klen, uint32_t *hashkey) { int i, off; memset(hashkey, 0, 2 * AES_BLOCK_LEN); off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) hashkey[i + off] = htobe32(le32dec(key)); } static void safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, char *buf) { int hashwords, i; switch (ahash->type) { case CRYPTO_SHA1_HMAC: hashwords = ahash->hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); break; case CRYPTO_SHA2_224_HMAC: hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); break; case CRYPTO_SHA2_256_HMAC: hashwords = ahash->hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); break; case CRYPTO_SHA2_384_HMAC: hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); for (i = 0; i < hashwords; i++) ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); break; case CRYPTO_SHA2_512_HMAC: hashwords = ahash->hashsize / sizeof(uint64_t); for (i = 0; i < hashwords; i++) ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); break; } } /* * Pre-compute the inner and outer digests used in the HMAC algorithm. */ static void safexcel_setkey_hmac(const struct crypto_session_params *csp, const uint8_t *key, int klen, uint8_t *ipad, uint8_t *opad) { union authctx ctx; struct auth_hash *ahash; ahash = crypto_auth_hash(csp); hmac_init_ipad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, ipad); hmac_init_opad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, opad); explicit_bzero(&ctx, ahash->ctxsize); } static void safexcel_setkey_xts(const uint8_t *key, int klen, uint8_t *tweakkey) { memcpy(tweakkey, key + klen, klen); } /* * Populate a context record with paramters from a session. Some consumers * specify per-request keys, in which case the context must be re-initialized * for each request. */ static int safexcel_set_context(struct safexcel_context_record *ctx, int op, const uint8_t *ckey, const uint8_t *akey, struct safexcel_session *sess) { const struct crypto_session_params *csp; uint8_t *data; uint32_t ctrl0, ctrl1; int aklen, alg, cklen, off; csp = crypto_get_params(sess->cses); aklen = csp->csp_auth_klen; cklen = csp->csp_cipher_klen; if (csp->csp_cipher_alg == CRYPTO_AES_XTS) cklen /= 2; ctrl0 = sess->alg | sess->digest | sess->hash; ctrl1 = sess->mode; data = (uint8_t *)ctx->data; if (csp->csp_cipher_alg != 0) { memcpy(data, ckey, cklen); off = cklen; } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { memcpy(data, akey, aklen); off = aklen; } else { off = 0; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: safexcel_setkey_ghash(ckey, cklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_AES_CCM_16: safexcel_setkey_xcbcmac(ckey, cklen, (uint32_t *)(data + off)); off += AES_BLOCK_LEN * 2 + cklen; break; case CRYPTO_AES_XTS: safexcel_setkey_xts(ckey, cklen, data + off); off += cklen; break; } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: safexcel_setkey_ghash(akey, aklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: safexcel_setkey_hmac(csp, akey, aklen, data + off, data + off + sess->statelen); off += sess->statelen * 2; break; } ctrl0 |= SAFEXCEL_CONTROL0_SIZE(off / sizeof(uint32_t)); alg = csp->csp_cipher_alg; if (alg == 0) alg = csp->csp_auth_alg; switch (alg) { case CRYPTO_AES_CCM_16: if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | SAFEXCEL_CONTROL0_KEY_EN; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN | SAFEXCEL_CONTROL0_KEY_EN; } ctrl1 |= SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | SAFEXCEL_CONTROL1_IV2 | SAFEXCEL_CONTROL1_IV3; break; case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN; if (csp->csp_auth_alg != 0) ctrl0 |= SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | SAFEXCEL_CONTROL0_KEY_EN; if (csp->csp_auth_alg != 0) ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; } break; case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_NIST_GMAC: if (CRYPTO_OP_IS_ENCRYPT(op) || csp->csp_auth_alg != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_OUT; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; } if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE | SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | SAFEXCEL_CONTROL1_IV2; } break; case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: ctrl0 |= SAFEXCEL_CONTROL0_RESTART_HASH; /* FALLTHROUGH */ case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_OUT; break; } ctx->control0 = ctrl0; ctx->control1 = ctrl1; return (off); } /* * Construct a no-op instruction, used to pad input tokens. */ static void safexcel_instr_nop(struct safexcel_instr **instrp) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = (1 << 2); instr->status = 0; instr->instructions = 0; *instrp = instr + 1; } /* * Insert the digest of the input payload. This is typically the last * instruction of a sequence. */ static void safexcel_instr_insert_digest(struct safexcel_instr **instrp, int len) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_INSERT_HASH_DIGEST; *instrp = instr + 1; } /* * Retrieve and verify a digest. */ static void safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE; instr->length = len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS; instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING; *instrp = instr + 1; } static void safexcel_instr_temp_aes_block(struct safexcel_instr **instrp) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT; instr->length = 0; instr->status = 0; instr->instructions = AES_BLOCK_LEN; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = AES_BLOCK_LEN; instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO; *instrp = instr + 1; } /* * Handle a request for an unauthenticated block cipher. */ static void safexcel_instr_cipher(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; crp = req->crp; /* Insert the payload. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT; cdesc->additional_cdata_size = 1; } static void safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; crp = req->crp; start = instr; /* Insert the AAD. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; /* Encrypt any data left in the request. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_DEST_OUTPUT; instr++; } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_sha_hash(struct safexcel_request *req, struct safexcel_instr *instr) { struct cryptop *crp; struct safexcel_instr *start; crp = req->crp; start = instr; /* Pass the input data to the hash engine. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; /* Insert the hash result into the output stream. */ safexcel_instr_insert_digest(&instr, req->sess->digestlen); /* Pad the rest of the inline instruction space. */ while (instr != start + SAFEXCEL_MAX_ITOKENS) safexcel_instr_nop(&instr); } static void safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_instr *start; uint8_t *a0, *b0, *alenp, L; int aalign, blen; crp = req->crp; csp = crypto_get_params(crp->crp_session); start = instr; /* * Construct two blocks, A0 and B0, used in encryption and * authentication, respectively. A0 is embedded in the token * descriptor, and B0 is inserted directly into the data stream using * instructions below. * * An explicit check for overflow of the length field is not * needed since the maximum driver size of 65535 bytes fits in * the smallest length field used for a 13-byte nonce. */ blen = AES_BLOCK_LEN; L = 15 - csp->csp_ivlen; a0 = (uint8_t *)&cdesc->control_data.token[0]; memset(a0, 0, blen); a0[0] = L - 1; memcpy(&a0[1], req->iv, csp->csp_ivlen); /* * Insert B0 and the AAD length into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INSERT_IMMEDIATE; instr++; b0 = (uint8_t *)instr; memset(b0, 0, blen); b0[0] = (L - 1) | /* payload length size */ ((req->sess->digestlen - 2) / 2) << 3 /* digest length */ | (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; memcpy(&b0[1], req->iv, csp->csp_ivlen); b0[14] = crp->crp_payload_length >> 8; b0[15] = crp->crp_payload_length & 0xff; instr += blen / sizeof(*instr); /* Insert the AAD length and data into the input stream. */ if (crp->crp_aad_length > 0) { alenp = (uint8_t *)instr; alenp[0] = crp->crp_aad_length >> 8; alenp[1] = crp->crp_aad_length & 0xff; alenp[2] = 0; alenp[3] = 0; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; /* Insert zero padding. */ aalign = (crp->crp_aad_length + 2) & (blen - 1); instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = aalign == 0 ? 0 : blen - ((crp->crp_aad_length + 2) & (blen - 1)); instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; } safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INS_LAST; instr++; /* Insert zero padding. */ if (crp->crp_payload_length & (blen - 1)) { instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = blen - (crp->crp_payload_length & (blen - 1)); instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; } } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); cdesc->control_data.token[3] = htobe32(1); crp = req->crp; start = instr; /* Insert the AAD into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INS_LAST; instr++; } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); cdesc->control_data.token[3] = htobe32(1); crp = req->crp; start = instr; instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; safexcel_instr_temp_aes_block(&instr); safexcel_instr_insert_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_set_token(struct safexcel_request *req) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_cmd_descr *cdesc; struct safexcel_context_record *ctx; struct safexcel_context_template *ctxtmp; struct safexcel_instr *instr; struct safexcel_softc *sc; const uint8_t *akey, *ckey; int ringidx; crp = req->crp; csp = crypto_get_params(crp->crp_session); cdesc = req->cdesc; sc = req->sc; ringidx = req->ringidx; akey = crp->crp_auth_key; ckey = crp->crp_cipher_key; if (akey != NULL || ckey != NULL) { /* * If we have a per-request key we have to generate the context * record on the fly. */ if (akey == NULL) akey = csp->csp_auth_key; if (ckey == NULL) ckey = csp->csp_cipher_key; ctx = (struct safexcel_context_record *)req->ctx.vaddr; (void)safexcel_set_context(ctx, crp->crp_op, ckey, akey, req->sess); } else { /* * Use the context record template computed at session * initialization time. */ ctxtmp = CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ? &req->sess->encctx : &req->sess->decctx; ctx = &ctxtmp->ctx; memcpy(req->ctx.vaddr + 2 * sizeof(uint32_t), ctx->data, ctxtmp->len); } cdesc->control_data.control0 = ctx->control0; cdesc->control_data.control1 = ctx->control1; /* * For keyless hash operations, the token instructions can be embedded * in the token itself. Otherwise we use an additional token descriptor * and the embedded instruction space is used to store the IV. */ if (csp->csp_cipher_alg == 0 && csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { instr = (void *)cdesc->control_data.token; } else { instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + sc->sc_config.atok_offset * (cdesc - sc->sc_ring[ringidx].cdr.desc)); cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: safexcel_instr_gcm(req, instr, cdesc); break; case CRYPTO_AES_CCM_16: safexcel_instr_ccm(req, instr, cdesc); break; case CRYPTO_AES_XTS: memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); memset(cdesc->control_data.token + AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); safexcel_instr_cipher(req, instr, cdesc); break; case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); if (csp->csp_auth_alg != 0) safexcel_instr_eta(req, instr, cdesc); else safexcel_instr_cipher(req, instr, cdesc); break; default: switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: safexcel_instr_sha_hash(req, instr); break; case CRYPTO_AES_NIST_GMAC: safexcel_instr_gmac(req, instr, cdesc); break; default: panic("unhandled auth request %d", csp->csp_auth_alg); } break; } } static struct safexcel_res_descr * safexcel_res_descr_add(struct safexcel_ring *ring, bool first, bool last, bus_addr_t data, uint32_t len) { struct safexcel_res_descr *rdesc; struct safexcel_res_descr_ring *rring; mtx_assert(&ring->mtx, MA_OWNED); rring = &ring->rdr; if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) return (NULL); rdesc = &rring->desc[rring->write]; rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; rdesc->particle_size = len; rdesc->rsvd0 = 0; rdesc->descriptor_overflow = 0; rdesc->buffer_overflow = 0; rdesc->last_seg = last; rdesc->first_seg = first; rdesc->result_size = sizeof(struct safexcel_res_data) / sizeof(uint32_t); rdesc->rsvd1 = 0; rdesc->data_lo = SAFEXCEL_ADDR_LO(data); rdesc->data_hi = SAFEXCEL_ADDR_HI(data); if (first) { rdesc->result_data.packet_length = 0; rdesc->result_data.error_code = 0; } return (rdesc); } static struct safexcel_cmd_descr * safexcel_cmd_descr_add(struct safexcel_ring *ring, bool first, bool last, bus_addr_t data, uint32_t seglen, uint32_t reqlen, bus_addr_t context) { struct safexcel_cmd_descr *cdesc; struct safexcel_cmd_descr_ring *cring; KASSERT(reqlen <= SAFEXCEL_MAX_REQUEST_SIZE, ("%s: request length %u too long", __func__, reqlen)); mtx_assert(&ring->mtx, MA_OWNED); cring = &ring->cdr; if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) return (NULL); cdesc = &cring->desc[cring->write]; cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; cdesc->particle_size = seglen; cdesc->rsvd0 = 0; cdesc->last_seg = last; cdesc->first_seg = first; cdesc->additional_cdata_size = 0; cdesc->rsvd1 = 0; cdesc->data_lo = SAFEXCEL_ADDR_LO(data); cdesc->data_hi = SAFEXCEL_ADDR_HI(data); if (first) { cdesc->control_data.packet_length = reqlen; cdesc->control_data.options = SAFEXCEL_OPTION_IP | SAFEXCEL_OPTION_CP | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD | SAFEXCEL_OPTION_RC_AUTO; cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | SAFEXCEL_CONTEXT_SMALL; cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); } return (cdesc); } static void safexcel_cmd_descr_rollback(struct safexcel_ring *ring, int count) { struct safexcel_cmd_descr_ring *cring; mtx_assert(&ring->mtx, MA_OWNED); cring = &ring->cdr; cring->write -= count; if (cring->write < 0) cring->write += SAFEXCEL_RING_SIZE; } static void safexcel_res_descr_rollback(struct safexcel_ring *ring, int count) { struct safexcel_res_descr_ring *rring; mtx_assert(&ring->mtx, MA_OWNED); rring = &ring->rdr; rring->write -= count; if (rring->write < 0) rring->write += SAFEXCEL_RING_SIZE; } static void safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg, int start, int len) { bus_dma_segment_t *seg; size_t seglen; int error, i; for (i = 0; i < nseg && len > 0; i++) { seg = &segs[i]; if (seg->ds_len <= start) { start -= seg->ds_len; continue; } seglen = MIN(len, seg->ds_len - start); error = sglist_append_phys(sg, seg->ds_addr + start, seglen); if (error != 0) panic("%s: ran out of segments: %d", __func__, error); len -= seglen; start = 0; } } static void safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_cmd_descr *cdesc; struct safexcel_request *req; struct safexcel_ring *ring; struct safexcel_session *sess; struct sglist *sg; size_t inlen; int i; bool first, last; req = arg; if (error != 0) { req->error = error; return; } crp = req->crp; csp = crypto_get_params(crp->crp_session); sess = req->sess; ring = &req->sc->sc_ring[req->ringidx]; mtx_assert(&ring->mtx, MA_OWNED); /* * Set up descriptors for input and output data. * * The processing engine programs require that any AAD comes first, * followed by the cipher plaintext, followed by the digest. Some * consumers place the digest first in the input buffer, in which case * we have to create an extra descriptor. * * As an optimization, unmodified data is not passed to the output * stream. */ sglist_reset(ring->cmd_data); sglist_reset(ring->res_data); if (crp->crp_aad_length != 0) { safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_aad_start, crp->crp_aad_length); } safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_payload_start, crp->crp_payload_length); if (csp->csp_cipher_alg != 0) { safexcel_append_segs(segs, nseg, ring->res_data, crp->crp_payload_start, crp->crp_payload_length); } if (sess->digestlen > 0) { if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_digest_start, sess->digestlen); } else { safexcel_append_segs(segs, nseg, ring->res_data, crp->crp_digest_start, sess->digestlen); } } sg = ring->cmd_data; if (sg->sg_nseg == 0) { /* * Fake a segment for the command descriptor if the input has * length zero. The EIP97 apparently does not handle * zero-length packets properly since subsequent requests return * bogus errors, so provide a dummy segment using the context * descriptor. Also, we must allocate at least one command ring * entry per request to keep the request shadow ring in sync. */ (void)sglist_append_phys(sg, req->ctx.paddr, 1); } for (i = 0, inlen = 0; i < sg->sg_nseg; i++) inlen += sg->sg_segs[i].ss_len; for (i = 0; i < sg->sg_nseg; i++) { first = i == 0; last = i == sg->sg_nseg - 1; cdesc = safexcel_cmd_descr_add(ring, first, last, sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, (uint32_t)inlen, req->ctx.paddr); if (cdesc == NULL) { safexcel_cmd_descr_rollback(ring, i); counter_u64_add(req->sc->sc_cdesc_alloc_failures, 1); req->error = ERESTART; return; } if (i == 0) req->cdesc = cdesc; } req->cdescs = sg->sg_nseg; sg = ring->res_data; if (sg->sg_nseg == 0) { /* * We need a result descriptor even if the output stream will be * empty, for example when verifying an AAD digest. */ sg->sg_segs[0].ss_paddr = 0; sg->sg_segs[0].ss_len = 0; sg->sg_nseg = 1; } for (i = 0; i < sg->sg_nseg; i++) { first = i == 0; last = i == sg->sg_nseg - 1; if (safexcel_res_descr_add(ring, first, last, sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { safexcel_cmd_descr_rollback(ring, ring->cmd_data->sg_nseg); safexcel_res_descr_rollback(ring, i); counter_u64_add(req->sc->sc_rdesc_alloc_failures, 1); req->error = ERESTART; return; } } req->rdescs = sg->sg_nseg; } static int safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req) { int error; req->error = 0; req->cdescs = req->rdescs = 0; error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); if (error == 0) req->dmap_loaded = true; if (req->error != 0) error = req->error; return (error); } static bool safexcel_probe_cipher(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (true); } /* * Determine whether the driver can implement a session with the requested * parameters. */ static int safexcel_probesession(device_t dev, const struct crypto_session_params *csp) { if (csp->csp_flags != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_CIPHER: if (!safexcel_probe_cipher(csp)) return (EINVAL); break; case CSP_MODE_DIGEST: switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); break; case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: break; default: return (EINVAL); } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); - break; case CRYPTO_AES_CCM_16: break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!safexcel_probe_cipher(csp)) return (EINVAL); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: /* * The EIP-97 does not support combining AES-XTS with * hash operations. */ if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) return (EINVAL); break; default: return (EINVAL); } break; default: return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } static uint32_t safexcel_aes_algid(int keylen) { switch (keylen) { case 16: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128); case 24: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192); case 32: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256); default: panic("invalid AES key length %d", keylen); } } static uint32_t safexcel_aes_ccm_hashid(int keylen) { switch (keylen) { case 16: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC128); case 24: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC192); case 32: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC256); default: panic("invalid AES key length %d", keylen); } } static uint32_t safexcel_sha_hashid(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA1); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA224); case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA256); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA384); case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA512); default: __assert_unreachable(); } } static int safexcel_sha_hashlen(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SHA1_HASH_LEN); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: return (SHA2_224_HASH_LEN); case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SHA2_256_HASH_LEN); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: return (SHA2_384_HASH_LEN); case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SHA2_512_HASH_LEN); default: __assert_unreachable(); } } static int safexcel_sha_statelen(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SHA1_HASH_LEN); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SHA2_256_HASH_LEN); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SHA2_512_HASH_LEN); default: __assert_unreachable(); } } static int safexcel_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct safexcel_session *sess; sess = crypto_get_driver_session(cses); sess->cses = cses; switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED; sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); break; case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC; sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); break; case CRYPTO_AES_NIST_GMAC: sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; sess->digestlen = GMAC_DIGEST_LEN; sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; sess->alg = safexcel_aes_algid(csp->csp_auth_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; break; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; sess->digestlen = GMAC_DIGEST_LEN; sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; break; case CRYPTO_AES_CCM_16: sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM; sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM; break; case CRYPTO_AES_CBC: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC; break; case CRYPTO_AES_ICM: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR; break; case CRYPTO_AES_XTS: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS; break; } if (csp->csp_auth_mlen != 0) sess->digestlen = csp->csp_auth_mlen; sess->encctx.len = safexcel_set_context(&sess->encctx.ctx, CRYPTO_OP_ENCRYPT, csp->csp_cipher_key, csp->csp_auth_key, sess); sess->decctx.len = safexcel_set_context(&sess->decctx.ctx, CRYPTO_OP_DECRYPT, csp->csp_cipher_key, csp->csp_auth_key, sess); return (0); } static int safexcel_process(device_t dev, struct cryptop *crp, int hint) { struct safexcel_request *req; struct safexcel_ring *ring; struct safexcel_session *sess; struct safexcel_softc *sc; int error; sc = device_get_softc(dev); sess = crypto_get_driver_session(crp->crp_session); if (__predict_false(crypto_buffer_len(&crp->crp_buf) > SAFEXCEL_MAX_REQUEST_SIZE)) { crp->crp_etype = E2BIG; crypto_done(crp); return (0); } ring = &sc->sc_ring[curcpu % sc->sc_config.rings]; mtx_lock(&ring->mtx); req = safexcel_alloc_request(sc, ring); if (__predict_false(req == NULL)) { ring->blocked = CRYPTO_SYMQ; mtx_unlock(&ring->mtx); counter_u64_add(sc->sc_req_alloc_failures, 1); return (ERESTART); } req->crp = crp; req->sess = sess; crypto_read_iv(crp, req->iv); error = safexcel_create_chain(ring, req); if (__predict_false(error != 0)) { safexcel_free_request(ring, req); if (error == ERESTART) ring->blocked = CRYPTO_SYMQ; mtx_unlock(&ring->mtx); if (error != ERESTART) { crp->crp_etype = error; crypto_done(crp); return (0); } else { return (ERESTART); } } safexcel_set_token(req); bus_dmamap_sync(ring->data_dtag, req->dmap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(req->ctx.tag, req->ctx.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); safexcel_execute(sc, ring, req, hint); mtx_unlock(&ring->mtx); return (0); } static device_method_t safexcel_methods[] = { /* Device interface */ DEVMETHOD(device_probe, safexcel_probe), DEVMETHOD(device_attach, safexcel_attach), DEVMETHOD(device_detach, safexcel_detach), /* Cryptodev interface */ DEVMETHOD(cryptodev_probesession, safexcel_probesession), DEVMETHOD(cryptodev_newsession, safexcel_newsession), DEVMETHOD(cryptodev_process, safexcel_process), DEVMETHOD_END }; static devclass_t safexcel_devclass; static driver_t safexcel_driver = { .name = "safexcel", .methods = safexcel_methods, .size = sizeof(struct safexcel_softc), }; DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0); MODULE_VERSION(safexcel, 1); MODULE_DEPEND(safexcel, crypto, 1, 1, 1); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index 576382406d88..5332ab5f07ce 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -1,2333 +1,2336 @@ /*- * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ararat River * Consulting, LLC under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Cryptographic Subsystem. * * This code is derived from the Openbsd Cryptographic Framework (OCF) * that has the copyright shown below. Very little of the original * code remains. */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000, 2001 Angelos D. Keromytis * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include "opt_compat.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) #include #endif SDT_PROVIDER_DEFINE(opencrypto); /* * Crypto drivers register themselves by allocating a slot in the * crypto_drivers table with crypto_get_driverid() and then registering * each asym algorithm they support with crypto_kregister(). */ static struct mtx crypto_drivers_mtx; /* lock on driver table */ #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED) /* * Crypto device/driver capabilities structure. * * Synchronization: * (d) - protected by CRYPTO_DRIVER_LOCK() * (q) - protected by CRYPTO_Q_LOCK() * Not tagged fields are read-only. */ struct cryptocap { device_t cc_dev; uint32_t cc_hid; uint32_t cc_sessions; /* (d) # of sessions */ uint32_t cc_koperations; /* (d) # os asym operations */ uint8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; int cc_flags; /* (d) flags */ #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ int cc_qblocked; /* (q) symmetric q blocked */ int cc_kqblocked; /* (q) asymmetric q blocked */ size_t cc_session_size; volatile int cc_refs; }; static struct cryptocap **crypto_drivers = NULL; static int crypto_drivers_size = 0; struct crypto_session { struct cryptocap *cap; struct crypto_session_params csp; uint64_t id; /* Driver softc follows. */ }; /* * There are two queues for crypto requests; one for symmetric (e.g. * cipher) operations and one for asymmetric (e.g. MOD)operations. * A single mutex is used to lock access to both queues. We could * have one per-queue but having one simplifies handling of block/unblock * operations. */ static int crp_sleep = 0; static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */ static TAILQ_HEAD(,cryptkop) crp_kq; static struct mtx crypto_q_mtx; #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0, "In-kernel cryptography"); /* * Taskqueue used to dispatch the crypto requests * that have the CRYPTO_F_ASYNC flag */ static struct taskqueue *crypto_tq; /* * Crypto seq numbers are operated on with modular arithmetic */ #define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0) struct crypto_ret_worker { struct mtx crypto_ret_mtx; TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */ TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */ TAILQ_HEAD(,cryptkop) crp_ret_kq; /* callback queue for asym jobs */ uint32_t reorder_ops; /* total ordered sym jobs received */ uint32_t reorder_cur_seq; /* current sym job dispatched */ struct thread *td; }; static struct crypto_ret_worker *crypto_ret_workers = NULL; #define CRYPTO_RETW(i) (&crypto_ret_workers[i]) #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers) #define FOREACH_CRYPTO_RETW(w) \ for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) #define CRYPTO_RETW_EMPTY(w) \ (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q)) static int crypto_workers_num = 0; SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, &crypto_workers_num, 0, "Number of crypto workers used to dispatch crypto jobs"); #ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, &crypto_workers_num, 0, "Number of crypto workers used to dispatch crypto jobs"); #endif static uma_zone_t cryptop_zone; int crypto_userasymcrypto = 1; SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW, &crypto_userasymcrypto, 0, "Enable user-mode access to asymmetric crypto support"); #ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, &crypto_userasymcrypto, 0, "Enable/disable user-mode access to asymmetric crypto support"); #endif int crypto_devallowsoft = 0; SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RWTUN, &crypto_devallowsoft, 0, "Enable use of software crypto by /dev/crypto"); #ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RWTUN, &crypto_devallowsoft, 0, "Enable/disable use of software crypto by /dev/crypto"); #endif MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); static void crypto_dispatch_thread(void *arg); static struct thread *cryptotd; static void crypto_ret_thread(void *arg); static void crypto_destroy(void); static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); static int crypto_kinvoke(struct cryptkop *krp); static void crypto_task_invoke(void *ctx, int pending); static void crypto_batch_enqueue(struct cryptop *crp); static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)]; SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, cryptostats, nitems(cryptostats), "Crypto system statistics"); #define CRYPTOSTAT_INC(stat) do { \ counter_u64_add( \ cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\ 1); \ } while (0) static void cryptostats_init(void *arg __unused) { COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK); } SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL); static void cryptostats_fini(void *arg __unused) { COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats)); } SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini, NULL); /* Try to avoid directly exposing the key buffer as a symbol */ static struct keybuf *keybuf; static struct keybuf empty_keybuf = { .kb_nents = 0 }; /* Obtain the key buffer from boot metadata */ static void keybuf_init(void) { caddr_t kmdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); keybuf = (struct keybuf *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_KEYBUF); if (keybuf == NULL) keybuf = &empty_keybuf; } /* It'd be nice if we could store these in some kind of secure memory... */ struct keybuf * get_keybuf(void) { return (keybuf); } static struct cryptocap * cap_ref(struct cryptocap *cap) { refcount_acquire(&cap->cc_refs); return (cap); } static void cap_rele(struct cryptocap *cap) { if (refcount_release(&cap->cc_refs) == 0) return; KASSERT(cap->cc_sessions == 0, ("freeing crypto driver with active sessions")); KASSERT(cap->cc_koperations == 0, ("freeing crypto driver with active key operations")); free(cap, M_CRYPTO_DATA); } static int crypto_init(void) { struct crypto_ret_worker *ret_worker; struct proc *p; int error; mtx_init(&crypto_drivers_mtx, "crypto driver table", NULL, MTX_DEF); TAILQ_INIT(&crp_q); TAILQ_INIT(&crp_kq); mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF); cryptop_zone = uma_zcreate("cryptop", sizeof(struct cryptop), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); crypto_drivers_size = CRYPTO_DRIVERS_INITIAL; crypto_drivers = malloc(crypto_drivers_size * sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO); if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus) crypto_workers_num = mp_ncpus; crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &crypto_tq); taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN, "crypto"); p = NULL; error = kproc_kthread_add(crypto_dispatch_thread, NULL, &p, &cryptotd, 0, 0, "crypto", "crypto"); if (error) { printf("crypto_init: cannot start crypto thread; error %d", error); goto bad; } crypto_ret_workers = mallocarray(crypto_workers_num, sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO); FOREACH_CRYPTO_RETW(ret_worker) { TAILQ_INIT(&ret_worker->crp_ordered_ret_q); TAILQ_INIT(&ret_worker->crp_ret_q); TAILQ_INIT(&ret_worker->crp_ret_kq); ret_worker->reorder_ops = 0; ret_worker->reorder_cur_seq = 0; mtx_init(&ret_worker->crypto_ret_mtx, "crypto return queues", NULL, MTX_DEF); error = kthread_add(crypto_ret_thread, ret_worker, p, &ret_worker->td, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker)); if (error) { printf("crypto_init: cannot start cryptoret thread; error %d", error); goto bad; } } keybuf_init(); return 0; bad: crypto_destroy(); return error; } /* * Signal a crypto thread to terminate. We use the driver * table lock to synchronize the sleep/wakeups so that we * are sure the threads have terminated before we release * the data structures they use. See crypto_finis below * for the other half of this song-and-dance. */ static void crypto_terminate(struct thread **tdp, void *q) { struct thread *td; mtx_assert(&crypto_drivers_mtx, MA_OWNED); td = *tdp; *tdp = NULL; if (td != NULL) { wakeup_one(q); mtx_sleep(td, &crypto_drivers_mtx, PWAIT, "crypto_destroy", 0); } } static void hmac_init_pad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx, uint8_t padval) { uint8_t hmac_key[HMAC_MAX_BLOCK_LEN]; u_int i; KASSERT(axf->blocksize <= sizeof(hmac_key), ("Invalid HMAC block size %d", axf->blocksize)); /* * If the key is larger than the block size, use the digest of * the key as the key instead. */ memset(hmac_key, 0, sizeof(hmac_key)); if (klen > axf->blocksize) { axf->Init(auth_ctx); axf->Update(auth_ctx, key, klen); axf->Final(hmac_key, auth_ctx); klen = axf->hashsize; } else memcpy(hmac_key, key, klen); for (i = 0; i < axf->blocksize; i++) hmac_key[i] ^= padval; axf->Init(auth_ctx); axf->Update(auth_ctx, hmac_key, axf->blocksize); explicit_bzero(hmac_key, sizeof(hmac_key)); } void hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx) { hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL); } void hmac_init_opad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx) { hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL); } static void crypto_destroy(void) { struct crypto_ret_worker *ret_worker; int i; /* * Terminate any crypto threads. */ if (crypto_tq != NULL) taskqueue_drain_all(crypto_tq); CRYPTO_DRIVER_LOCK(); crypto_terminate(&cryptotd, &crp_q); FOREACH_CRYPTO_RETW(ret_worker) crypto_terminate(&ret_worker->td, &ret_worker->crp_ret_q); CRYPTO_DRIVER_UNLOCK(); /* XXX flush queues??? */ /* * Reclaim dynamically allocated resources. */ for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] != NULL) cap_rele(crypto_drivers[i]); } free(crypto_drivers, M_CRYPTO_DATA); if (cryptop_zone != NULL) uma_zdestroy(cryptop_zone); mtx_destroy(&crypto_q_mtx); FOREACH_CRYPTO_RETW(ret_worker) mtx_destroy(&ret_worker->crypto_ret_mtx); free(crypto_ret_workers, M_CRYPTO_DATA); if (crypto_tq != NULL) taskqueue_free(crypto_tq); mtx_destroy(&crypto_drivers_mtx); } uint32_t crypto_ses2hid(crypto_session_t crypto_session) { return (crypto_session->cap->cc_hid); } uint32_t crypto_ses2caps(crypto_session_t crypto_session) { return (crypto_session->cap->cc_flags & 0xff000000); } void * crypto_get_driver_session(crypto_session_t crypto_session) { return (crypto_session + 1); } const struct crypto_session_params * crypto_get_params(crypto_session_t crypto_session) { return (&crypto_session->csp); } struct auth_hash * crypto_auth_hash(const struct crypto_session_params *csp) { switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: return (&auth_hash_hmac_sha1); case CRYPTO_SHA2_224_HMAC: return (&auth_hash_hmac_sha2_224); case CRYPTO_SHA2_256_HMAC: return (&auth_hash_hmac_sha2_256); case CRYPTO_SHA2_384_HMAC: return (&auth_hash_hmac_sha2_384); case CRYPTO_SHA2_512_HMAC: return (&auth_hash_hmac_sha2_512); case CRYPTO_NULL_HMAC: return (&auth_hash_null); case CRYPTO_RIPEMD160_HMAC: return (&auth_hash_hmac_ripemd_160); case CRYPTO_SHA1: return (&auth_hash_sha1); case CRYPTO_SHA2_224: return (&auth_hash_sha2_224); case CRYPTO_SHA2_256: return (&auth_hash_sha2_256); case CRYPTO_SHA2_384: return (&auth_hash_sha2_384); case CRYPTO_SHA2_512: return (&auth_hash_sha2_512); case CRYPTO_AES_NIST_GMAC: switch (csp->csp_auth_klen) { case 128 / 8: return (&auth_hash_nist_gmac_aes_128); case 192 / 8: return (&auth_hash_nist_gmac_aes_192); case 256 / 8: return (&auth_hash_nist_gmac_aes_256); default: return (NULL); } case CRYPTO_BLAKE2B: return (&auth_hash_blake2b); case CRYPTO_BLAKE2S: return (&auth_hash_blake2s); case CRYPTO_POLY1305: return (&auth_hash_poly1305); case CRYPTO_AES_CCM_CBC_MAC: switch (csp->csp_auth_klen) { case 128 / 8: return (&auth_hash_ccm_cbc_mac_128); case 192 / 8: return (&auth_hash_ccm_cbc_mac_192); case 256 / 8: return (&auth_hash_ccm_cbc_mac_256); default: return (NULL); } default: return (NULL); } } struct enc_xform * crypto_cipher(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_RIJNDAEL128_CBC: return (&enc_xform_rijndael128); case CRYPTO_AES_XTS: return (&enc_xform_aes_xts); case CRYPTO_AES_ICM: return (&enc_xform_aes_icm); case CRYPTO_AES_NIST_GCM_16: return (&enc_xform_aes_nist_gcm); case CRYPTO_CAMELLIA_CBC: return (&enc_xform_camellia); case CRYPTO_NULL_CBC: return (&enc_xform_null); case CRYPTO_CHACHA20: return (&enc_xform_chacha20); case CRYPTO_AES_CCM_16: return (&enc_xform_ccm); case CRYPTO_CHACHA20_POLY1305: return (&enc_xform_chacha20_poly1305); default: return (NULL); } } static struct cryptocap * crypto_checkdriver(uint32_t hid) { return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]); } /* * Select a driver for a new session that supports the specified * algorithms and, optionally, is constrained according to the flags. */ static struct cryptocap * crypto_select_driver(const struct crypto_session_params *csp, int flags) { struct cryptocap *cap, *best; int best_match, error, hid; CRYPTO_DRIVER_ASSERT(); best = NULL; for (hid = 0; hid < crypto_drivers_size; hid++) { /* * If there is no driver for this slot, or the driver * is not appropriate (hardware or software based on * match), then skip. */ cap = crypto_drivers[hid]; if (cap == NULL || (cap->cc_flags & flags) == 0) continue; error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp); if (error >= 0) continue; /* * Use the driver with the highest probe value. * Hardware drivers use a higher probe value than * software. In case of a tie, prefer the driver with * the fewest active sessions. */ if (best == NULL || error > best_match || (error == best_match && cap->cc_sessions < best->cc_sessions)) { best = cap; best_match = error; } } return best; } static enum alg_type { ALG_NONE = 0, ALG_CIPHER, ALG_DIGEST, ALG_KEYED_DIGEST, ALG_COMPRESSION, ALG_AEAD } alg_types[] = { [CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_CBC] = ALG_CIPHER, [CRYPTO_SHA1] = ALG_DIGEST, [CRYPTO_NULL_HMAC] = ALG_DIGEST, [CRYPTO_NULL_CBC] = ALG_CIPHER, [CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION, [CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_CAMELLIA_CBC] = ALG_CIPHER, [CRYPTO_AES_XTS] = ALG_CIPHER, [CRYPTO_AES_ICM] = ALG_CIPHER, [CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_NIST_GCM_16] = ALG_AEAD, [CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST, [CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST, [CRYPTO_CHACHA20] = ALG_CIPHER, [CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_RIPEMD160] = ALG_DIGEST, [CRYPTO_SHA2_224] = ALG_DIGEST, [CRYPTO_SHA2_256] = ALG_DIGEST, [CRYPTO_SHA2_384] = ALG_DIGEST, [CRYPTO_SHA2_512] = ALG_DIGEST, [CRYPTO_POLY1305] = ALG_KEYED_DIGEST, [CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_CCM_16] = ALG_AEAD, [CRYPTO_CHACHA20_POLY1305] = ALG_AEAD, }; static enum alg_type alg_type(int alg) { if (alg < nitems(alg_types)) return (alg_types[alg]); return (ALG_NONE); } static bool alg_is_compression(int alg) { return (alg_type(alg) == ALG_COMPRESSION); } static bool alg_is_cipher(int alg) { return (alg_type(alg) == ALG_CIPHER); } static bool alg_is_digest(int alg) { return (alg_type(alg) == ALG_DIGEST || alg_type(alg) == ALG_KEYED_DIGEST); } static bool alg_is_keyed_digest(int alg) { return (alg_type(alg) == ALG_KEYED_DIGEST); } static bool alg_is_aead(int alg) { return (alg_type(alg) == ALG_AEAD); } static bool ccm_tag_length_valid(int len) { /* RFC 3610 */ switch (len) { case 4: case 6: case 8: case 10: case 12: case 14: case 16: return (true); default: return (false); } } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) /* Various sanity checks on crypto session parameters. */ static bool check_csp(const struct crypto_session_params *csp) { struct auth_hash *axf; /* Mode-independent checks. */ if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (false); if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 || csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0) return (false); if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0) return (false); if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0) return (false); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: if (!alg_is_compression(csp->csp_cipher_alg)) return (false); if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 || csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || csp->csp_auth_mlen != 0) return (false); break; case CSP_MODE_CIPHER: if (!alg_is_cipher(csp->csp_cipher_alg)) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0) return (false); } if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || csp->csp_auth_mlen != 0) return (false); break; case CSP_MODE_DIGEST: if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); /* IV is optional for digests (e.g. GMAC). */ switch (csp->csp_auth_alg) { case CRYPTO_AES_CCM_CBC_MAC: if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13) return (false); break; case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (false); break; default: if (csp->csp_ivlen != 0) return (false); break; } if (!alg_is_digest(csp->csp_auth_alg)) return (false); /* Key is optional for BLAKE2 digests. */ if (csp->csp_auth_alg == CRYPTO_BLAKE2B || csp->csp_auth_alg == CRYPTO_BLAKE2S) ; else if (alg_is_keyed_digest(csp->csp_auth_alg)) { if (csp->csp_auth_klen == 0) return (false); } else { if (csp->csp_auth_klen != 0) return (false); } if (csp->csp_auth_mlen != 0) { axf = crypto_auth_hash(csp); if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) return (false); if (csp->csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC && !ccm_tag_length_valid(csp->csp_auth_mlen)) return (false); } break; case CSP_MODE_AEAD: if (!alg_is_aead(csp->csp_cipher_alg)) return (false); if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0 || csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0) return (false); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CCM_16: if (csp->csp_auth_mlen != 0 && !ccm_tag_length_valid(csp->csp_auth_mlen)) return (false); if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13) return (false); break; case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_auth_mlen > 16) + if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN) + return (false); + + if (csp->csp_ivlen != AES_GCM_IV_LEN) return (false); break; case CRYPTO_CHACHA20_POLY1305: if (csp->csp_ivlen != 8 && csp->csp_ivlen != 12) return (false); if (csp->csp_auth_mlen > POLY1305_HASH_LEN) return (false); break; } break; case CSP_MODE_ETA: if (!alg_is_cipher(csp->csp_cipher_alg)) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0) return (false); } if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (!alg_is_digest(csp->csp_auth_alg)) return (false); /* Key is optional for BLAKE2 digests. */ if (csp->csp_auth_alg == CRYPTO_BLAKE2B || csp->csp_auth_alg == CRYPTO_BLAKE2S) ; else if (alg_is_keyed_digest(csp->csp_auth_alg)) { if (csp->csp_auth_klen == 0) return (false); } else { if (csp->csp_auth_klen != 0) return (false); } if (csp->csp_auth_mlen != 0) { axf = crypto_auth_hash(csp); if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) return (false); } break; default: return (false); } return (true); } /* * Delete a session after it has been detached from its driver. */ static void crypto_deletesession(crypto_session_t cses) { struct cryptocap *cap; cap = cses->cap; zfree(cses, M_CRYPTO_DATA); CRYPTO_DRIVER_LOCK(); cap->cc_sessions--; if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP) wakeup(cap); CRYPTO_DRIVER_UNLOCK(); cap_rele(cap); } /* * Create a new session. The crid argument specifies a crypto * driver to use or constraints on a driver to select (hardware * only, software only, either). Whatever driver is selected * must be capable of the requested crypto algorithms. */ int crypto_newsession(crypto_session_t *cses, const struct crypto_session_params *csp, int crid) { static uint64_t sessid = 0; crypto_session_t res; struct cryptocap *cap; int err; if (!check_csp(csp)) return (EINVAL); res = NULL; CRYPTO_DRIVER_LOCK(); if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { /* * Use specified driver; verify it is capable. */ cap = crypto_checkdriver(crid); if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0) cap = NULL; } else { /* * No requested driver; select based on crid flags. */ cap = crypto_select_driver(csp, crid); } if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); CRYPTDEB("no driver"); return (EOPNOTSUPP); } cap_ref(cap); cap->cc_sessions++; CRYPTO_DRIVER_UNLOCK(); /* Allocate a single block for the generic session and driver softc. */ res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO); res->cap = cap; res->csp = *csp; res->id = atomic_fetchadd_64(&sessid, 1); /* Call the driver initialization routine. */ err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp); if (err != 0) { CRYPTDEB("dev newsession failed: %d", err); crypto_deletesession(res); return (err); } *cses = res; return (0); } /* * Delete an existing session (or a reserved session on an unregistered * driver). */ void crypto_freesession(crypto_session_t cses) { struct cryptocap *cap; if (cses == NULL) return; cap = cses->cap; /* Call the driver cleanup routine, if available. */ CRYPTODEV_FREESESSION(cap->cc_dev, cses); crypto_deletesession(cses); } /* * Return a new driver id. Registers a driver with the system so that * it can be probed by subsequent sessions. */ int32_t crypto_get_driverid(device_t dev, size_t sessionsize, int flags) { struct cryptocap *cap, **newdrv; int i; if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { device_printf(dev, "no flags specified when registering driver\n"); return -1; } cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO); cap->cc_dev = dev; cap->cc_session_size = sessionsize; cap->cc_flags = flags; refcount_init(&cap->cc_refs, 1); CRYPTO_DRIVER_LOCK(); for (;;) { for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] == NULL) break; } if (i < crypto_drivers_size) break; /* Out of entries, allocate some more. */ if (2 * crypto_drivers_size <= crypto_drivers_size) { CRYPTO_DRIVER_UNLOCK(); printf("crypto: driver count wraparound!\n"); cap_rele(cap); return (-1); } CRYPTO_DRIVER_UNLOCK(); newdrv = malloc(2 * crypto_drivers_size * sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO); CRYPTO_DRIVER_LOCK(); memcpy(newdrv, crypto_drivers, crypto_drivers_size * sizeof(*crypto_drivers)); crypto_drivers_size *= 2; free(crypto_drivers, M_CRYPTO_DATA); crypto_drivers = newdrv; } cap->cc_hid = i; crypto_drivers[i] = cap; CRYPTO_DRIVER_UNLOCK(); if (bootverbose) printf("crypto: assign %s driver id %u, flags 0x%x\n", device_get_nameunit(dev), i, flags); return i; } /* * Lookup a driver by name. We match against the full device * name and unit, and against just the name. The latter gives * us a simple widlcarding by device name. On success return the * driver/hardware identifier; otherwise return -1. */ int crypto_find_driver(const char *match) { struct cryptocap *cap; int i, len = strlen(match); CRYPTO_DRIVER_LOCK(); for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] == NULL) continue; cap = crypto_drivers[i]; if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 || strncmp(match, device_get_name(cap->cc_dev), len) == 0) { CRYPTO_DRIVER_UNLOCK(); return (i); } } CRYPTO_DRIVER_UNLOCK(); return (-1); } /* * Return the device_t for the specified driver or NULL * if the driver identifier is invalid. */ device_t crypto_find_device_byhid(int hid) { struct cryptocap *cap; device_t dev; dev = NULL; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(hid); if (cap != NULL) dev = cap->cc_dev; CRYPTO_DRIVER_UNLOCK(); return (dev); } /* * Return the device/driver capabilities. */ int crypto_getcaps(int hid) { struct cryptocap *cap; int flags; flags = 0; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(hid); if (cap != NULL) flags = cap->cc_flags; CRYPTO_DRIVER_UNLOCK(); return (flags); } /* * Register support for a key-related algorithm. This routine * is called once for each algorithm supported a driver. */ int crypto_kregister(uint32_t driverid, int kalg, uint32_t flags) { struct cryptocap *cap; int err; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(driverid); if (cap != NULL && (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { /* * XXX Do some performance testing to determine placing. * XXX We probably need an auxiliary data structure that * XXX describes relative performances. */ cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; if (bootverbose) printf("crypto: %s registers key alg %u flags %u\n" , device_get_nameunit(cap->cc_dev) , kalg , flags ); gone_in_dev(cap->cc_dev, 14, "asymmetric crypto"); err = 0; } else err = EINVAL; CRYPTO_DRIVER_UNLOCK(); return err; } /* * Unregister all algorithms associated with a crypto driver. * If there are pending sessions using it, leave enough information * around so that subsequent calls using those sessions will * correctly detect the driver has been unregistered and reroute * requests. */ int crypto_unregister_all(uint32_t driverid) { struct cryptocap *cap; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(driverid); if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); return (EINVAL); } cap->cc_flags |= CRYPTOCAP_F_CLEANUP; crypto_drivers[driverid] = NULL; /* * XXX: This doesn't do anything to kick sessions that * have no pending operations. */ while (cap->cc_sessions != 0 || cap->cc_koperations != 0) mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0); CRYPTO_DRIVER_UNLOCK(); cap_rele(cap); return (0); } /* * Clear blockage on a driver. The what parameter indicates whether * the driver is now ready for cryptop's and/or cryptokop's. */ int crypto_unblock(uint32_t driverid, int what) { struct cryptocap *cap; int err; CRYPTO_Q_LOCK(); cap = crypto_checkdriver(driverid); if (cap != NULL) { if (what & CRYPTO_SYMQ) cap->cc_qblocked = 0; if (what & CRYPTO_ASYMQ) cap->cc_kqblocked = 0; if (crp_sleep) wakeup_one(&crp_q); err = 0; } else err = EINVAL; CRYPTO_Q_UNLOCK(); return err; } size_t crypto_buffer_len(struct crypto_buffer *cb) { switch (cb->cb_type) { case CRYPTO_BUF_CONTIG: return (cb->cb_buf_len); case CRYPTO_BUF_MBUF: if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); case CRYPTO_BUF_SINGLE_MBUF: return (cb->cb_mbuf->m_len); case CRYPTO_BUF_VMPAGE: return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: return (cb->cb_uio->uio_resid); default: return (0); } } #ifdef INVARIANTS /* Various sanity checks on crypto requests. */ static void cb_sanity(struct crypto_buffer *cb, const char *name) { KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid %s buffer type", name)); switch (cb->cb_type) { case CRYPTO_BUF_CONTIG: KASSERT(cb->cb_buf_len >= 0, ("incoming crp with -ve %s buffer length", name)); break; case CRYPTO_BUF_VMPAGE: KASSERT(CRYPTO_HAS_VMPAGE, ("incoming crp uses dmap on supported arch")); KASSERT(cb->cb_vm_page_len >= 0, ("incoming crp with -ve %s buffer length", name)); KASSERT(cb->cb_vm_page_offset >= 0, ("incoming crp with -ve %s buffer offset", name)); KASSERT(cb->cb_vm_page_offset < PAGE_SIZE, ("incoming crp with %s buffer offset greater than page size" , name)); break; default: break; } } static void crp_sanity(struct cryptop *crp) { struct crypto_session_params *csp; struct crypto_buffer *out; size_t ilen, len, olen; KASSERT(crp->crp_session != NULL, ("incoming crp without a session")); KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE && crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid output buffer type")); KASSERT(crp->crp_etype == 0, ("incoming crp with error")); KASSERT(!(crp->crp_flags & CRYPTO_F_DONE), ("incoming crp already done")); csp = &crp->crp_session->csp; cb_sanity(&crp->crp_buf, "input"); ilen = crypto_buffer_len(&crp->crp_buf); olen = ilen; out = NULL; if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) { if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) { cb_sanity(&crp->crp_obuf, "output"); out = &crp->crp_obuf; olen = crypto_buffer_len(out); } } else KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE, ("incoming crp with separate output buffer " "but no session support")); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS || crp->crp_op == CRYPTO_OP_DECOMPRESS, ("invalid compression op %x", crp->crp_op)); break; case CSP_MODE_CIPHER: KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT || crp->crp_op == CRYPTO_OP_DECRYPT, ("invalid cipher op %x", crp->crp_op)); break; case CSP_MODE_DIGEST: KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST || crp->crp_op == CRYPTO_OP_VERIFY_DIGEST, ("invalid digest op %x", crp->crp_op)); break; case CSP_MODE_AEAD: KASSERT(crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || crp->crp_op == (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), ("invalid AEAD op %x", crp->crp_op)); KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE, ("AEAD without a separate IV")); break; case CSP_MODE_ETA: KASSERT(crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || crp->crp_op == (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), ("invalid ETA op %x", crp->crp_op)); break; } if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { if (crp->crp_aad == NULL) { KASSERT(crp->crp_aad_start == 0 || crp->crp_aad_start < ilen, ("invalid AAD start")); KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0, ("AAD with zero length and non-zero start")); KASSERT(crp->crp_aad_length == 0 || crp->crp_aad_start + crp->crp_aad_length <= ilen, ("AAD outside input length")); } else { KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD, ("session doesn't support separate AAD buffer")); KASSERT(crp->crp_aad_start == 0, ("separate AAD buffer with non-zero AAD start")); KASSERT(crp->crp_aad_length != 0, ("separate AAD buffer with zero length")); } } else { KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 && crp->crp_aad_length == 0, ("AAD region in request not supporting AAD")); } if (csp->csp_ivlen == 0) { KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0, ("IV_SEPARATE set when IV isn't used")); KASSERT(crp->crp_iv_start == 0, ("crp_iv_start set when IV isn't used")); } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { KASSERT(crp->crp_iv_start == 0, ("IV_SEPARATE used with non-zero IV start")); } else { KASSERT(crp->crp_iv_start < ilen, ("invalid IV start")); KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen, ("IV outside buffer length")); } /* XXX: payload_start of 0 should always be < ilen? */ KASSERT(crp->crp_payload_start == 0 || crp->crp_payload_start < ilen, ("invalid payload start")); KASSERT(crp->crp_payload_start + crp->crp_payload_length <= ilen, ("payload outside input buffer")); if (out == NULL) { KASSERT(crp->crp_payload_output_start == 0, ("payload output start non-zero without output buffer")); } else { KASSERT(crp->crp_payload_output_start < olen, ("invalid payload output start")); KASSERT(crp->crp_payload_output_start + crp->crp_payload_length <= olen, ("payload outside output buffer")); } if (csp->csp_mode == CSP_MODE_DIGEST || csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) len = ilen; else len = olen; KASSERT(crp->crp_digest_start == 0 || crp->crp_digest_start < len, ("invalid digest start")); /* XXX: For the mlen == 0 case this check isn't perfect. */ KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len, ("digest outside buffer")); } else { KASSERT(crp->crp_digest_start == 0, ("non-zero digest start for request without a digest")); } if (csp->csp_cipher_klen != 0) KASSERT(csp->csp_cipher_key != NULL || crp->crp_cipher_key != NULL, ("cipher request without a key")); if (csp->csp_auth_klen != 0) KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL, ("auth request without a key")); KASSERT(crp->crp_callback != NULL, ("incoming crp without callback")); } #endif /* * Add a crypto request to a queue, to be processed by the kernel thread. */ int crypto_dispatch(struct cryptop *crp) { struct cryptocap *cap; int result; #ifdef INVARIANTS crp_sanity(crp); #endif CRYPTOSTAT_INC(cs_ops); crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; if (CRYPTOP_ASYNC(crp)) { if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) { struct crypto_ret_worker *ret_worker; ret_worker = CRYPTO_RETW(crp->crp_retw_id); CRYPTO_RETW_LOCK(ret_worker); crp->crp_seq = ret_worker->reorder_ops++; CRYPTO_RETW_UNLOCK(ret_worker); } TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); taskqueue_enqueue(crypto_tq, &crp->crp_task); return (0); } if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { /* * Caller marked the request to be processed * immediately; dispatch it directly to the * driver unless the driver is currently blocked. */ cap = crp->crp_session->cap; if (!cap->cc_qblocked) { result = crypto_invoke(cap, crp, 0); if (result != ERESTART) return (result); /* * The driver ran out of resources, put the request on * the queue. */ } } crypto_batch_enqueue(crp); return 0; } void crypto_batch_enqueue(struct cryptop *crp) { CRYPTO_Q_LOCK(); TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); if (crp_sleep) wakeup_one(&crp_q); CRYPTO_Q_UNLOCK(); } /* * Add an asymetric crypto request to a queue, * to be processed by the kernel thread. */ int crypto_kdispatch(struct cryptkop *krp) { int error; CRYPTOSTAT_INC(cs_kops); krp->krp_cap = NULL; error = crypto_kinvoke(krp); if (error == ERESTART) { CRYPTO_Q_LOCK(); TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); if (crp_sleep) wakeup_one(&crp_q); CRYPTO_Q_UNLOCK(); error = 0; } return error; } /* * Verify a driver is suitable for the specified operation. */ static __inline int kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) { return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; } /* * Select a driver for an asym operation. The driver must * support the necessary algorithm. The caller can constrain * which device is selected with the flags parameter. The * algorithm we use here is pretty stupid; just use the first * driver that supports the algorithms we need. If there are * multiple suitable drivers we choose the driver with the * fewest active operations. We prefer hardware-backed * drivers to software ones when either may be used. */ static struct cryptocap * crypto_select_kdriver(const struct cryptkop *krp, int flags) { struct cryptocap *cap, *best; int match, hid; CRYPTO_DRIVER_ASSERT(); /* * Look first for hardware crypto devices if permitted. */ if (flags & CRYPTOCAP_F_HARDWARE) match = CRYPTOCAP_F_HARDWARE; else match = CRYPTOCAP_F_SOFTWARE; best = NULL; again: for (hid = 0; hid < crypto_drivers_size; hid++) { /* * If there is no driver for this slot, or the driver * is not appropriate (hardware or software based on * match), then skip. */ cap = crypto_drivers[hid]; if (cap == NULL || (cap->cc_flags & match) == 0) continue; /* verify all the algorithms are supported. */ if (kdriver_suitable(cap, krp)) { if (best == NULL || cap->cc_koperations < best->cc_koperations) best = cap; } } if (best != NULL) return best; if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { /* sort of an Algol 68-style for loop */ match = CRYPTOCAP_F_SOFTWARE; goto again; } return best; } /* * Choose a driver for an asymmetric crypto request. */ static struct cryptocap * crypto_lookup_kdriver(struct cryptkop *krp) { struct cryptocap *cap; uint32_t crid; /* If this request is requeued, it might already have a driver. */ cap = krp->krp_cap; if (cap != NULL) return (cap); /* Use krp_crid to choose a driver. */ crid = krp->krp_crid; if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { cap = crypto_checkdriver(crid); if (cap != NULL) { /* * Driver present, it must support the * necessary algorithm and, if s/w drivers are * excluded, it must be registered as * hardware-backed. */ if (!kdriver_suitable(cap, krp) || (!crypto_devallowsoft && (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) cap = NULL; } } else { /* * No requested driver; select based on crid flags. */ if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ crid &= ~CRYPTOCAP_F_SOFTWARE; cap = crypto_select_kdriver(krp, crid); } if (cap != NULL) { krp->krp_cap = cap_ref(cap); krp->krp_hid = cap->cc_hid; } return (cap); } /* * Dispatch an asymmetric crypto request. */ static int crypto_kinvoke(struct cryptkop *krp) { struct cryptocap *cap = NULL; int error; KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); KASSERT(krp->krp_callback != NULL, ("%s: krp->crp_callback == NULL", __func__)); CRYPTO_DRIVER_LOCK(); cap = crypto_lookup_kdriver(krp); if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); krp->krp_status = ENODEV; crypto_kdone(krp); return (0); } /* * If the device is blocked, return ERESTART to requeue it. */ if (cap->cc_kqblocked) { /* * XXX: Previously this set krp_status to ERESTART and * invoked crypto_kdone but the caller would still * requeue it. */ CRYPTO_DRIVER_UNLOCK(); return (ERESTART); } cap->cc_koperations++; CRYPTO_DRIVER_UNLOCK(); error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); if (error == ERESTART) { CRYPTO_DRIVER_LOCK(); cap->cc_koperations--; CRYPTO_DRIVER_UNLOCK(); return (error); } KASSERT(error == 0, ("error %d returned from crypto_kprocess", error)); return (0); } static void crypto_task_invoke(void *ctx, int pending) { struct cryptocap *cap; struct cryptop *crp; int result; crp = (struct cryptop *)ctx; cap = crp->crp_session->cap; result = crypto_invoke(cap, crp, 0); if (result == ERESTART) crypto_batch_enqueue(crp); } /* * Dispatch a crypto request to the appropriate crypto devices. */ static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) { KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); KASSERT(crp->crp_callback != NULL, ("%s: crp->crp_callback == NULL", __func__)); KASSERT(crp->crp_session != NULL, ("%s: crp->crp_session == NULL", __func__)); if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { struct crypto_session_params csp; crypto_session_t nses; /* * Driver has unregistered; migrate the session and return * an error to the caller so they'll resubmit the op. * * XXX: What if there are more already queued requests for this * session? * * XXX: Real solution is to make sessions refcounted * and force callers to hold a reference when * assigning to crp_session. Could maybe change * crypto_getreq to accept a session pointer to make * that work. Alternatively, we could abandon the * notion of rewriting crp_session in requests forcing * the caller to deal with allocating a new session. * Perhaps provide a method to allow a crp's session to * be swapped that callers could use. */ csp = crp->crp_session->csp; crypto_freesession(crp->crp_session); /* * XXX: Key pointers may no longer be valid. If we * really want to support this we need to define the * KPI such that 'csp' is required to be valid for the * duration of a session by the caller perhaps. * * XXX: If the keys have been changed this will reuse * the old keys. This probably suggests making * rekeying more explicit and updating the key * pointers in 'csp' when the keys change. */ if (crypto_newsession(&nses, &csp, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) crp->crp_session = nses; crp->crp_etype = EAGAIN; crypto_done(crp); return 0; } else { /* * Invoke the driver to process the request. */ return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); } } void crypto_destroyreq(struct cryptop *crp) { #ifdef DIAGNOSTIC { struct cryptop *crp2; struct crypto_ret_worker *ret_worker; CRYPTO_Q_LOCK(); TAILQ_FOREACH(crp2, &crp_q, crp_next) { KASSERT(crp2 != crp, ("Freeing cryptop from the crypto queue (%p).", crp)); } CRYPTO_Q_UNLOCK(); FOREACH_CRYPTO_RETW(ret_worker) { CRYPTO_RETW_LOCK(ret_worker); TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) { KASSERT(crp2 != crp, ("Freeing cryptop from the return queue (%p).", crp)); } CRYPTO_RETW_UNLOCK(ret_worker); } } #endif } void crypto_freereq(struct cryptop *crp) { if (crp == NULL) return; crypto_destroyreq(crp); uma_zfree(cryptop_zone, crp); } static void _crypto_initreq(struct cryptop *crp, crypto_session_t cses) { crp->crp_session = cses; } void crypto_initreq(struct cryptop *crp, crypto_session_t cses) { memset(crp, 0, sizeof(*crp)); _crypto_initreq(crp, cses); } struct cryptop * crypto_getreq(crypto_session_t cses, int how) { struct cryptop *crp; MPASS(how == M_WAITOK || how == M_NOWAIT); crp = uma_zalloc(cryptop_zone, how | M_ZERO); if (crp != NULL) _crypto_initreq(crp, cses); return (crp); } /* * Invoke the callback on behalf of the driver. */ void crypto_done(struct cryptop *crp) { KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); crp->crp_flags |= CRYPTO_F_DONE; if (crp->crp_etype != 0) CRYPTOSTAT_INC(cs_errs); /* * CBIMM means unconditionally do the callback immediately; * CBIFSYNC means do the callback immediately only if the * operation was done synchronously. Both are used to avoid * doing extraneous context switches; the latter is mostly * used with the software crypto driver. */ if (!CRYPTOP_ASYNC_KEEPORDER(crp) && ((crp->crp_flags & CRYPTO_F_CBIMM) || ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) { /* * Do the callback directly. This is ok when the * callback routine does very little (e.g. the * /dev/crypto callback method just does a wakeup). */ crp->crp_callback(crp); } else { struct crypto_ret_worker *ret_worker; bool wake; ret_worker = CRYPTO_RETW(crp->crp_retw_id); wake = false; /* * Normal case; queue the callback for the thread. */ CRYPTO_RETW_LOCK(ret_worker); if (CRYPTOP_ASYNC_KEEPORDER(crp)) { struct cryptop *tmp; TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q, cryptop_q, crp_next) { if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q, tmp, crp, crp_next); break; } } if (tmp == NULL) { TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q, crp, crp_next); } if (crp->crp_seq == ret_worker->reorder_cur_seq) wake = true; } else { if (CRYPTO_RETW_EMPTY(ret_worker)) wake = true; TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); } if (wake) wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ CRYPTO_RETW_UNLOCK(ret_worker); } } /* * Invoke the callback on behalf of the driver. */ void crypto_kdone(struct cryptkop *krp) { struct crypto_ret_worker *ret_worker; struct cryptocap *cap; if (krp->krp_status != 0) CRYPTOSTAT_INC(cs_kerrs); cap = krp->krp_cap; if (cap != NULL) { CRYPTO_DRIVER_LOCK(); KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); cap->cc_koperations--; if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP) wakeup(cap); CRYPTO_DRIVER_UNLOCK(); krp->krp_cap = NULL; cap_rele(cap); } ret_worker = CRYPTO_RETW(0); CRYPTO_RETW_LOCK(ret_worker); if (CRYPTO_RETW_EMPTY(ret_worker)) wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next); CRYPTO_RETW_UNLOCK(ret_worker); } int crypto_getfeat(int *featp) { int hid, kalg, feat = 0; CRYPTO_DRIVER_LOCK(); for (hid = 0; hid < crypto_drivers_size; hid++) { const struct cryptocap *cap = crypto_drivers[hid]; if (cap == NULL || ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && !crypto_devallowsoft)) { continue; } for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) feat |= 1 << kalg; } CRYPTO_DRIVER_UNLOCK(); *featp = feat; return (0); } /* * Terminate a thread at module unload. The process that * initiated this is waiting for us to signal that we're gone; * wake it up and exit. We use the driver table lock to insure * we don't do the wakeup before they're waiting. There is no * race here because the waiter sleeps on the proc lock for the * thread so it gets notified at the right time because of an * extra wakeup that's done in exit1(). */ static void crypto_finis(void *chan) { CRYPTO_DRIVER_LOCK(); wakeup_one(chan); CRYPTO_DRIVER_UNLOCK(); kthread_exit(); } /* * Crypto thread, dispatches crypto requests. */ static void crypto_dispatch_thread(void *arg __unused) { struct cryptop *crp, *submit; struct cryptkop *krp; struct cryptocap *cap; int result, hint; #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) fpu_kern_thread(FPU_KERN_NORMAL); #endif CRYPTO_Q_LOCK(); for (;;) { /* * Find the first element in the queue that can be * processed and look-ahead to see if multiple ops * are ready for the same driver. */ submit = NULL; hint = 0; TAILQ_FOREACH(crp, &crp_q, crp_next) { cap = crp->crp_session->cap; /* * Driver cannot disappeared when there is an active * session. */ KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { /* Op needs to be migrated, process it. */ if (submit == NULL) submit = crp; break; } if (!cap->cc_qblocked) { if (submit != NULL) { /* * We stop on finding another op, * regardless whether its for the same * driver or not. We could keep * searching the queue but it might be * better to just use a per-driver * queue instead. */ if (submit->crp_session->cap == cap) hint = CRYPTO_HINT_MORE; break; } else { submit = crp; if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) break; /* keep scanning for more are q'd */ } } } if (submit != NULL) { TAILQ_REMOVE(&crp_q, submit, crp_next); cap = submit->crp_session->cap; KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); CRYPTO_Q_UNLOCK(); result = crypto_invoke(cap, submit, hint); CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the * driver ``blocked'' for cryptop's and put * the request back in the queue. It would * best to put the request back where we got * it but that's hard so for now we put it * at the front. This should be ok; putting * it at the end does not work. */ cap->cc_qblocked = 1; TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); CRYPTOSTAT_INC(cs_blocks); } } /* As above, but for key ops */ TAILQ_FOREACH(krp, &crp_kq, krp_next) { cap = krp->krp_cap; if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { /* * Operation needs to be migrated, * clear krp_cap so a new driver is * selected. */ krp->krp_cap = NULL; cap_rele(cap); break; } if (!cap->cc_kqblocked) break; } if (krp != NULL) { TAILQ_REMOVE(&crp_kq, krp, krp_next); CRYPTO_Q_UNLOCK(); result = crypto_kinvoke(krp); CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the * driver ``blocked'' for cryptkop's and put * the request back in the queue. It would * best to put the request back where we got * it but that's hard so for now we put it * at the front. This should be ok; putting * it at the end does not work. */ krp->krp_cap->cc_kqblocked = 1; TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); CRYPTOSTAT_INC(cs_kblocks); } } if (submit == NULL && krp == NULL) { /* * Nothing more to be processed. Sleep until we're * woken because there are more ops to process. * This happens either by submission or by a driver * becoming unblocked and notifying us through * crypto_unblock. Note that when we wakeup we * start processing each queue again from the * front. It's not clear that it's important to * preserve this ordering since ops may finish * out of order if dispatched to different devices * and some become blocked while others do not. */ crp_sleep = 1; msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); crp_sleep = 0; if (cryptotd == NULL) break; CRYPTOSTAT_INC(cs_intrs); } } CRYPTO_Q_UNLOCK(); crypto_finis(&crp_q); } /* * Crypto returns thread, does callbacks for processed crypto requests. * Callbacks are done here, rather than in the crypto drivers, because * callbacks typically are expensive and would slow interrupt handling. */ static void crypto_ret_thread(void *arg) { struct crypto_ret_worker *ret_worker = arg; struct cryptop *crpt; struct cryptkop *krpt; CRYPTO_RETW_LOCK(ret_worker); for (;;) { /* Harvest return q's for completed ops */ crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q); if (crpt != NULL) { if (crpt->crp_seq == ret_worker->reorder_cur_seq) { TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next); ret_worker->reorder_cur_seq++; } else { crpt = NULL; } } if (crpt == NULL) { crpt = TAILQ_FIRST(&ret_worker->crp_ret_q); if (crpt != NULL) TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next); } krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq); if (krpt != NULL) TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next); if (crpt != NULL || krpt != NULL) { CRYPTO_RETW_UNLOCK(ret_worker); /* * Run callbacks unlocked. */ if (crpt != NULL) crpt->crp_callback(crpt); if (krpt != NULL) krpt->krp_callback(krpt); CRYPTO_RETW_LOCK(ret_worker); } else { /* * Nothing more to be processed. Sleep until we're * woken because there are more returns to process. */ msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT, "crypto_ret_wait", 0); if (ret_worker->td == NULL) break; CRYPTOSTAT_INC(cs_rets); } } CRYPTO_RETW_UNLOCK(ret_worker); crypto_finis(&ret_worker->crp_ret_q); } #ifdef DDB static void db_show_drivers(void) { int hid; db_printf("%12s %4s %4s %8s %2s %2s\n" , "Device" , "Ses" , "Kops" , "Flags" , "QB" , "KB" ); for (hid = 0; hid < crypto_drivers_size; hid++) { const struct cryptocap *cap = crypto_drivers[hid]; if (cap == NULL) continue; db_printf("%-12s %4u %4u %08x %2u %2u\n" , device_get_nameunit(cap->cc_dev) , cap->cc_sessions , cap->cc_koperations , cap->cc_flags , cap->cc_qblocked , cap->cc_kqblocked ); } } DB_SHOW_COMMAND(crypto, db_show_crypto) { struct cryptop *crp; struct crypto_ret_worker *ret_worker; db_show_drivers(); db_printf("\n"); db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", "Device", "Callback"); TAILQ_FOREACH(crp, &crp_q, crp_next) { db_printf("%4u %08x %4u %4u %04x %8p %8p\n" , crp->crp_session->cap->cc_hid , (int) crypto_ses2caps(crp->crp_session) , crp->crp_olen , crp->crp_etype , crp->crp_flags , device_get_nameunit(crp->crp_session->cap->cc_dev) , crp->crp_callback ); } FOREACH_CRYPTO_RETW(ret_worker) { db_printf("\n%8s %4s %4s %4s %8s\n", "ret_worker", "HID", "Etype", "Flags", "Callback"); if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) { db_printf("%8td %4u %4u %04x %8p\n" , CRYPTO_RETW_ID(ret_worker) , crp->crp_session->cap->cc_hid , crp->crp_etype , crp->crp_flags , crp->crp_callback ); } } } } DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) { struct cryptkop *krp; struct crypto_ret_worker *ret_worker; db_show_drivers(); db_printf("\n"); db_printf("%4s %5s %4s %4s %8s %4s %8s\n", "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); TAILQ_FOREACH(krp, &crp_kq, krp_next) { db_printf("%4u %5u %4u %4u %08x %4u %8p\n" , krp->krp_op , krp->krp_status , krp->krp_iparams, krp->krp_oparams , krp->krp_crid, krp->krp_hid , krp->krp_callback ); } ret_worker = CRYPTO_RETW(0); if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { db_printf("%4s %5s %8s %4s %8s\n", "Op", "Status", "CRID", "HID", "Callback"); TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) { db_printf("%4u %5u %08x %4u %8p\n" , krp->krp_op , krp->krp_status , krp->krp_crid, krp->krp_hid , krp->krp_callback ); } } } #endif int crypto_modevent(module_t mod, int type, void *unused); /* * Initialization code, both for static and dynamic loading. * Note this is not invoked with the usual MODULE_DECLARE * mechanism but instead is listed as a dependency by the * cryptosoft driver. This guarantees proper ordering of * calls on module load/unload. */ int crypto_modevent(module_t mod, int type, void *unused) { int error = EINVAL; switch (type) { case MOD_LOAD: error = crypto_init(); if (error == 0 && bootverbose) printf("crypto: \n"); break; case MOD_UNLOAD: /*XXX disallow if active sessions */ error = 0; crypto_destroy(); return 0; } return error; } MODULE_VERSION(crypto, 1); MODULE_DEPEND(crypto, zlib, 1, 1, 1); diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c index 567a0f4748d5..b6c964b540bc 100644 --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -1,1760 +1,1757 @@ /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000, 2001 Angelos D. Keromytis * Copyright (c) 2014-2021 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Portions of this software were developed by Ararat River * Consulting, LLC under sponsorship of the FreeBSD Foundation. * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" struct swcr_auth { void *sw_ictx; void *sw_octx; struct auth_hash *sw_axf; uint16_t sw_mlen; }; struct swcr_encdec { void *sw_kschedule; struct enc_xform *sw_exf; }; struct swcr_compdec { struct comp_algo *sw_cxf; }; struct swcr_session { struct mtx swcr_lock; int (*swcr_process)(struct swcr_session *, struct cryptop *); struct swcr_auth swcr_auth; struct swcr_encdec swcr_encdec; struct swcr_compdec swcr_compdec; }; static int32_t swcr_id; static void swcr_freesession(device_t dev, crypto_session_t cses); /* Used for CRYPTO_NULL_CBC. */ static int swcr_null(struct swcr_session *ses, struct cryptop *crp) { return (0); } /* * Apply a symmetric encryption/decryption algorithm. */ static int swcr_encdec(struct swcr_session *ses, struct cryptop *crp) { unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; const struct crypto_session_params *csp; struct swcr_encdec *sw; struct enc_xform *exf; size_t inlen, outlen; int i, blks, resid; struct crypto_buffer_cursor cc_in, cc_out; const unsigned char *inblk; unsigned char *outblk; int error; bool encrypting; error = 0; sw = &ses->swcr_encdec; exf = sw->sw_exf; csp = crypto_get_params(crp->crp_session); if (exf->native_blocksize == 0) { /* Check for non-padded data */ if ((crp->crp_payload_length % exf->blocksize) != 0) return (EINVAL); blks = exf->blocksize; } else blks = exf->native_blocksize; if (exf == &enc_xform_aes_icm && (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); if (crp->crp_cipher_key != NULL) { error = exf->setkey(sw->sw_kschedule, crp->crp_cipher_key, csp->csp_cipher_klen); if (error) return (error); } crypto_read_iv(crp, iv); if (exf->reinit) { /* * xforms that provide a reinit method perform all IV * handling themselves. */ exf->reinit(sw->sw_kschedule, iv, csp->csp_ivlen); } ivp = iv; crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); inblk = crypto_cursor_segment(&cc_in, &inlen); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; outblk = crypto_cursor_segment(&cc_out, &outlen); resid = crp->crp_payload_length; encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); /* * Loop through encrypting blocks. 'inlen' is the remaining * length of the current segment in the input buffer. * 'outlen' is the remaining length of current segment in the * output buffer. */ while (resid >= blks) { /* * If the current block is not contained within the * current input/output segment, use 'blk' as a local * buffer. */ if (inlen < blks) { crypto_cursor_copydata(&cc_in, blks, blk); inblk = blk; } if (outlen < blks) outblk = blk; /* * Ciphers without a 'reinit' hook are assumed to be * used in CBC mode where the chaining is done here. */ if (exf->reinit != NULL) { if (encrypting) exf->encrypt(sw->sw_kschedule, inblk, outblk); else exf->decrypt(sw->sw_kschedule, inblk, outblk); } else if (encrypting) { /* XOR with previous block */ for (i = 0; i < blks; i++) outblk[i] = inblk[i] ^ ivp[i]; exf->encrypt(sw->sw_kschedule, outblk, outblk); /* * Keep encrypted block for XOR'ing * with next block */ memcpy(iv, outblk, blks); ivp = iv; } else { /* decrypt */ /* * Keep encrypted block for XOR'ing * with next block */ nivp = (ivp == iv) ? iv2 : iv; memcpy(nivp, inblk, blks); exf->decrypt(sw->sw_kschedule, inblk, outblk); /* XOR with previous block */ for (i = 0; i < blks; i++) outblk[i] ^= ivp[i]; ivp = nivp; } if (inlen < blks) { inblk = crypto_cursor_segment(&cc_in, &inlen); } else { crypto_cursor_advance(&cc_in, blks); inlen -= blks; inblk += blks; } if (outlen < blks) { crypto_cursor_copyback(&cc_out, blks, blk); outblk = crypto_cursor_segment(&cc_out, &outlen); } else { crypto_cursor_advance(&cc_out, blks); outlen -= blks; outblk += blks; } resid -= blks; } /* Handle trailing partial block for stream ciphers. */ if (resid > 0) { KASSERT(exf->native_blocksize != 0, ("%s: partial block of %d bytes for cipher %s", __func__, i, exf->name)); KASSERT(exf->reinit != NULL, ("%s: partial block cipher %s without reinit hook", __func__, exf->name)); KASSERT(resid < blks, ("%s: partial block too big", __func__)); inblk = crypto_cursor_segment(&cc_in, &inlen); outblk = crypto_cursor_segment(&cc_out, &outlen); if (inlen < resid) { crypto_cursor_copydata(&cc_in, resid, blk); inblk = blk; } if (outlen < resid) outblk = blk; if (encrypting) exf->encrypt_last(sw->sw_kschedule, inblk, outblk, resid); else exf->decrypt_last(sw->sw_kschedule, inblk, outblk, resid); if (outlen < resid) crypto_cursor_copyback(&cc_out, resid, blk); } explicit_bzero(blk, sizeof(blk)); explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv2, sizeof(iv2)); return (0); } static void swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw, const uint8_t *key, int klen) { switch (axf->type) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: hmac_init_ipad(axf, key, klen, sw->sw_ictx); hmac_init_opad(axf, key, klen, sw->sw_octx); break; case CRYPTO_POLY1305: case CRYPTO_BLAKE2B: case CRYPTO_BLAKE2S: axf->Setkey(sw->sw_ictx, key, klen); axf->Init(sw->sw_ictx); break; default: panic("%s: algorithm %d doesn't use keys", __func__, axf->type); } } /* * Compute or verify hash. */ static int swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) { u_char aalg[HASH_MAX_LEN]; const struct crypto_session_params *csp; struct swcr_auth *sw; struct auth_hash *axf; union authctx ctx; int err; sw = &ses->swcr_auth; axf = sw->sw_axf; csp = crypto_get_params(crp->crp_session); if (crp->crp_auth_key != NULL) { swcr_authprepare(axf, sw, crp->crp_auth_key, csp->csp_auth_klen); } bcopy(sw->sw_ictx, &ctx, axf->ctxsize); if (crp->crp_aad != NULL) err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (err) goto out; if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) err = crypto_apply_buf(&crp->crp_obuf, crp->crp_payload_output_start, crp->crp_payload_length, axf->Update, &ctx); else err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); if (err) goto out; if (csp->csp_flags & CSP_F_ESN) axf->Update(&ctx, crp->crp_esn, 4); axf->Final(aalg, &ctx); if (sw->sw_octx != NULL) { bcopy(sw->sw_octx, &ctx, axf->ctxsize); axf->Update(&ctx, aalg, axf->hashsize); axf->Final(aalg, &ctx); } if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char uaalg[HASH_MAX_LEN]; crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) err = EBADMSG; explicit_bzero(uaalg, sizeof(uaalg)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); } explicit_bzero(aalg, sizeof(aalg)); out: explicit_bzero(&ctx, sizeof(ctx)); return (err); } CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ static int swcr_gmac(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[GMAC_DIGEST_LEN]; u_char iv[AES_BLOCK_LEN]; struct crypto_buffer_cursor cc; const u_char *inblk; union authctx ctx; struct swcr_auth *swa; struct auth_hash *axf; uint32_t *blkp; size_t len; int blksz, error, ivlen, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = GMAC_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); /* Initialize the IV */ ivlen = AES_GCM_IV_LEN; crypto_read_iv(crp, iv); axf->Reinit(&ctx, iv, ivlen); crypto_cursor_init(&cc, &crp->crp_buf); crypto_cursor_advance(&cc, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { inblk = crypto_cursor_segment(&cc, &len); if (len >= blksz) { len = rounddown(MIN(len, resid), blksz); crypto_cursor_advance(&cc, len); } else { len = blksz; crypto_cursor_copydata(&cc, len, blk); inblk = blk; } axf->Update(&ctx, inblk, len); } if (resid > 0) { memset(blk, 0, blksz); crypto_cursor_copydata(&cc, resid, blk); axf->Update(&ctx, blk, blksz); } /* length block */ memset(blk, 0, blksz); blkp = (uint32_t *)blk + 1; *blkp = htobe32(crp->crp_payload_length * 8); axf->Update(&ctx, blk, blksz); /* Finalize MAC */ axf->Final(tag, &ctx); error = 0; if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char tag2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) error = EBADMSG; explicit_bzero(tag2, sizeof(tag2)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_gcm(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[GMAC_DIGEST_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; struct auth_hash *axf; struct enc_xform *exf; uint32_t *blkp; size_t len; int blksz, error, ivlen, r, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = GMAC_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); swe = &ses->swcr_encdec; exf = swe->sw_exf; KASSERT(axf->blocksize == exf->native_blocksize, ("%s: blocksize mismatch", __func__)); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); ivlen = AES_GCM_IV_LEN; /* Supply MAC with IV */ axf->Reinit(&ctx, crp->crp_iv, ivlen); /* Supply MAC with AAD */ if (crp->crp_aad != NULL) { len = rounddown(crp->crp_aad_length, blksz); if (len != 0) axf->Update(&ctx, crp->crp_aad, len); if (crp->crp_aad_length != len) { memset(blk, 0, blksz); memcpy(blk, (char *)crp->crp_aad + len, crp->crp_aad_length - len); axf->Update(&ctx, blk, blksz); } } else { crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_aad_start); for (resid = crp->crp_aad_length; resid >= blksz; resid -= len) { inblk = crypto_cursor_segment(&cc_in, &len); if (len >= blksz) { len = rounddown(MIN(len, resid), blksz); crypto_cursor_advance(&cc_in, len); } else { len = blksz; crypto_cursor_copydata(&cc_in, len, blk); inblk = blk; } axf->Update(&ctx, inblk, len); } if (resid > 0) { memset(blk, 0, blksz); crypto_cursor_copydata(&cc_in, resid, blk); axf->Update(&ctx, blk, blksz); } } if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, crypto_get_params(crp->crp_session)->csp_cipher_klen); exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); /* Do encryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else { crypto_cursor_advance(&cc_in, blksz); } if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->encrypt(swe->sw_kschedule, inblk, outblk); axf->Update(&ctx, outblk, blksz); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { axf->Update(&ctx, inblk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } axf->Update(&ctx, blk, resid); } /* length block */ memset(blk, 0, blksz); blkp = (uint32_t *)blk + 1; *blkp = htobe32(crp->crp_aad_length * 8); blkp = (uint32_t *)blk + 3; *blkp = htobe32(crp->crp_payload_length * 8); axf->Update(&ctx, blk, blksz); /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); return (error); } static void build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, u_int data_length, u_int tag_length, uint8_t *b0) { uint8_t *bp; uint8_t flags, L; KASSERT(nonce_length >= 7 && nonce_length <= 13, ("nonce_length must be between 7 and 13 bytes")); /* * Need to determine the L field value. This is the number of * bytes needed to specify the length of the message; the length * is whatever is left in the 16 bytes after specifying flags and * the nonce. */ L = 15 - nonce_length; flags = ((aad_length > 0) << 6) + (((tag_length - 2) / 2) << 3) + L - 1; /* * Now we need to set up the first block, which has flags, nonce, * and the message length. */ b0[0] = flags; memcpy(b0 + 1, nonce, nonce_length); bp = b0 + 1 + nonce_length; /* Need to copy L' [aka L-1] bytes of data_length */ for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { *dst = data_length; data_length >>= 8; } } /* NB: OCF only supports AAD lengths < 2^32. */ static int build_ccm_aad_length(u_int aad_length, uint8_t *blk) { if (aad_length < ((1 << 16) - (1 << 8))) { be16enc(blk, aad_length); return (sizeof(uint16_t)); } else { blk[0] = 0xff; blk[1] = 0xfe; be32enc(blk + 2, aad_length); return (2 + sizeof(uint32_t)); } } static int swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) { u_char iv[AES_BLOCK_LEN]; u_char blk[CCM_CBC_BLOCK_LEN]; u_char tag[AES_CBC_MAC_HASH_LEN]; union authctx ctx; const struct crypto_session_params *csp; struct swcr_auth *swa; struct auth_hash *axf; int error, ivlen, len; csp = crypto_get_params(crp->crp_session); swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); /* Initialize the IV */ ivlen = csp->csp_ivlen; crypto_read_iv(crp, iv); /* Supply MAC with IV */ axf->Reinit(&ctx, crp->crp_iv, ivlen); /* Supply MAC with b0. */ build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, swa->sw_mlen, blk); axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); len = build_ccm_aad_length(crp->crp_payload_length, blk); axf->Update(&ctx, blk, len); crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); /* Finalize MAC */ axf->Final(tag, &ctx); error = 0; if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char tag2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) error = EBADMSG; explicit_bzero(tag2, sizeof(tag)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } explicit_bzero(tag, sizeof(tag)); explicit_bzero(blk, sizeof(blk)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_ccm(struct swcr_session *ses, struct cryptop *crp) { const struct crypto_session_params *csp; uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[AES_CBC_MAC_HASH_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; struct auth_hash *axf; struct enc_xform *exf; size_t len; int blksz, error, ivlen, r, resid; csp = crypto_get_params(crp->crp_session); swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = AES_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); swe = &ses->swcr_encdec; exf = swe->sw_exf; KASSERT(axf->blocksize == exf->native_blocksize, ("%s: blocksize mismatch", __func__)); if (crp->crp_payload_length > ccm_max_payload_length(csp)) return (EMSGSIZE); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); ivlen = csp->csp_ivlen; /* Supply MAC with IV */ axf->Reinit(&ctx, crp->crp_iv, ivlen); /* Supply MAC with b0. */ _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, "blkbuf too small for b0"); build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, crp->crp_payload_length, swa->sw_mlen, blk); axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); /* Supply MAC with AAD */ if (crp->crp_aad_length != 0) { len = build_ccm_aad_length(crp->crp_aad_length, blk); axf->Update(&ctx, blk, len); if (crp->crp_aad != NULL) axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); /* Pad the AAD (including length field) to a full block. */ len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; if (len != 0) { len = CCM_CBC_BLOCK_LEN - len; memset(blk, 0, CCM_CBC_BLOCK_LEN); axf->Update(&ctx, blk, len); } } if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, crypto_get_params(crp->crp_session)->csp_cipher_klen); exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); /* Do encryption/decryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; axf->Update(&ctx, inblk, blksz); exf->encrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { /* * One of the problems with CCM+CBC is that * the authentication is done on the * unencrypted data. As a result, we have to * decrypt the data twice: once to generate * the tag and a second time after the tag is * verified. */ exf->decrypt(swe->sw_kschedule, inblk, blk); axf->Update(&ctx, blk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { axf->Update(&ctx, blk, resid); exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } else { exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); axf->Update(&ctx, blk, resid); } } /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); return (error); } static int swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) { const struct crypto_session_params *csp; uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[POLY1305_HASH_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; uint64_t *blkp; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; struct auth_hash *axf; struct enc_xform *exf; size_t len; int blksz, error, r, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; swe = &ses->swcr_encdec; exf = swe->sw_exf; blksz = exf->native_blocksize; KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); csp = crypto_get_params(crp->crp_session); /* Generate Poly1305 key. */ if (crp->crp_cipher_key != NULL) axf->Setkey(&ctx, crp->crp_cipher_key, csp->csp_cipher_klen); else axf->Setkey(&ctx, csp->csp_cipher_key, csp->csp_cipher_klen); axf->Reinit(&ctx, crp->crp_iv, csp->csp_ivlen); /* Supply MAC with AAD */ if (crp->crp_aad != NULL) axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (crp->crp_aad_length % 16 != 0) { /* padding1 */ memset(blk, 0, 16); axf->Update(&ctx, blk, 16 - crp->crp_aad_length % 16); } if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, csp->csp_cipher_klen); exf->reinit(swe->sw_kschedule, crp->crp_iv, csp->csp_ivlen); /* Do encryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->encrypt(swe->sw_kschedule, inblk, outblk); axf->Update(&ctx, outblk, blksz); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { axf->Update(&ctx, inblk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } axf->Update(&ctx, blk, resid); if (resid % 16 != 0) { /* padding2 */ memset(blk, 0, 16); axf->Update(&ctx, blk, 16 - resid % 16); } } /* lengths */ blkp = (uint64_t *)blk; blkp[0] = htole64(crp->crp_aad_length); blkp[1] = htole64(crp->crp_payload_length); axf->Update(&ctx, blk, sizeof(uint64_t) * 2); /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[POLY1305_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(&ctx, sizeof(ctx)); return (error); } /* * Apply a cipher and a digest to perform EtA. */ static int swcr_eta(struct swcr_session *ses, struct cryptop *crp) { int error; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { error = swcr_encdec(ses, crp); if (error == 0) error = swcr_authcompute(ses, crp); } else { error = swcr_authcompute(ses, crp); if (error == 0) error = swcr_encdec(ses, crp); } return (error); } /* * Apply a compression/decompression algorithm */ static int swcr_compdec(struct swcr_session *ses, struct cryptop *crp) { uint8_t *data, *out; struct comp_algo *cxf; int adj; uint32_t result; cxf = ses->swcr_compdec.sw_cxf; /* We must handle the whole buffer of data in one time * then if there is not all the data in the mbuf, we must * copy in a buffer. */ data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); if (data == NULL) return (EINVAL); crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, data); if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) result = cxf->compress(data, crp->crp_payload_length, &out); else result = cxf->decompress(data, crp->crp_payload_length, &out); free(data, M_CRYPTO_DATA); if (result == 0) return (EINVAL); crp->crp_olen = result; /* Check the compressed size when doing compression */ if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { if (result >= crp->crp_payload_length) { /* Compression was useless, we lost time */ free(out, M_CRYPTO_DATA); return (0); } } /* Copy back the (de)compressed data. m_copyback is * extending the mbuf as necessary. */ crypto_copyback(crp, crp->crp_payload_start, result, out); if (result < crp->crp_payload_length) { switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: case CRYPTO_BUF_SINGLE_MBUF: adj = result - crp->crp_payload_length; m_adj(crp->crp_buf.cb_mbuf, adj); break; case CRYPTO_BUF_UIO: { struct uio *uio = crp->crp_buf.cb_uio; int ind; adj = crp->crp_payload_length - result; ind = uio->uio_iovcnt - 1; while (adj > 0 && ind >= 0) { if (adj < uio->uio_iov[ind].iov_len) { uio->uio_iov[ind].iov_len -= adj; break; } adj -= uio->uio_iov[ind].iov_len; uio->uio_iov[ind].iov_len = 0; ind--; uio->uio_iovcnt--; } } break; case CRYPTO_BUF_VMPAGE: adj = crp->crp_payload_length - result; crp->crp_buf.cb_vm_page_len -= adj; break; default: break; } } free(out, M_CRYPTO_DATA); return 0; } static int swcr_setup_cipher(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_encdec *swe; struct enc_xform *txf; int error; swe = &ses->swcr_encdec; txf = crypto_cipher(csp); if (txf->ctxsize != 0) { swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swe->sw_kschedule == NULL) return (ENOMEM); } if (csp->csp_cipher_key != NULL) { error = txf->setkey(swe->sw_kschedule, csp->csp_cipher_key, csp->csp_cipher_klen); if (error) return (error); } swe->sw_exf = txf; return (0); } static int swcr_setup_auth(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; struct auth_hash *axf; swa = &ses->swcr_auth; axf = crypto_auth_hash(csp); swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_octx == NULL) return (ENOBUFS); if (csp->csp_auth_key != NULL) { swcr_authprepare(axf, swa, csp->csp_auth_key, csp->csp_auth_klen); } if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: axf->Init(swa->sw_ictx); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_AES_NIST_GMAC: axf->Init(swa->sw_ictx); axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_gmac; break; case CRYPTO_POLY1305: case CRYPTO_BLAKE2B: case CRYPTO_BLAKE2S: /* * Blake2b and Blake2s support an optional key but do * not require one. */ if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); axf->Init(swa->sw_ictx); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_AES_CCM_CBC_MAC: axf->Init(swa->sw_ictx); axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_ccm_cbc_mac; break; } return (0); } static int swcr_setup_gcm(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; struct auth_hash *axf; - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); - /* First, setup the auth side. */ swa = &ses->swcr_auth; switch (csp->csp_cipher_klen * 8) { case 128: axf = &auth_hash_nist_gmac_aes_128; break; case 192: axf = &auth_hash_nist_gmac_aes_192; break; case 256: axf = &auth_hash_nist_gmac_aes_256; break; default: return (EINVAL); } swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); axf->Init(swa->sw_ictx); if (csp->csp_cipher_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, csp->csp_cipher_klen); /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static int swcr_setup_ccm(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; struct auth_hash *axf; /* First, setup the auth side. */ swa = &ses->swcr_auth; switch (csp->csp_cipher_klen * 8) { case 128: axf = &auth_hash_ccm_cbc_mac_128; break; case 192: axf = &auth_hash_ccm_cbc_mac_192; break; case 256: axf = &auth_hash_ccm_cbc_mac_256; break; default: return (EINVAL); } swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); axf->Init(swa->sw_ictx); if (csp->csp_cipher_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, csp->csp_cipher_klen); /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static int swcr_setup_chacha20_poly1305(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; struct auth_hash *axf; /* First, setup the auth side. */ swa = &ses->swcr_auth; axf = &auth_hash_chacha20_poly1305; swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; /* The auth state is regenerated for each nonce. */ /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static bool swcr_auth_supported(const struct crypto_session_params *csp) { struct auth_hash *axf; axf = crypto_auth_hash(csp); if (axf == NULL) return (false); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: break; case CRYPTO_AES_NIST_GMAC: switch (csp->csp_auth_klen * 8) { case 128: case 192: case 256: break; default: return (false); } if (csp->csp_auth_key == NULL) return (false); if (csp->csp_ivlen != AES_GCM_IV_LEN) return (false); break; case CRYPTO_POLY1305: if (csp->csp_auth_klen != POLY1305_KEY_LEN) return (false); break; case CRYPTO_AES_CCM_CBC_MAC: switch (csp->csp_auth_klen * 8) { case 128: case 192: case 256: break; default: return (false); } if (csp->csp_auth_key == NULL) return (false); break; } return (true); } static bool swcr_cipher_supported(const struct crypto_session_params *csp) { struct enc_xform *txf; txf = crypto_cipher(csp); if (txf == NULL) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && txf->ivsize != csp->csp_ivlen) return (false); return (true); } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) static int swcr_probesession(device_t dev, const struct crypto_session_params *csp) { if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: switch (csp->csp_cipher_alg) { case CRYPTO_DEFLATE_COMP: break; default: return (EINVAL); } break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: return (EINVAL); default: if (!swcr_cipher_supported(csp)) return (EINVAL); break; } break; case CSP_MODE_DIGEST: if (!swcr_auth_supported(csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: break; default: return (EINVAL); } break; case CSP_MODE_ETA: /* AEAD algorithms cannot be used for EtA. */ switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: return (EINVAL); } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: case CRYPTO_AES_CCM_CBC_MAC: return (EINVAL); } if (!swcr_cipher_supported(csp) || !swcr_auth_supported(csp)) return (EINVAL); break; default: return (EINVAL); } return (CRYPTODEV_PROBE_SOFTWARE); } /* * Generate a new software session. */ static int swcr_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct swcr_session *ses; struct swcr_encdec *swe; struct swcr_auth *swa; struct comp_algo *cxf; int error; ses = crypto_get_driver_session(cses); mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); error = 0; swe = &ses->swcr_encdec; swa = &ses->swcr_auth; switch (csp->csp_mode) { case CSP_MODE_COMPRESS: switch (csp->csp_cipher_alg) { case CRYPTO_DEFLATE_COMP: cxf = &comp_algo_deflate; break; #ifdef INVARIANTS default: panic("bad compression algo"); #endif } ses->swcr_compdec.sw_cxf = cxf; ses->swcr_process = swcr_compdec; break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_NULL_CBC: ses->swcr_process = swcr_null; break; #ifdef INVARIANTS case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: panic("bad cipher algo"); #endif default: error = swcr_setup_cipher(ses, csp); if (error == 0) ses->swcr_process = swcr_encdec; } break; case CSP_MODE_DIGEST: error = swcr_setup_auth(ses, csp); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: error = swcr_setup_gcm(ses, csp); if (error == 0) ses->swcr_process = swcr_gcm; break; case CRYPTO_AES_CCM_16: error = swcr_setup_ccm(ses, csp); if (error == 0) ses->swcr_process = swcr_ccm; break; case CRYPTO_CHACHA20_POLY1305: error = swcr_setup_chacha20_poly1305(ses, csp); if (error == 0) ses->swcr_process = swcr_chacha20_poly1305; break; #ifdef INVARIANTS default: panic("bad aead algo"); #endif } break; case CSP_MODE_ETA: #ifdef INVARIANTS switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: panic("bad eta cipher algo"); } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: case CRYPTO_AES_CCM_CBC_MAC: panic("bad eta auth algo"); } #endif error = swcr_setup_auth(ses, csp); if (error) break; if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { /* Effectively degrade to digest mode. */ ses->swcr_process = swcr_authcompute; break; } error = swcr_setup_cipher(ses, csp); if (error == 0) ses->swcr_process = swcr_eta; break; default: error = EINVAL; } if (error) swcr_freesession(dev, cses); return (error); } static void swcr_freesession(device_t dev, crypto_session_t cses) { struct swcr_session *ses; ses = crypto_get_driver_session(cses); mtx_destroy(&ses->swcr_lock); zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); } /* * Process a software request. */ static int swcr_process(device_t dev, struct cryptop *crp, int hint) { struct swcr_session *ses; ses = crypto_get_driver_session(crp->crp_session); mtx_lock(&ses->swcr_lock); crp->crp_etype = ses->swcr_process(ses, crp); mtx_unlock(&ses->swcr_lock); crypto_done(crp); return (0); } static void swcr_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "cryptosoft", -1) == NULL && BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) panic("cryptosoft: could not attach"); } static int swcr_probe(device_t dev) { device_set_desc(dev, "software crypto"); device_quiet(dev); return (BUS_PROBE_NOWILDCARD); } static int swcr_attach(device_t dev) { swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); if (swcr_id < 0) { device_printf(dev, "cannot initialize!"); return (ENXIO); } return (0); } static int swcr_detach(device_t dev) { crypto_unregister_all(swcr_id); return 0; } static device_method_t swcr_methods[] = { DEVMETHOD(device_identify, swcr_identify), DEVMETHOD(device_probe, swcr_probe), DEVMETHOD(device_attach, swcr_attach), DEVMETHOD(device_detach, swcr_detach), DEVMETHOD(cryptodev_probesession, swcr_probesession), DEVMETHOD(cryptodev_newsession, swcr_newsession), DEVMETHOD(cryptodev_freesession,swcr_freesession), DEVMETHOD(cryptodev_process, swcr_process), {0, 0}, }; static driver_t swcr_driver = { "cryptosoft", swcr_methods, 0, /* NB: no softc */ }; static devclass_t swcr_devclass; /* * NB: We explicitly reference the crypto module so we * get the necessary ordering when built as a loadable * module. This is required because we bundle the crypto * module code together with the cryptosoft driver (otherwise * normal module dependencies would handle things). */ extern int crypto_modevent(struct module *, int, void *); /* XXX where to attach */ DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); MODULE_VERSION(cryptosoft, 1); MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);