diff --git a/sys/crypto/blake2/blake2-sw.c b/sys/crypto/blake2/blake2-sw.c index b7291ac6ef46..449ef2be94f5 100644 --- a/sys/crypto/blake2/blake2-sw.c +++ b/sys/crypto/blake2/blake2-sw.c @@ -1,163 +1,163 @@ /* This file is in the public domain. */ #include __FBSDID("$FreeBSD$"); #include #include extern int blake2b_init_ref(blake2b_state *S, size_t outlen); extern int blake2b_init_param_ref(blake2b_state *S, const blake2b_param *P); extern int blake2b_init_key_ref(blake2b_state *S, size_t outlen, const void *key, size_t keylen); extern int blake2b_update_ref(blake2b_state *S, const uint8_t *in, size_t inlen); extern int blake2b_final_ref(blake2b_state *S, uint8_t *out, size_t outlen); extern int blake2b_ref(uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen); extern int blake2s_init_ref(blake2s_state *S, size_t outlen); extern int blake2s_init_param_ref(blake2s_state *S, const blake2s_param *P); extern int blake2s_init_key_ref(blake2s_state *S, size_t outlen, const void *key, size_t keylen); extern int blake2s_update_ref(blake2s_state *S, const uint8_t *in, size_t inlen); extern int blake2s_final_ref(blake2s_state *S, uint8_t *out, size_t outlen); extern int blake2s_ref(uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen); struct blake2b_xform_ctx { blake2b_state state; uint8_t key[BLAKE2B_KEYBYTES]; uint16_t klen; }; CTASSERT(sizeof(union authctx) >= sizeof(struct blake2b_xform_ctx)); static void blake2b_xform_init(void *vctx) { struct blake2b_xform_ctx *ctx = vctx; int rc; if (ctx->klen > 0) rc = blake2b_init_key_ref(&ctx->state, BLAKE2B_OUTBYTES, ctx->key, ctx->klen); else rc = blake2b_init_ref(&ctx->state, BLAKE2B_OUTBYTES); if (rc != 0) panic("blake2b_init_key: invalid arguments"); } static void blake2b_xform_setkey(void *vctx, const uint8_t *key, u_int klen) { struct blake2b_xform_ctx *ctx = vctx; if (klen > sizeof(ctx->key)) panic("invalid klen %u", (unsigned)klen); memcpy(ctx->key, key, klen); ctx->klen = klen; } static int blake2b_xform_update(void *vctx, const void *data, u_int len) { struct blake2b_xform_ctx *ctx = vctx; int rc; rc = blake2b_update_ref(&ctx->state, data, len); if (rc != 0) return (EINVAL); return (0); } static void blake2b_xform_final(uint8_t *out, void *vctx) { struct blake2b_xform_ctx *ctx = vctx; int rc; rc = blake2b_final_ref(&ctx->state, out, BLAKE2B_OUTBYTES); if (rc != 0) panic("blake2b_final: invalid"); } -struct auth_hash auth_hash_blake2b = { +const struct auth_hash auth_hash_blake2b = { .type = CRYPTO_BLAKE2B, .name = "Blake2b", .keysize = BLAKE2B_KEYBYTES, .hashsize = BLAKE2B_OUTBYTES, .ctxsize = sizeof(struct blake2b_xform_ctx), .Setkey = blake2b_xform_setkey, .Init = blake2b_xform_init, .Update = blake2b_xform_update, .Final = blake2b_xform_final, }; struct blake2s_xform_ctx { blake2s_state state; uint8_t key[BLAKE2S_KEYBYTES]; uint16_t klen; }; CTASSERT(sizeof(union authctx) >= sizeof(struct blake2s_xform_ctx)); static void blake2s_xform_init(void *vctx) { struct blake2s_xform_ctx *ctx = vctx; int rc; if (ctx->klen > 0) rc = blake2s_init_key_ref(&ctx->state, BLAKE2S_OUTBYTES, ctx->key, ctx->klen); else rc = blake2s_init_ref(&ctx->state, BLAKE2S_OUTBYTES); if (rc != 0) panic("blake2s_init_key: invalid arguments"); } static void blake2s_xform_setkey(void *vctx, const uint8_t *key, u_int klen) { struct blake2s_xform_ctx *ctx = vctx; if (klen > sizeof(ctx->key)) panic("invalid klen %u", (unsigned)klen); memcpy(ctx->key, key, klen); ctx->klen = klen; } static int blake2s_xform_update(void *vctx, const void *data, u_int len) { struct blake2s_xform_ctx *ctx = vctx; int rc; rc = blake2s_update_ref(&ctx->state, data, len); if (rc != 0) return (EINVAL); return (0); } static void blake2s_xform_final(uint8_t *out, void *vctx) { struct blake2s_xform_ctx *ctx = vctx; int rc; rc = blake2s_final_ref(&ctx->state, out, BLAKE2S_OUTBYTES); if (rc != 0) panic("blake2s_final: invalid"); } -struct auth_hash auth_hash_blake2s = { +const struct auth_hash auth_hash_blake2s = { .type = CRYPTO_BLAKE2S, .name = "Blake2s", .keysize = BLAKE2S_KEYBYTES, .hashsize = BLAKE2S_OUTBYTES, .ctxsize = sizeof(struct blake2s_xform_ctx), .Setkey = blake2s_xform_setkey, .Init = blake2s_xform_init, .Update = blake2s_xform_update, .Final = blake2s_xform_final, }; diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c index 7cc38b14f3fd..51679942c386 100644 --- a/sys/crypto/ccp/ccp.c +++ b/sys/crypto/ccp/ccp.c @@ -1,797 +1,797 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 Chelsio Communications, Inc. * Copyright (c) 2017 Conrad Meyer * All rights reserved. * Largely borrowed from ccr(4), Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #include #include #include #include #include "cryptodev_if.h" #include "ccp.h" #include "ccp_hardware.h" MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto"); /* * Need a global softc available for garbage random_source API, which lacks any * context pointer. It's also handy for debugging. */ struct ccp_softc *g_ccp_softc; bool g_debug_print = false; SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0, "Set to enable debugging log messages"); static struct pciid { uint32_t devid; const char *desc; } ccp_ids[] = { { 0x14561022, "AMD CCP-5a" }, { 0x14681022, "AMD CCP-5b" }, { 0x15df1022, "AMD CCP-5a" }, }; static struct random_source random_ccp = { .rs_ident = "AMD CCP TRNG", .rs_source = RANDOM_PURE_CCP, .rs_read = random_ccp_read, }; /* * ccp_populate_sglist() generates a scatter/gather list that covers the entire * crypto operation buffer. */ static int ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) { int error; sglist_reset(sg); switch (cb->cb_type) { case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_SINGLE_MBUF: error = sglist_append_single_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; case CRYPTO_BUF_VMPAGE: error = sglist_append_vmpages(sg, cb->cb_vm_page, cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; } return (error); } static int ccp_probe(device_t dev) { struct pciid *ip; uint32_t id; id = pci_get_devid(dev); for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) { if (id == ip->devid) { device_set_desc(dev, ip->desc); return (0); } } return (ENXIO); } static void ccp_initialize_queues(struct ccp_softc *sc) { struct ccp_queue *qp; size_t i; for (i = 0; i < nitems(sc->queues); i++) { qp = &sc->queues[i]; qp->cq_softc = sc; qp->cq_qindex = i; mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF); /* XXX - arbitrarily chosen sizes */ qp->cq_sg_crp = sglist_alloc(32, M_WAITOK); /* Two more SGEs than sg_crp to accommodate ipad. */ qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK); qp->cq_sg_dst = sglist_alloc(2, M_WAITOK); } } static void ccp_free_queues(struct ccp_softc *sc) { struct ccp_queue *qp; size_t i; for (i = 0; i < nitems(sc->queues); i++) { qp = &sc->queues[i]; mtx_destroy(&qp->cq_lock); sglist_free(qp->cq_sg_crp); sglist_free(qp->cq_sg_ulptx); sglist_free(qp->cq_sg_dst); } } static int ccp_attach(device_t dev) { struct ccp_softc *sc; int error; sc = device_get_softc(dev); sc->dev = dev; sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session), CRYPTOCAP_F_HARDWARE); if (sc->cid < 0) { device_printf(dev, "could not get crypto driver id\n"); return (ENXIO); } error = ccp_hw_attach(dev); if (error != 0) return (error); mtx_init(&sc->lock, "ccp", NULL, MTX_DEF); ccp_initialize_queues(sc); if (g_ccp_softc == NULL) { g_ccp_softc = sc; if ((sc->hw_features & VERSION_CAP_TRNG) != 0) random_source_register(&random_ccp); } return (0); } static int ccp_detach(device_t dev) { struct ccp_softc *sc; sc = device_get_softc(dev); mtx_lock(&sc->lock); sc->detaching = true; mtx_unlock(&sc->lock); crypto_unregister_all(sc->cid); if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0) random_source_deregister(&random_ccp); ccp_hw_detach(dev); ccp_free_queues(sc); if (g_ccp_softc == sc) g_ccp_softc = NULL; mtx_destroy(&sc->lock); return (0); } static void ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen) { union authctx auth_ctx; - struct auth_hash *axf; + const struct auth_hash *axf; u_int i; /* * If the key is larger than the block size, use the digest of * the key as the key instead. */ axf = s->hmac.auth_hash; if (klen > axf->blocksize) { axf->Init(&auth_ctx); axf->Update(&auth_ctx, key, klen); axf->Final(s->hmac.ipad, &auth_ctx); explicit_bzero(&auth_ctx, sizeof(auth_ctx)); klen = axf->hashsize; } else memcpy(s->hmac.ipad, key, klen); memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); for (i = 0; i < axf->blocksize; i++) { s->hmac.ipad[i] ^= HMAC_IPAD_VAL; s->hmac.opad[i] ^= HMAC_OPAD_VAL; } } static bool ccp_aes_check_keylen(int alg, int klen) { switch (klen * 8) { case 128: case 192: if (alg == CRYPTO_AES_XTS) return (false); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) return (false); break; default: return (false); } return (true); } static void ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen) { unsigned kbits; if (alg == CRYPTO_AES_XTS) kbits = (klen / 2) * 8; else kbits = klen * 8; switch (kbits) { case 128: s->blkcipher.cipher_type = CCP_AES_TYPE_128; break; case 192: s->blkcipher.cipher_type = CCP_AES_TYPE_192; break; case 256: s->blkcipher.cipher_type = CCP_AES_TYPE_256; break; default: panic("should not get here"); } s->blkcipher.key_len = klen; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); } static bool ccp_auth_supported(struct ccp_softc *sc, const struct crypto_session_params *csp) { if ((sc->hw_features & VERSION_CAP_SHA) == 0) return (false); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: if (csp->csp_auth_key == NULL) return (false); break; default: return (false); } return (true); } static bool ccp_cipher_supported(struct ccp_softc *sc, const struct crypto_session_params *csp) { if ((sc->hw_features & VERSION_CAP_AES) == 0) return (false); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (ccp_aes_check_keylen(csp->csp_cipher_alg, csp->csp_cipher_klen)); } static int ccp_probesession(device_t dev, const struct crypto_session_params *csp) { struct ccp_softc *sc; if (csp->csp_flags != 0) return (EINVAL); sc = device_get_softc(dev); switch (csp->csp_mode) { case CSP_MODE_DIGEST: if (!ccp_auth_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_CIPHER: if (!ccp_cipher_supported(sc, csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > AES_GMAC_HASH_LEN) return (EINVAL); if ((sc->hw_features & VERSION_CAP_AES) == 0) return (EINVAL); break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!ccp_auth_supported(sc, csp) || !ccp_cipher_supported(sc, csp)) return (EINVAL); break; default: return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } static int ccp_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ccp_softc *sc; struct ccp_session *s; - struct auth_hash *auth_hash; + const struct auth_hash *auth_hash; enum ccp_aes_mode cipher_mode; unsigned auth_mode; unsigned q; /* XXX reconcile auth_mode with use by ccp_sha */ switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: auth_hash = &auth_hash_hmac_sha1; auth_mode = SHA1; break; case CRYPTO_SHA2_256_HMAC: auth_hash = &auth_hash_hmac_sha2_256; auth_mode = SHA2_256; break; case CRYPTO_SHA2_384_HMAC: auth_hash = &auth_hash_hmac_sha2_384; auth_mode = SHA2_384; break; case CRYPTO_SHA2_512_HMAC: auth_hash = &auth_hash_hmac_sha2_512; auth_mode = SHA2_512; break; default: auth_hash = NULL; auth_mode = 0; break; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: cipher_mode = CCP_AES_MODE_CBC; break; case CRYPTO_AES_ICM: cipher_mode = CCP_AES_MODE_CTR; break; case CRYPTO_AES_NIST_GCM_16: cipher_mode = CCP_AES_MODE_GCTR; break; case CRYPTO_AES_XTS: cipher_mode = CCP_AES_MODE_XTS; break; default: cipher_mode = CCP_AES_MODE_ECB; break; } sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); return (ENXIO); } s = crypto_get_driver_session(cses); /* Just grab the first usable queue for now. */ for (q = 0; q < nitems(sc->queues); q++) if ((sc->valid_queues & (1 << q)) != 0) break; if (q == nitems(sc->queues)) { mtx_unlock(&sc->lock); return (ENXIO); } s->queue = q; switch (csp->csp_mode) { case CSP_MODE_AEAD: s->mode = GCM; break; case CSP_MODE_ETA: s->mode = AUTHENC; break; case CSP_MODE_DIGEST: s->mode = HMAC; break; case CSP_MODE_CIPHER: s->mode = BLKCIPHER; break; } if (s->mode == GCM) { if (csp->csp_auth_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = csp->csp_auth_mlen; } else if (auth_hash != NULL) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; if (csp->csp_auth_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else s->hmac.hash_len = csp->csp_auth_mlen; ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen); } if (cipher_mode != CCP_AES_MODE_ECB) { s->blkcipher.cipher_mode = cipher_mode; if (csp->csp_cipher_key != NULL) ccp_aes_setkey(s, csp->csp_cipher_alg, csp->csp_cipher_key, csp->csp_cipher_klen); } s->active = true; mtx_unlock(&sc->lock); return (0); } static void ccp_freesession(device_t dev, crypto_session_t cses) { struct ccp_session *s; s = crypto_get_driver_session(cses); if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); s->active = false; } static int ccp_process(device_t dev, struct cryptop *crp, int hint) { const struct crypto_session_params *csp; struct ccp_softc *sc; struct ccp_queue *qp; struct ccp_session *s; int error; bool qpheld; qpheld = false; qp = NULL; csp = crypto_get_params(crp->crp_session); s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); mtx_lock(&sc->lock); qp = &sc->queues[s->queue]; mtx_unlock(&sc->lock); error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT); if (error != 0) goto out; qpheld = true; error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf); if (error != 0) goto out; if (crp->crp_auth_key != NULL) { KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC")); ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen); } if (crp->crp_cipher_key != NULL) ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key, csp->csp_cipher_klen); switch (s->mode) { case HMAC: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_hmac(qp, s, crp); break; case BLKCIPHER: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_blkcipher(qp, s, crp); break; case AUTHENC: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_authenc(qp, s, crp); break; case GCM: if (s->pending != 0) { error = EAGAIN; break; } error = ccp_gcm(qp, s, crp); break; } if (error == 0) s->pending++; out: if (qpheld) { if (error != 0) { /* * Squash EAGAIN so callers don't uselessly and * expensively retry if the ring was full. */ if (error == EAGAIN) error = ENOMEM; ccp_queue_abort(qp); } else ccp_queue_release(qp); } if (error != 0) { DPRINTF(dev, "%s: early error:%d\n", __func__, error); crp->crp_etype = error; crypto_done(crp); } return (0); } static device_method_t ccp_methods[] = { DEVMETHOD(device_probe, ccp_probe), DEVMETHOD(device_attach, ccp_attach), DEVMETHOD(device_detach, ccp_detach), DEVMETHOD(cryptodev_probesession, ccp_probesession), DEVMETHOD(cryptodev_newsession, ccp_newsession), DEVMETHOD(cryptodev_freesession, ccp_freesession), DEVMETHOD(cryptodev_process, ccp_process), DEVMETHOD_END }; static driver_t ccp_driver = { "ccp", ccp_methods, sizeof(struct ccp_softc) }; static devclass_t ccp_devclass; DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL); MODULE_VERSION(ccp, 1); MODULE_DEPEND(ccp, crypto, 1, 1, 1); MODULE_DEPEND(ccp, random_device, 1, 1, 1); #if 0 /* There are enough known issues that we shouldn't load automatically */ MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, nitems(ccp_ids)); #endif static int ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags) { struct ccp_softc *sc; mtx_assert(&qp->cq_lock, MA_OWNED); sc = qp->cq_softc; if (n < 1 || n >= (1 << sc->ring_size_order)) return (EINVAL); while (true) { if (ccp_queue_get_ring_space(qp) >= n) return (0); if ((mflags & M_WAITOK) == 0) return (EAGAIN); qp->cq_waiting = true; msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0); } } int ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags) { int error; mtx_lock(&qp->cq_lock); qp->cq_acq_tail = qp->cq_tail; error = ccp_queue_reserve_space(qp, n, mflags); if (error != 0) mtx_unlock(&qp->cq_lock); return (error); } void ccp_queue_release(struct ccp_queue *qp) { mtx_assert(&qp->cq_lock, MA_OWNED); if (qp->cq_tail != qp->cq_acq_tail) { wmb(); ccp_queue_write_tail(qp); } mtx_unlock(&qp->cq_lock); } void ccp_queue_abort(struct ccp_queue *qp) { unsigned i; mtx_assert(&qp->cq_lock, MA_OWNED); /* Wipe out any descriptors associated with this aborted txn. */ for (i = qp->cq_acq_tail; i != qp->cq_tail; i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) { memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i])); } qp->cq_tail = qp->cq_acq_tail; mtx_unlock(&qp->cq_lock); } #ifdef DDB #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) static void db_show_ccp_sc(struct ccp_softc *sc) { db_printf("ccp softc at %p\n", sc); db_printf(" cid: %d\n", (int)sc->cid); db_printf(" lock: "); db_show_lock(&sc->lock); db_printf(" detaching: %d\n", (int)sc->detaching); db_printf(" ring_size_order: %u\n", sc->ring_size_order); db_printf(" hw_version: %d\n", (int)sc->hw_version); db_printf(" hw_features: %b\n", (int)sc->hw_features, "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA" "\11SHA\0103DES\07AES"); db_printf(" hw status:\n"); db_ccp_show_hw(sc); } static void db_show_ccp_qp(struct ccp_queue *qp) { db_printf(" lock: "); db_show_lock(&qp->cq_lock); db_printf(" cq_qindex: %u\n", qp->cq_qindex); db_printf(" cq_softc: %p\n", qp->cq_softc); db_printf(" head: %u\n", qp->cq_head); db_printf(" tail: %u\n", qp->cq_tail); db_printf(" acq_tail: %u\n", qp->cq_acq_tail); db_printf(" desc_ring: %p\n", qp->desc_ring); db_printf(" completions_ring: %p\n", qp->completions_ring); db_printf(" descriptors (phys): 0x%jx\n", (uintmax_t)qp->desc_ring_bus_addr); db_printf(" hw status:\n"); db_ccp_show_queue_hw(qp); } DB_SHOW_COMMAND(ccp, db_show_ccp) { struct ccp_softc *sc; unsigned unit, qindex; if (!have_addr) goto usage; unit = (unsigned)addr; sc = devclass_get_softc(ccp_devclass, unit); if (sc == NULL) { db_printf("No such device ccp%u\n", unit); goto usage; } if (count == -1) { db_show_ccp_sc(sc); return; } qindex = (unsigned)count; if (qindex >= nitems(sc->queues)) { db_printf("No such queue %u\n", qindex); goto usage; } db_show_ccp_qp(&sc->queues[qindex]); return; usage: db_printf("usage: show ccp [,]\n"); return; } #endif /* DDB */ diff --git a/sys/crypto/ccp/ccp.h b/sys/crypto/ccp/ccp.h index 197cbc6b4c36..14e42a9e9d1f 100644 --- a/sys/crypto/ccp/ccp.h +++ b/sys/crypto/ccp/ccp.h @@ -1,260 +1,260 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 Conrad Meyer * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #pragma once /* * Keccak SHAKE128 (if supported by the device?) uses a 1344 bit block. * SHA3-224 is the next largest block size, at 1152 bits. However, crypto(4) * doesn't support any SHA3 hash, so SHA2 is the constraint: */ #define CCP_HASH_MAX_BLOCK_SIZE (SHA2_512_BLOCK_LEN) #define CCP_AES_MAX_KEY_LEN (AES_XTS_MAX_KEY) #define CCP_MAX_CRYPTO_IV_LEN 32 /* GCM IV + GHASH context */ #define MAX_HW_QUEUES 5 #define MAX_LSB_REGIONS 8 #ifndef __must_check #define __must_check __attribute__((__warn_unused_result__)) #endif /* * Internal data structures. */ enum sha_version { SHA1, #if 0 SHA2_224, #endif SHA2_256, SHA2_384, SHA2_512 }; /* * XXX: The hmac.res, gmac.final_block, and blkcipher.iv fields are * used by individual requests meaning that sessions cannot have more * than a single request in flight at a time. */ struct ccp_session_hmac { - struct auth_hash *auth_hash; + const struct auth_hash *auth_hash; int hash_len; unsigned int auth_mode; char ipad[CCP_HASH_MAX_BLOCK_SIZE]; char opad[CCP_HASH_MAX_BLOCK_SIZE]; char res[CCP_HASH_MAX_BLOCK_SIZE]; }; struct ccp_session_gmac { int hash_len; char final_block[GMAC_BLOCK_LEN]; }; struct ccp_session_blkcipher { unsigned cipher_mode; unsigned cipher_type; unsigned key_len; char enckey[CCP_AES_MAX_KEY_LEN]; char iv[CCP_MAX_CRYPTO_IV_LEN]; }; struct ccp_session { bool active; int pending; enum { HMAC, BLKCIPHER, AUTHENC, GCM } mode; unsigned queue; union { struct ccp_session_hmac hmac; struct ccp_session_gmac gmac; }; struct ccp_session_blkcipher blkcipher; }; struct ccp_softc; struct ccp_queue { struct mtx cq_lock; unsigned cq_qindex; struct ccp_softc *cq_softc; /* Host memory and tracking structures for descriptor ring. */ bus_dma_tag_t ring_desc_tag; bus_dmamap_t ring_desc_map; struct ccp_desc *desc_ring; bus_addr_t desc_ring_bus_addr; /* Callbacks and arguments ring; indices correspond to above ring. */ struct ccp_completion_ctx *completions_ring; uint32_t qcontrol; /* Cached register value */ unsigned lsb_mask; /* LSBs available to queue */ int private_lsb; /* Reserved LSB #, or -1 */ unsigned cq_head; unsigned cq_tail; unsigned cq_acq_tail; bool cq_waiting; /* Thread waiting for space */ struct sglist *cq_sg_crp; struct sglist *cq_sg_ulptx; struct sglist *cq_sg_dst; }; struct ccp_completion_ctx { void (*callback_fn)(struct ccp_queue *qp, struct ccp_session *s, void *arg, int error); void *callback_arg; struct ccp_session *session; }; struct ccp_softc { device_t dev; int32_t cid; struct mtx lock; bool detaching; unsigned ring_size_order; /* * Each command queue is either public or private. "Private" * (PSP-only) by default. PSP grants access to some queues to host via * QMR (Queue Mask Register). Set bits are host accessible. */ uint8_t valid_queues; uint8_t hw_version; uint8_t num_queues; uint16_t hw_features; uint16_t num_lsb_entries; /* Primary BAR (RID 2) used for register access */ bus_space_tag_t pci_bus_tag; bus_space_handle_t pci_bus_handle; int pci_resource_id; struct resource *pci_resource; /* Secondary BAR (RID 5) apparently used for MSI-X */ int pci_resource_id_msix; struct resource *pci_resource_msix; /* Interrupt resources */ void *intr_tag[2]; struct resource *intr_res[2]; unsigned intr_count; struct ccp_queue queues[MAX_HW_QUEUES]; }; /* Internal globals */ SYSCTL_DECL(_hw_ccp); MALLOC_DECLARE(M_CCP); extern bool g_debug_print; extern struct ccp_softc *g_ccp_softc; /* * Debug macros. */ #define DPRINTF(dev, ...) do { \ if (!g_debug_print) \ break; \ if ((dev) != NULL) \ device_printf((dev), "XXX " __VA_ARGS__); \ else \ printf("ccpXXX: " __VA_ARGS__); \ } while (0) #if 0 #define INSECURE_DEBUG(dev, ...) do { \ if (!g_debug_print) \ break; \ if ((dev) != NULL) \ device_printf((dev), "XXX " __VA_ARGS__); \ else \ printf("ccpXXX: " __VA_ARGS__); \ } while (0) #else #define INSECURE_DEBUG(dev, ...) #endif /* * Internal hardware manipulation routines. */ int ccp_hw_attach(device_t dev); void ccp_hw_detach(device_t dev); void ccp_queue_write_tail(struct ccp_queue *qp); #ifdef DDB void db_ccp_show_hw(struct ccp_softc *sc); void db_ccp_show_queue_hw(struct ccp_queue *qp); #endif /* * Internal hardware crypt-op submission routines. */ int ccp_authenc(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp) __must_check; int ccp_blkcipher(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp) __must_check; int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp) __must_check; int ccp_hmac(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp) __must_check; /* * Internal hardware TRNG read routine. */ u_int random_ccp_read(void *v, u_int c); /* XXX */ int ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags) __must_check; void ccp_queue_abort(struct ccp_queue *qp); void ccp_queue_release(struct ccp_queue *qp); /* * Internal inline routines. */ static inline unsigned ccp_queue_get_active(struct ccp_queue *qp) { struct ccp_softc *sc; sc = qp->cq_softc; return ((qp->cq_tail - qp->cq_head) & ((1 << sc->ring_size_order) - 1)); } static inline unsigned ccp_queue_get_ring_space(struct ccp_queue *qp) { struct ccp_softc *sc; sc = qp->cq_softc; return ((1 << sc->ring_size_order) - ccp_queue_get_active(qp) - 1); } diff --git a/sys/crypto/ccp/ccp_hardware.c b/sys/crypto/ccp/ccp_hardware.c index a2ca8e1cb71a..5175343ffc90 100644 --- a/sys/crypto/ccp/ccp_hardware.c +++ b/sys/crypto/ccp/ccp_hardware.c @@ -1,2107 +1,2107 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 Chelsio Communications, Inc. * Copyright (c) 2017 Conrad Meyer * All rights reserved. * Largely borrowed from ccr(4), Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "ccp.h" #include "ccp_hardware.h" #include "ccp_lsb.h" CTASSERT(sizeof(struct ccp_desc) == 32); static struct ccp_xts_unitsize_map_entry { enum ccp_xts_unitsize cxu_id; unsigned cxu_size; } ccp_xts_unitsize_map[] = { { CCP_XTS_AES_UNIT_SIZE_16, 16 }, { CCP_XTS_AES_UNIT_SIZE_512, 512 }, { CCP_XTS_AES_UNIT_SIZE_1024, 1024 }, { CCP_XTS_AES_UNIT_SIZE_2048, 2048 }, { CCP_XTS_AES_UNIT_SIZE_4096, 4096 }, }; SYSCTL_NODE(_hw, OID_AUTO, ccp, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "ccp node"); unsigned g_ccp_ring_order = 11; SYSCTL_UINT(_hw_ccp, OID_AUTO, ring_order, CTLFLAG_RDTUN, &g_ccp_ring_order, 0, "Set CCP ring order. (1 << this) == ring size. Min: 6, Max: 16"); /* * Zero buffer, sufficient for padding LSB entries, that does not span a page * boundary */ static const char g_zeroes[32] __aligned(32); static inline uint32_t ccp_read_4(struct ccp_softc *sc, uint32_t offset) { return (bus_space_read_4(sc->pci_bus_tag, sc->pci_bus_handle, offset)); } static inline void ccp_write_4(struct ccp_softc *sc, uint32_t offset, uint32_t value) { bus_space_write_4(sc->pci_bus_tag, sc->pci_bus_handle, offset, value); } static inline uint32_t ccp_read_queue_4(struct ccp_softc *sc, unsigned queue, uint32_t offset) { /* * Each queue gets its own 4kB register space. Queue 0 is at 0x1000. */ return (ccp_read_4(sc, (CMD_Q_STATUS_INCR * (1 + queue)) + offset)); } static inline void ccp_write_queue_4(struct ccp_softc *sc, unsigned queue, uint32_t offset, uint32_t value) { ccp_write_4(sc, (CMD_Q_STATUS_INCR * (1 + queue)) + offset, value); } void ccp_queue_write_tail(struct ccp_queue *qp) { ccp_write_queue_4(qp->cq_softc, qp->cq_qindex, CMD_Q_TAIL_LO_BASE, ((uint32_t)qp->desc_ring_bus_addr) + (Q_DESC_SIZE * qp->cq_tail)); } /* * Given a queue and a reserved LSB entry index, compute the LSB *entry id* of * that entry for the queue's private LSB region. */ static inline uint8_t ccp_queue_lsb_entry(struct ccp_queue *qp, unsigned lsb_entry) { return ((qp->private_lsb * LSB_REGION_LENGTH + lsb_entry)); } /* * Given a queue and a reserved LSB entry index, compute the LSB *address* of * that entry for the queue's private LSB region. */ static inline uint32_t ccp_queue_lsb_address(struct ccp_queue *qp, unsigned lsb_entry) { return (ccp_queue_lsb_entry(qp, lsb_entry) * LSB_ENTRY_SIZE); } /* * Some terminology: * * LSB - Local Storage Block * ========================= * * 8 segments/regions, each containing 16 entries. * * Each entry contains 256 bits (32 bytes). * * Segments are virtually addressed in commands, but accesses cannot cross * segment boundaries. Virtual map uses an identity mapping by default * (virtual segment N corresponds to physical segment N). * * Access to a physical region can be restricted to any subset of all five * queues. * * "Pass-through" mode * =================== * * Pass-through is a generic DMA engine, much like ioat(4). Some nice * features: * * - Supports byte-swapping for endian conversion (32- or 256-bit words) * - AND, OR, XOR with fixed 256-bit mask * - CRC32 of data (may be used in tandem with bswap, but not bit operations) * - Read/write of LSB * - Memset * * If bit manipulation mode is enabled, input must be a multiple of 256 bits * (32 bytes). * * If byte-swapping is enabled, input must be a multiple of the word size. * * Zlib mode -- only usable from one queue at a time, single job at a time. * ======================================================================== * * Only usable from private host, aka PSP? Not host processor? * * RNG. * ==== * * Raw bits are conditioned with AES and fed through CTR_DRBG. Output goes in * a ring buffer readable by software. * * NIST SP 800-90B Repetition Count and Adaptive Proportion health checks are * implemented on the raw input stream and may be enabled to verify min-entropy * of 0.5 bits per bit. */ static void ccp_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *baddr; KASSERT(error == 0, ("%s: error:%d", __func__, error)); baddr = arg; *baddr = segs->ds_addr; } static int ccp_hw_attach_queue(device_t dev, uint64_t lsbmask, unsigned queue) { struct ccp_softc *sc; struct ccp_queue *qp; void *desc; size_t ringsz, num_descriptors; int error; desc = NULL; sc = device_get_softc(dev); qp = &sc->queues[queue]; /* * Don't bother allocating a ring for queues the host isn't allowed to * drive. */ if ((sc->valid_queues & (1 << queue)) == 0) return (0); ccp_queue_decode_lsb_regions(sc, lsbmask, queue); /* Ignore queues that do not have any LSB access. */ if (qp->lsb_mask == 0) { device_printf(dev, "Ignoring queue %u with no LSB access\n", queue); sc->valid_queues &= ~(1 << queue); return (0); } num_descriptors = 1 << sc->ring_size_order; ringsz = sizeof(struct ccp_desc) * num_descriptors; /* * "Queue_Size" is order - 1. * * Queue must be aligned to 5+Queue_Size+1 == 5 + order bits. */ error = bus_dma_tag_create(bus_get_dma_tag(dev), 1 << (5 + sc->ring_size_order), #if defined(__i386__) && !defined(PAE) 0, BUS_SPACE_MAXADDR, #else (bus_addr_t)1 << 32, BUS_SPACE_MAXADDR_48BIT, #endif BUS_SPACE_MAXADDR, NULL, NULL, ringsz, 1, ringsz, 0, NULL, NULL, &qp->ring_desc_tag); if (error != 0) goto out; error = bus_dmamem_alloc(qp->ring_desc_tag, &desc, BUS_DMA_ZERO | BUS_DMA_WAITOK, &qp->ring_desc_map); if (error != 0) goto out; error = bus_dmamap_load(qp->ring_desc_tag, qp->ring_desc_map, desc, ringsz, ccp_dmamap_cb, &qp->desc_ring_bus_addr, BUS_DMA_WAITOK); if (error != 0) goto out; qp->desc_ring = desc; qp->completions_ring = malloc(num_descriptors * sizeof(*qp->completions_ring), M_CCP, M_ZERO | M_WAITOK); /* Zero control register; among other things, clears the RUN flag. */ qp->qcontrol = 0; ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol); ccp_write_queue_4(sc, queue, CMD_Q_INT_ENABLE_BASE, 0); /* Clear any leftover interrupt status flags */ ccp_write_queue_4(sc, queue, CMD_Q_INTERRUPT_STATUS_BASE, ALL_INTERRUPTS); qp->qcontrol |= (sc->ring_size_order - 1) << CMD_Q_SIZE_SHIFT; ccp_write_queue_4(sc, queue, CMD_Q_TAIL_LO_BASE, (uint32_t)qp->desc_ring_bus_addr); ccp_write_queue_4(sc, queue, CMD_Q_HEAD_LO_BASE, (uint32_t)qp->desc_ring_bus_addr); /* * Enable completion interrupts, as well as error or administrative * halt interrupts. We don't use administrative halts, but they * shouldn't trip unless we do, so it ought to be harmless. */ ccp_write_queue_4(sc, queue, CMD_Q_INT_ENABLE_BASE, INT_COMPLETION | INT_ERROR | INT_QUEUE_STOPPED); qp->qcontrol |= (qp->desc_ring_bus_addr >> 32) << CMD_Q_PTR_HI_SHIFT; qp->qcontrol |= CMD_Q_RUN; ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol); out: if (error != 0) { if (qp->desc_ring != NULL) bus_dmamap_unload(qp->ring_desc_tag, qp->ring_desc_map); if (desc != NULL) bus_dmamem_free(qp->ring_desc_tag, desc, qp->ring_desc_map); if (qp->ring_desc_tag != NULL) bus_dma_tag_destroy(qp->ring_desc_tag); } return (error); } static void ccp_hw_detach_queue(device_t dev, unsigned queue) { struct ccp_softc *sc; struct ccp_queue *qp; sc = device_get_softc(dev); qp = &sc->queues[queue]; /* * Don't bother allocating a ring for queues the host isn't allowed to * drive. */ if ((sc->valid_queues & (1 << queue)) == 0) return; free(qp->completions_ring, M_CCP); bus_dmamap_unload(qp->ring_desc_tag, qp->ring_desc_map); bus_dmamem_free(qp->ring_desc_tag, qp->desc_ring, qp->ring_desc_map); bus_dma_tag_destroy(qp->ring_desc_tag); } static int ccp_map_pci_bar(device_t dev) { struct ccp_softc *sc; sc = device_get_softc(dev); sc->pci_resource_id = PCIR_BAR(2); sc->pci_resource = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->pci_resource_id, RF_ACTIVE); if (sc->pci_resource == NULL) { device_printf(dev, "unable to allocate pci resource\n"); return (ENODEV); } sc->pci_resource_id_msix = PCIR_BAR(5); sc->pci_resource_msix = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->pci_resource_id_msix, RF_ACTIVE); if (sc->pci_resource_msix == NULL) { device_printf(dev, "unable to allocate pci resource msix\n"); bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id, sc->pci_resource); return (ENODEV); } sc->pci_bus_tag = rman_get_bustag(sc->pci_resource); sc->pci_bus_handle = rman_get_bushandle(sc->pci_resource); return (0); } static void ccp_unmap_pci_bar(device_t dev) { struct ccp_softc *sc; sc = device_get_softc(dev); bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id_msix, sc->pci_resource_msix); bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id, sc->pci_resource); } const static struct ccp_error_code { uint8_t ce_code; const char *ce_name; int ce_errno; const char *ce_desc; } ccp_error_codes[] = { { 0x01, "ILLEGAL_ENGINE", EIO, "Requested engine was invalid" }, { 0x03, "ILLEGAL_FUNCTION_TYPE", EIO, "A non-supported function type was specified" }, { 0x04, "ILLEGAL_FUNCTION_MODE", EIO, "A non-supported function mode was specified" }, { 0x05, "ILLEGAL_FUNCTION_ENCRYPT", EIO, "A CMAC type was specified when ENCRYPT was not specified" }, { 0x06, "ILLEGAL_FUNCTION_SIZE", EIO, "A non-supported function size was specified.\n" "AES-CFB: Size was not 127 or 7;\n" "3DES-CFB: Size was not 7;\n" "RSA: See supported size table (7.4.2);\n" "ECC: Size was greater than 576 bits." }, { 0x07, "Zlib_MISSING_INIT_EOM", EIO, "Zlib command does not have INIT and EOM set" }, { 0x08, "ILLEGAL_FUNCTION_RSVD", EIO, "Reserved bits in a function specification were not 0" }, { 0x09, "ILLEGAL_BUFFER_LENGTH", EIO, "The buffer length specified was not correct for the selected engine" }, { 0x0A, "VLSB_FAULT", EIO, "Illegal VLSB segment mapping:\n" "Undefined VLSB segment mapping or\n" "mapping to unsupported LSB segment id" }, { 0x0B, "ILLEGAL_MEM_ADDR", EFAULT, "The specified source/destination buffer access was illegal:\n" "Data buffer located in a LSB location disallowed by the LSB protection masks; or\n" "Data buffer not completely contained within a single segment; or\n" "Pointer with Fixed=1 is not 32-bit aligned; or\n" "Pointer with Fixed=1 attempted to reference non-AXI1 (local) memory." }, { 0x0C, "ILLEGAL_MEM_SEL", EIO, "A src_mem, dst_mem, or key_mem field was illegal:\n" "A field was set to a reserved value; or\n" "A public command attempted to reference AXI1 (local) or GART memory; or\n" "A Zlib command attmpted to use the LSB." }, { 0x0D, "ILLEGAL_CONTEXT_ADDR", EIO, "The specified context location was illegal:\n" "Context located in a LSB location disallowed by the LSB protection masks; or\n" "Context not completely contained within a single segment." }, { 0x0E, "ILLEGAL_KEY_ADDR", EIO, "The specified key location was illegal:\n" "Key located in a LSB location disallowed by the LSB protection masks; or\n" "Key not completely contained within a single segment." }, { 0x12, "CMD_TIMEOUT", EIO, "A command timeout violation occurred" }, /* XXX Could fill out these descriptions too */ { 0x13, "IDMA0_AXI_SLVERR", EIO, "" }, { 0x14, "IDMA0_AXI_DECERR", EIO, "" }, { 0x16, "IDMA1_AXI_SLVERR", EIO, "" }, { 0x17, "IDMA1_AXI_DECERR", EIO, "" }, { 0x19, "ZLIBVHB_AXI_SLVERR", EIO, "" }, { 0x1A, "ZLIBVHB_AXI_DECERR", EIO, "" }, { 0x1C, "ZLIB_UNEXPECTED_EOM", EIO, "" }, { 0x1D, "ZLIB_EXTRA_DATA", EIO, "" }, { 0x1E, "ZLIB_BTYPE", EIO, "" }, { 0x20, "ZLIB_UNDEFINED_DISTANCE_SYMBOL", EIO, "" }, { 0x21, "ZLIB_CODE_LENGTH_SYMBOL", EIO, "" }, { 0x22, "ZLIB_VHB_ILLEGAL_FETCH", EIO, "" }, { 0x23, "ZLIB_UNCOMPRESSED_LEN", EIO, "" }, { 0x24, "ZLIB_LIMIT_REACHED", EIO, "" }, { 0x25, "ZLIB_CHECKSUM_MISMATCH", EIO, "" }, { 0x26, "ODMA0_AXI_SLVERR", EIO, "" }, { 0x27, "ODMA0_AXI_DECERR", EIO, "" }, { 0x29, "ODMA1_AXI_SLVERR", EIO, "" }, { 0x2A, "ODMA1_AXI_DECERR", EIO, "" }, { 0x2B, "LSB_PARITY_ERR", EIO, "A read from the LSB encountered a parity error" }, }; static void ccp_intr_handle_error(struct ccp_queue *qp, const struct ccp_desc *desc) { struct ccp_completion_ctx *cctx; const struct ccp_error_code *ec; struct ccp_softc *sc; uint32_t status, error, esource, faultblock; unsigned q, idx; int errno; sc = qp->cq_softc; q = qp->cq_qindex; status = ccp_read_queue_4(sc, q, CMD_Q_STATUS_BASE); error = status & STATUS_ERROR_MASK; /* Decode error status */ ec = NULL; for (idx = 0; idx < nitems(ccp_error_codes); idx++) if (ccp_error_codes[idx].ce_code == error) { ec = &ccp_error_codes[idx]; break; } esource = (status >> STATUS_ERRORSOURCE_SHIFT) & STATUS_ERRORSOURCE_MASK; faultblock = (status >> STATUS_VLSB_FAULTBLOCK_SHIFT) & STATUS_VLSB_FAULTBLOCK_MASK; device_printf(sc->dev, "Error: %s (%u) Source: %u Faulting LSB block: %u\n", (ec != NULL) ? ec->ce_name : "(reserved)", error, esource, faultblock); if (ec != NULL) device_printf(sc->dev, "Error description: %s\n", ec->ce_desc); /* TODO Could format the desc nicely here */ idx = desc - qp->desc_ring; DPRINTF(sc->dev, "Bad descriptor index: %u contents: %32D\n", idx, (const void *)desc, " "); /* * TODO Per § 14.4 "Error Handling," DMA_Status, DMA_Read/Write_Status, * Zlib Decompress status may be interesting. */ while (true) { /* Keep unused descriptors zero for next use. */ memset(&qp->desc_ring[idx], 0, sizeof(qp->desc_ring[idx])); cctx = &qp->completions_ring[idx]; /* * Restart procedure described in § 14.2.5. Could be used by HoC if we * used that. * * Advance HEAD_LO past bad descriptor + any remaining in * transaction manually, then restart queue. */ idx = (idx + 1) % (1 << sc->ring_size_order); /* Callback function signals end of transaction */ if (cctx->callback_fn != NULL) { if (ec == NULL) errno = EIO; else errno = ec->ce_errno; /* TODO More specific error code */ cctx->callback_fn(qp, cctx->session, cctx->callback_arg, errno); cctx->callback_fn = NULL; break; } } qp->cq_head = idx; qp->cq_waiting = false; wakeup(&qp->cq_tail); DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head); ccp_write_queue_4(sc, q, CMD_Q_HEAD_LO_BASE, (uint32_t)qp->desc_ring_bus_addr + (idx * Q_DESC_SIZE)); ccp_write_queue_4(sc, q, CMD_Q_CONTROL_BASE, qp->qcontrol); DPRINTF(sc->dev, "%s: Restarted queue\n", __func__); } static void ccp_intr_run_completions(struct ccp_queue *qp, uint32_t ints) { struct ccp_completion_ctx *cctx; struct ccp_softc *sc; const struct ccp_desc *desc; uint32_t headlo, idx; unsigned q, completed; sc = qp->cq_softc; q = qp->cq_qindex; mtx_lock(&qp->cq_lock); /* * Hardware HEAD_LO points to the first incomplete descriptor. Process * any submitted and completed descriptors, up to but not including * HEAD_LO. */ headlo = ccp_read_queue_4(sc, q, CMD_Q_HEAD_LO_BASE); idx = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE; DPRINTF(sc->dev, "%s: hw head:%u sw head:%u\n", __func__, idx, qp->cq_head); completed = 0; while (qp->cq_head != idx) { DPRINTF(sc->dev, "%s: completing:%u\n", __func__, qp->cq_head); cctx = &qp->completions_ring[qp->cq_head]; if (cctx->callback_fn != NULL) { cctx->callback_fn(qp, cctx->session, cctx->callback_arg, 0); cctx->callback_fn = NULL; } /* Keep unused descriptors zero for next use. */ memset(&qp->desc_ring[qp->cq_head], 0, sizeof(qp->desc_ring[qp->cq_head])); qp->cq_head = (qp->cq_head + 1) % (1 << sc->ring_size_order); completed++; } if (completed > 0) { qp->cq_waiting = false; wakeup(&qp->cq_tail); } DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head); /* * Desc points to the first incomplete descriptor, at the time we read * HEAD_LO. If there was an error flagged in interrupt status, the HW * will not proceed past the erroneous descriptor by itself. */ desc = &qp->desc_ring[idx]; if ((ints & INT_ERROR) != 0) ccp_intr_handle_error(qp, desc); mtx_unlock(&qp->cq_lock); } static void ccp_intr_handler(void *arg) { struct ccp_softc *sc = arg; size_t i; uint32_t ints; DPRINTF(sc->dev, "%s: interrupt\n", __func__); /* * We get one global interrupt per PCI device, shared over all of * its queues. Scan each valid queue on interrupt for flags indicating * activity. */ for (i = 0; i < nitems(sc->queues); i++) { if ((sc->valid_queues & (1 << i)) == 0) continue; ints = ccp_read_queue_4(sc, i, CMD_Q_INTERRUPT_STATUS_BASE); if (ints == 0) continue; #if 0 DPRINTF(sc->dev, "%s: %x interrupts on queue %zu\n", __func__, (unsigned)ints, i); #endif /* Write back 1s to clear interrupt status bits. */ ccp_write_queue_4(sc, i, CMD_Q_INTERRUPT_STATUS_BASE, ints); /* * If there was an error, we still need to run completions on * any descriptors prior to the error. The completions handler * invoked below will also handle the error descriptor. */ if ((ints & (INT_COMPLETION | INT_ERROR)) != 0) ccp_intr_run_completions(&sc->queues[i], ints); if ((ints & INT_QUEUE_STOPPED) != 0) device_printf(sc->dev, "%s: queue %zu stopped\n", __func__, i); } /* Re-enable interrupts after processing */ for (i = 0; i < nitems(sc->queues); i++) { if ((sc->valid_queues & (1 << i)) == 0) continue; ccp_write_queue_4(sc, i, CMD_Q_INT_ENABLE_BASE, INT_COMPLETION | INT_ERROR | INT_QUEUE_STOPPED); } } static int ccp_intr_filter(void *arg) { struct ccp_softc *sc = arg; size_t i; /* TODO: Split individual queues into separate taskqueues? */ for (i = 0; i < nitems(sc->queues); i++) { if ((sc->valid_queues & (1 << i)) == 0) continue; /* Mask interrupt until task completes */ ccp_write_queue_4(sc, i, CMD_Q_INT_ENABLE_BASE, 0); } return (FILTER_SCHEDULE_THREAD); } static int ccp_setup_interrupts(struct ccp_softc *sc) { uint32_t nvec; int rid, error, n, ridcopy; n = pci_msix_count(sc->dev); if (n < 1) { device_printf(sc->dev, "%s: msix_count: %d\n", __func__, n); return (ENXIO); } nvec = n; error = pci_alloc_msix(sc->dev, &nvec); if (error != 0) { device_printf(sc->dev, "%s: alloc_msix error: %d\n", __func__, error); return (error); } if (nvec < 1) { device_printf(sc->dev, "%s: alloc_msix: 0 vectors\n", __func__); return (ENXIO); } if (nvec > nitems(sc->intr_res)) { device_printf(sc->dev, "%s: too many vectors: %u\n", __func__, nvec); nvec = nitems(sc->intr_res); } for (rid = 1; rid < 1 + nvec; rid++) { ridcopy = rid; sc->intr_res[rid - 1] = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &ridcopy, RF_ACTIVE); if (sc->intr_res[rid - 1] == NULL) { device_printf(sc->dev, "%s: Failed to alloc IRQ resource\n", __func__); return (ENXIO); } sc->intr_tag[rid - 1] = NULL; error = bus_setup_intr(sc->dev, sc->intr_res[rid - 1], INTR_MPSAFE | INTR_TYPE_MISC, ccp_intr_filter, ccp_intr_handler, sc, &sc->intr_tag[rid - 1]); if (error != 0) device_printf(sc->dev, "%s: setup_intr: %d\n", __func__, error); } sc->intr_count = nvec; return (error); } static void ccp_release_interrupts(struct ccp_softc *sc) { unsigned i; for (i = 0; i < sc->intr_count; i++) { if (sc->intr_tag[i] != NULL) bus_teardown_intr(sc->dev, sc->intr_res[i], sc->intr_tag[i]); if (sc->intr_res[i] != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, rman_get_rid(sc->intr_res[i]), sc->intr_res[i]); } pci_release_msi(sc->dev); } int ccp_hw_attach(device_t dev) { struct ccp_softc *sc; uint64_t lsbmask; uint32_t version, lsbmasklo, lsbmaskhi; unsigned queue_idx, j; int error; bool bars_mapped, interrupts_setup; queue_idx = 0; bars_mapped = interrupts_setup = false; sc = device_get_softc(dev); error = ccp_map_pci_bar(dev); if (error != 0) { device_printf(dev, "%s: couldn't map BAR(s)\n", __func__); goto out; } bars_mapped = true; error = pci_enable_busmaster(dev); if (error != 0) { device_printf(dev, "%s: couldn't enable busmaster\n", __func__); goto out; } sc->ring_size_order = g_ccp_ring_order; if (sc->ring_size_order < 6 || sc->ring_size_order > 16) { device_printf(dev, "bogus hw.ccp.ring_order\n"); error = EINVAL; goto out; } sc->valid_queues = ccp_read_4(sc, CMD_QUEUE_MASK_OFFSET); version = ccp_read_4(sc, VERSION_REG); if ((version & VERSION_NUM_MASK) < 5) { device_printf(dev, "driver supports version 5 and later hardware\n"); error = ENXIO; goto out; } error = ccp_setup_interrupts(sc); if (error != 0) goto out; interrupts_setup = true; sc->hw_version = version & VERSION_NUM_MASK; sc->num_queues = (version >> VERSION_NUMVQM_SHIFT) & VERSION_NUMVQM_MASK; sc->num_lsb_entries = (version >> VERSION_LSBSIZE_SHIFT) & VERSION_LSBSIZE_MASK; sc->hw_features = version & VERSION_CAP_MASK; /* * Copy private LSB mask to public registers to enable access to LSB * from all queues allowed by BIOS. */ lsbmasklo = ccp_read_4(sc, LSB_PRIVATE_MASK_LO_OFFSET); lsbmaskhi = ccp_read_4(sc, LSB_PRIVATE_MASK_HI_OFFSET); ccp_write_4(sc, LSB_PUBLIC_MASK_LO_OFFSET, lsbmasklo); ccp_write_4(sc, LSB_PUBLIC_MASK_HI_OFFSET, lsbmaskhi); lsbmask = ((uint64_t)lsbmaskhi << 30) | lsbmasklo; for (; queue_idx < nitems(sc->queues); queue_idx++) { error = ccp_hw_attach_queue(dev, lsbmask, queue_idx); if (error != 0) { device_printf(dev, "%s: couldn't attach queue %u\n", __func__, queue_idx); goto out; } } ccp_assign_lsb_regions(sc, lsbmask); out: if (error != 0) { if (interrupts_setup) ccp_release_interrupts(sc); for (j = 0; j < queue_idx; j++) ccp_hw_detach_queue(dev, j); if (sc->ring_size_order != 0) pci_disable_busmaster(dev); if (bars_mapped) ccp_unmap_pci_bar(dev); } return (error); } void ccp_hw_detach(device_t dev) { struct ccp_softc *sc; unsigned i; sc = device_get_softc(dev); for (i = 0; i < nitems(sc->queues); i++) ccp_hw_detach_queue(dev, i); ccp_release_interrupts(sc); pci_disable_busmaster(dev); ccp_unmap_pci_bar(dev); } static int __must_check ccp_passthrough(struct ccp_queue *qp, bus_addr_t dst, enum ccp_memtype dst_type, bus_addr_t src, enum ccp_memtype src_type, bus_size_t len, enum ccp_passthru_byteswap swapmode, enum ccp_passthru_bitwise bitmode, bool interrupt, const struct ccp_completion_ctx *cctx) { struct ccp_desc *desc; if (ccp_queue_get_ring_space(qp) == 0) return (EAGAIN); desc = &qp->desc_ring[qp->cq_tail]; memset(desc, 0, sizeof(*desc)); desc->engine = CCP_ENGINE_PASSTHRU; desc->pt.ioc = interrupt; desc->pt.byteswap = swapmode; desc->pt.bitwise = bitmode; desc->length = len; desc->src_lo = (uint32_t)src; desc->src_hi = src >> 32; desc->src_mem = src_type; desc->dst_lo = (uint32_t)dst; desc->dst_hi = dst >> 32; desc->dst_mem = dst_type; if (bitmode != CCP_PASSTHRU_BITWISE_NOOP) desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_KEY); if (cctx != NULL) memcpy(&qp->completions_ring[qp->cq_tail], cctx, sizeof(*cctx)); qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); return (0); } static int __must_check ccp_passthrough_sgl(struct ccp_queue *qp, bus_addr_t lsb_addr, bool tolsb, struct sglist *sgl, bus_size_t len, bool interrupt, const struct ccp_completion_ctx *cctx) { struct sglist_seg *seg; size_t i, remain, nb; int error; remain = len; for (i = 0; i < sgl->sg_nseg && remain != 0; i++) { seg = &sgl->sg_segs[i]; /* crp lengths are int, so 32-bit min() is ok. */ nb = min(remain, seg->ss_len); if (tolsb) error = ccp_passthrough(qp, lsb_addr, CCP_MEMTYPE_SB, seg->ss_paddr, CCP_MEMTYPE_SYSTEM, nb, CCP_PASSTHRU_BYTESWAP_NOOP, CCP_PASSTHRU_BITWISE_NOOP, (nb == remain) && interrupt, cctx); else error = ccp_passthrough(qp, seg->ss_paddr, CCP_MEMTYPE_SYSTEM, lsb_addr, CCP_MEMTYPE_SB, nb, CCP_PASSTHRU_BYTESWAP_NOOP, CCP_PASSTHRU_BITWISE_NOOP, (nb == remain) && interrupt, cctx); if (error != 0) return (error); remain -= nb; } return (0); } /* * Note that these vectors are in reverse of the usual order. */ const struct SHA_vectors { uint32_t SHA1[8]; uint32_t SHA224[8]; uint32_t SHA256[8]; uint64_t SHA384[8]; uint64_t SHA512[8]; } SHA_H __aligned(PAGE_SIZE) = { .SHA1 = { 0xc3d2e1f0ul, 0x10325476ul, 0x98badcfeul, 0xefcdab89ul, 0x67452301ul, 0, 0, 0, }, .SHA224 = { 0xbefa4fa4ul, 0x64f98fa7ul, 0x68581511ul, 0xffc00b31ul, 0xf70e5939ul, 0x3070dd17ul, 0x367cd507ul, 0xc1059ed8ul, }, .SHA256 = { 0x5be0cd19ul, 0x1f83d9abul, 0x9b05688cul, 0x510e527ful, 0xa54ff53aul, 0x3c6ef372ul, 0xbb67ae85ul, 0x6a09e667ul, }, .SHA384 = { 0x47b5481dbefa4fa4ull, 0xdb0c2e0d64f98fa7ull, 0x8eb44a8768581511ull, 0x67332667ffc00b31ull, 0x152fecd8f70e5939ull, 0x9159015a3070dd17ull, 0x629a292a367cd507ull, 0xcbbb9d5dc1059ed8ull, }, .SHA512 = { 0x5be0cd19137e2179ull, 0x1f83d9abfb41bd6bull, 0x9b05688c2b3e6c1full, 0x510e527fade682d1ull, 0xa54ff53a5f1d36f1ull, 0x3c6ef372fe94f82bull, 0xbb67ae8584caa73bull, 0x6a09e667f3bcc908ull, }, }; /* * Ensure vectors do not cross a page boundary. * * Disabled due to a new Clang error: "expression is not an integral constant * expression." GCC (cross toolchain) seems to handle this assertion with * _Static_assert just fine. */ #if 0 CTASSERT(PAGE_SIZE - ((uintptr_t)&SHA_H % PAGE_SIZE) >= sizeof(SHA_H)); #endif const struct SHA_Defn { enum sha_version version; const void *H_vectors; size_t H_size; - struct auth_hash *axf; + const struct auth_hash *axf; enum ccp_sha_type engine_type; } SHA_definitions[] = { { .version = SHA1, .H_vectors = SHA_H.SHA1, .H_size = sizeof(SHA_H.SHA1), .axf = &auth_hash_hmac_sha1, .engine_type = CCP_SHA_TYPE_1, }, #if 0 { .version = SHA2_224, .H_vectors = SHA_H.SHA224, .H_size = sizeof(SHA_H.SHA224), .axf = &auth_hash_hmac_sha2_224, .engine_type = CCP_SHA_TYPE_224, }, #endif { .version = SHA2_256, .H_vectors = SHA_H.SHA256, .H_size = sizeof(SHA_H.SHA256), .axf = &auth_hash_hmac_sha2_256, .engine_type = CCP_SHA_TYPE_256, }, { .version = SHA2_384, .H_vectors = SHA_H.SHA384, .H_size = sizeof(SHA_H.SHA384), .axf = &auth_hash_hmac_sha2_384, .engine_type = CCP_SHA_TYPE_384, }, { .version = SHA2_512, .H_vectors = SHA_H.SHA512, .H_size = sizeof(SHA_H.SHA512), .axf = &auth_hash_hmac_sha2_512, .engine_type = CCP_SHA_TYPE_512, }, }; static int __must_check ccp_sha_single_desc(struct ccp_queue *qp, const struct SHA_Defn *defn, vm_paddr_t addr, size_t len, bool start, bool end, uint64_t msgbits) { struct ccp_desc *desc; if (ccp_queue_get_ring_space(qp) == 0) return (EAGAIN); desc = &qp->desc_ring[qp->cq_tail]; memset(desc, 0, sizeof(*desc)); desc->engine = CCP_ENGINE_SHA; desc->som = start; desc->eom = end; desc->sha.type = defn->engine_type; desc->length = len; if (end) { desc->sha_len_lo = (uint32_t)msgbits; desc->sha_len_hi = msgbits >> 32; } desc->src_lo = (uint32_t)addr; desc->src_hi = addr >> 32; desc->src_mem = CCP_MEMTYPE_SYSTEM; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_SHA); qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); return (0); } static int __must_check ccp_sha(struct ccp_queue *qp, enum sha_version version, struct sglist *sgl_src, struct sglist *sgl_dst, const struct ccp_completion_ctx *cctx) { const struct SHA_Defn *defn; struct sglist_seg *seg; size_t i, msgsize, remaining, nb; uint32_t lsbaddr; int error; for (i = 0; i < nitems(SHA_definitions); i++) if (SHA_definitions[i].version == version) break; if (i == nitems(SHA_definitions)) return (EINVAL); defn = &SHA_definitions[i]; /* XXX validate input ??? */ /* Load initial SHA state into LSB */ /* XXX ensure H_vectors don't span page boundaries */ error = ccp_passthrough(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_SHA), CCP_MEMTYPE_SB, pmap_kextract((vm_offset_t)defn->H_vectors), CCP_MEMTYPE_SYSTEM, roundup2(defn->H_size, LSB_ENTRY_SIZE), CCP_PASSTHRU_BYTESWAP_NOOP, CCP_PASSTHRU_BITWISE_NOOP, false, NULL); if (error != 0) return (error); /* Execute series of SHA updates on correctly sized buffers */ msgsize = 0; for (i = 0; i < sgl_src->sg_nseg; i++) { seg = &sgl_src->sg_segs[i]; msgsize += seg->ss_len; error = ccp_sha_single_desc(qp, defn, seg->ss_paddr, seg->ss_len, i == 0, i == sgl_src->sg_nseg - 1, msgsize << 3); if (error != 0) return (error); } /* Copy result out to sgl_dst */ remaining = roundup2(defn->H_size, LSB_ENTRY_SIZE); lsbaddr = ccp_queue_lsb_address(qp, LSB_ENTRY_SHA); for (i = 0; i < sgl_dst->sg_nseg; i++) { seg = &sgl_dst->sg_segs[i]; /* crp lengths are int, so 32-bit min() is ok. */ nb = min(remaining, seg->ss_len); error = ccp_passthrough(qp, seg->ss_paddr, CCP_MEMTYPE_SYSTEM, lsbaddr, CCP_MEMTYPE_SB, nb, CCP_PASSTHRU_BYTESWAP_NOOP, CCP_PASSTHRU_BITWISE_NOOP, (cctx != NULL) ? (nb == remaining) : false, (nb == remaining) ? cctx : NULL); if (error != 0) return (error); remaining -= nb; lsbaddr += nb; if (remaining == 0) break; } return (0); } static void byteswap256(uint64_t *buffer) { uint64_t t; t = bswap64(buffer[3]); buffer[3] = bswap64(buffer[0]); buffer[0] = t; t = bswap64(buffer[2]); buffer[2] = bswap64(buffer[1]); buffer[1] = t; } /* * Translate CCP internal LSB hash format into a standard hash ouput. * * Manipulates input buffer with byteswap256 operation. */ static void ccp_sha_copy_result(char *output, char *buffer, enum sha_version version) { const struct SHA_Defn *defn; size_t i; for (i = 0; i < nitems(SHA_definitions); i++) if (SHA_definitions[i].version == version) break; if (i == nitems(SHA_definitions)) panic("bogus sha version auth_mode %u\n", (unsigned)version); defn = &SHA_definitions[i]; /* Swap 256bit manually -- DMA engine can, but with limitations */ byteswap256((void *)buffer); if (defn->axf->hashsize > LSB_ENTRY_SIZE) byteswap256((void *)(buffer + LSB_ENTRY_SIZE)); switch (defn->version) { case SHA1: memcpy(output, buffer + 12, defn->axf->hashsize); break; #if 0 case SHA2_224: memcpy(output, buffer + XXX, defn->axf->hashsize); break; #endif case SHA2_256: memcpy(output, buffer, defn->axf->hashsize); break; case SHA2_384: memcpy(output, buffer + LSB_ENTRY_SIZE * 3 - defn->axf->hashsize, defn->axf->hashsize - LSB_ENTRY_SIZE); memcpy(output + defn->axf->hashsize - LSB_ENTRY_SIZE, buffer, LSB_ENTRY_SIZE); break; case SHA2_512: memcpy(output, buffer + LSB_ENTRY_SIZE, LSB_ENTRY_SIZE); memcpy(output + LSB_ENTRY_SIZE, buffer, LSB_ENTRY_SIZE); break; } } static void ccp_do_hmac_done(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp, int error) { char ihash[SHA2_512_HASH_LEN /* max hash len */]; union authctx auth_ctx; - struct auth_hash *axf; + const struct auth_hash *axf; axf = s->hmac.auth_hash; s->pending--; if (error != 0) { crp->crp_etype = error; goto out; } /* Do remaining outer hash over small inner hash in software */ axf->Init(&auth_ctx); axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize); ccp_sha_copy_result(ihash, s->hmac.res, s->hmac.auth_mode); #if 0 INSECURE_DEBUG(dev, "%s sha intermediate=%64D\n", __func__, (u_char *)ihash, " "); #endif axf->Update(&auth_ctx, ihash, axf->hashsize); axf->Final(s->hmac.res, &auth_ctx); if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len, ihash); if (timingsafe_bcmp(s->hmac.res, ihash, s->hmac.hash_len) != 0) crp->crp_etype = EBADMSG; } else crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len, s->hmac.res); /* Avoid leaking key material */ explicit_bzero(&auth_ctx, sizeof(auth_ctx)); explicit_bzero(s->hmac.res, sizeof(s->hmac.res)); out: crypto_done(crp); } static void ccp_hmac_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp, int error) { struct cryptop *crp; crp = vcrp; ccp_do_hmac_done(qp, s, crp, error); } static int __must_check ccp_do_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp, const struct ccp_completion_ctx *cctx) { device_t dev; - struct auth_hash *axf; + const struct auth_hash *axf; int error; dev = qp->cq_softc->dev; axf = s->hmac.auth_hash; /* * Populate the SGL describing inside hash contents. We want to hash * the ipad (key XOR fixed bit pattern) concatenated with the user * data. */ sglist_reset(qp->cq_sg_ulptx); error = sglist_append(qp->cq_sg_ulptx, s->hmac.ipad, axf->blocksize); if (error != 0) return (error); if (crp->crp_aad_length != 0) { error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_aad_start, crp->crp_aad_length); if (error != 0) return (error); } error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_payload_start, crp->crp_payload_length); if (error != 0) { DPRINTF(dev, "%s: sglist too short\n", __func__); return (error); } /* Populate SGL for output -- use hmac.res buffer. */ sglist_reset(qp->cq_sg_dst); error = sglist_append(qp->cq_sg_dst, s->hmac.res, roundup2(axf->hashsize, LSB_ENTRY_SIZE)); if (error != 0) return (error); error = ccp_sha(qp, s->hmac.auth_mode, qp->cq_sg_ulptx, qp->cq_sg_dst, cctx); if (error != 0) { DPRINTF(dev, "%s: ccp_sha error\n", __func__); return (error); } return (0); } int __must_check ccp_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp) { struct ccp_completion_ctx ctx; ctx.callback_fn = ccp_hmac_done; ctx.callback_arg = crp; ctx.session = s; return (ccp_do_hmac(qp, s, crp, &ctx)); } static void ccp_byteswap(char *data, size_t len) { size_t i; char t; len--; for (i = 0; i < len; i++, len--) { t = data[i]; data[i] = data[len]; data[len] = t; } } static void ccp_blkcipher_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp, int error) { struct cryptop *crp; explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv)); crp = vcrp; s->pending--; if (error != 0) crp->crp_etype = error; DPRINTF(qp->cq_softc->dev, "%s: qp=%p crp=%p\n", __func__, qp, crp); crypto_done(crp); } static void ccp_collect_iv(struct cryptop *crp, const struct crypto_session_params *csp, char *iv) { crypto_read_iv(crp, iv); /* * If the input IV is 12 bytes, append an explicit counter of 1. */ if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 && csp->csp_ivlen == 12) *(uint32_t *)&iv[12] = htobe32(1); if (csp->csp_cipher_alg == CRYPTO_AES_XTS && csp->csp_ivlen < AES_BLOCK_LEN) memset(&iv[csp->csp_ivlen], 0, AES_BLOCK_LEN - csp->csp_ivlen); /* Reverse order of IV material for HW */ INSECURE_DEBUG(NULL, "%s: IV: %16D len: %u\n", __func__, iv, " ", csp->csp_ivlen); /* * For unknown reasons, XTS mode expects the IV in the reverse byte * order to every other AES mode. */ if (csp->csp_cipher_alg != CRYPTO_AES_XTS) ccp_byteswap(iv, AES_BLOCK_LEN); } static int __must_check ccp_do_pst_to_lsb(struct ccp_queue *qp, uint32_t lsbaddr, const void *src, size_t len) { int error; sglist_reset(qp->cq_sg_ulptx); error = sglist_append(qp->cq_sg_ulptx, __DECONST(void *, src), len); if (error != 0) return (error); error = ccp_passthrough_sgl(qp, lsbaddr, true, qp->cq_sg_ulptx, len, false, NULL); return (error); } static int __must_check ccp_do_xts(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp, enum ccp_cipher_dir dir, const struct ccp_completion_ctx *cctx) { struct ccp_desc *desc; device_t dev; unsigned i; enum ccp_xts_unitsize usize; /* IV and Key data are already loaded */ dev = qp->cq_softc->dev; for (i = 0; i < nitems(ccp_xts_unitsize_map); i++) if (ccp_xts_unitsize_map[i].cxu_size == crp->crp_payload_length) { usize = ccp_xts_unitsize_map[i].cxu_id; break; } if (i >= nitems(ccp_xts_unitsize_map)) return (EINVAL); for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) { struct sglist_seg *seg; seg = &qp->cq_sg_ulptx->sg_segs[i]; desc = &qp->desc_ring[qp->cq_tail]; desc->engine = CCP_ENGINE_XTS_AES; desc->som = (i == 0); desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1); desc->ioc = (desc->eom && cctx != NULL); DPRINTF(dev, "%s: XTS %u: som:%d eom:%d ioc:%d dir:%d\n", __func__, qp->cq_tail, (int)desc->som, (int)desc->eom, (int)desc->ioc, (int)dir); if (desc->ioc) memcpy(&qp->completions_ring[qp->cq_tail], cctx, sizeof(*cctx)); desc->aes_xts.encrypt = dir; desc->aes_xts.type = s->blkcipher.cipher_type; desc->aes_xts.size = usize; DPRINTF(dev, "XXX %s: XTS %u: type:%u size:%u\n", __func__, qp->cq_tail, (unsigned)desc->aes_xts.type, (unsigned)desc->aes_xts.size); desc->length = seg->ss_len; desc->src_lo = (uint32_t)seg->ss_paddr; desc->src_hi = (seg->ss_paddr >> 32); desc->src_mem = CCP_MEMTYPE_SYSTEM; /* Crypt in-place */ desc->dst_lo = desc->src_lo; desc->dst_hi = desc->src_hi; desc->dst_mem = desc->src_mem; desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY); desc->key_hi = 0; desc->key_mem = CCP_MEMTYPE_SB; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV); qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); } return (0); } static int __must_check ccp_do_blkcipher(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp, const struct ccp_completion_ctx *cctx) { const struct crypto_session_params *csp; struct ccp_desc *desc; char *keydata; device_t dev; enum ccp_cipher_dir dir; int error, iv_len; size_t keydata_len; unsigned i, j; dev = qp->cq_softc->dev; if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) { DPRINTF(dev, "%s: empty\n", __func__); return (EINVAL); } if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { DPRINTF(dev, "%s: len modulo: %d\n", __func__, crp->crp_payload_length); return (EINVAL); } /* * Individual segments must be multiples of AES block size for the HW * to process it. Non-compliant inputs aren't bogus, just not doable * on this hardware. */ for (i = 0; i < qp->cq_sg_crp->sg_nseg; i++) if ((qp->cq_sg_crp->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) { DPRINTF(dev, "%s: seg modulo: %zu\n", __func__, qp->cq_sg_crp->sg_segs[i].ss_len); return (EINVAL); } /* Gather IV/nonce data */ csp = crypto_get_params(crp->crp_session); ccp_collect_iv(crp, csp, s->blkcipher.iv); iv_len = csp->csp_ivlen; if (csp->csp_cipher_alg == CRYPTO_AES_XTS) iv_len = AES_BLOCK_LEN; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) dir = CCP_CIPHER_DIR_ENCRYPT; else dir = CCP_CIPHER_DIR_DECRYPT; /* Set up passthrough op(s) to copy IV into LSB */ error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV), s->blkcipher.iv, iv_len); if (error != 0) return (error); /* * Initialize keydata and keydata_len for GCC. The default case of the * following switch is impossible to reach, but GCC doesn't know that. */ keydata_len = 0; keydata = NULL; switch (csp->csp_cipher_alg) { case CRYPTO_AES_XTS: for (j = 0; j < nitems(ccp_xts_unitsize_map); j++) if (ccp_xts_unitsize_map[j].cxu_size == crp->crp_payload_length) break; /* Input buffer must be a supported UnitSize */ if (j >= nitems(ccp_xts_unitsize_map)) { device_printf(dev, "%s: rejected block size: %u\n", __func__, crp->crp_payload_length); return (EOPNOTSUPP); } /* FALLTHROUGH */ case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: keydata = s->blkcipher.enckey; keydata_len = s->blkcipher.key_len; break; } INSECURE_DEBUG(dev, "%s: KEY(%zu): %16D\n", __func__, keydata_len, keydata, " "); if (csp->csp_cipher_alg == CRYPTO_AES_XTS) INSECURE_DEBUG(dev, "%s: KEY(XTS): %64D\n", __func__, keydata, " "); /* Reverse order of key material for HW */ ccp_byteswap(keydata, keydata_len); /* Store key material into LSB to avoid page boundaries */ if (csp->csp_cipher_alg == CRYPTO_AES_XTS) { /* * XTS mode uses 2 256-bit vectors for the primary key and the * tweak key. For 128-bit keys, the vectors are zero-padded. * * After byteswapping the combined OCF-provided K1:K2 vector * above, we need to reverse the order again so the hardware * gets the swapped keys in the order K1':K2'. */ error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1), keydata, keydata_len / 2); if (error != 0) return (error); error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY), keydata + (keydata_len / 2), keydata_len / 2); /* Zero-pad 128 bit keys */ if (keydata_len == 32) { if (error != 0) return (error); error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY) + keydata_len / 2, g_zeroes, keydata_len / 2); if (error != 0) return (error); error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1) + keydata_len / 2, g_zeroes, keydata_len / 2); } } else error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY), keydata, keydata_len); if (error != 0) return (error); /* * Point SGLs at the subset of cryptop buffer contents representing the * data. */ sglist_reset(qp->cq_sg_ulptx); error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_payload_start, crp->crp_payload_length); if (error != 0) return (error); INSECURE_DEBUG(dev, "%s: Contents: %16D\n", __func__, (void *)PHYS_TO_DMAP(qp->cq_sg_ulptx->sg_segs[0].ss_paddr), " "); DPRINTF(dev, "%s: starting AES ops @ %u\n", __func__, qp->cq_tail); if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg) return (EAGAIN); if (csp->csp_cipher_alg == CRYPTO_AES_XTS) return (ccp_do_xts(qp, s, crp, dir, cctx)); for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) { struct sglist_seg *seg; seg = &qp->cq_sg_ulptx->sg_segs[i]; desc = &qp->desc_ring[qp->cq_tail]; desc->engine = CCP_ENGINE_AES; desc->som = (i == 0); desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1); desc->ioc = (desc->eom && cctx != NULL); DPRINTF(dev, "%s: AES %u: som:%d eom:%d ioc:%d dir:%d\n", __func__, qp->cq_tail, (int)desc->som, (int)desc->eom, (int)desc->ioc, (int)dir); if (desc->ioc) memcpy(&qp->completions_ring[qp->cq_tail], cctx, sizeof(*cctx)); desc->aes.encrypt = dir; desc->aes.mode = s->blkcipher.cipher_mode; desc->aes.type = s->blkcipher.cipher_type; if (csp->csp_cipher_alg == CRYPTO_AES_ICM) /* * Size of CTR value in bits, - 1. ICM mode uses all * 128 bits as counter. */ desc->aes.size = 127; DPRINTF(dev, "%s: AES %u: mode:%u type:%u size:%u\n", __func__, qp->cq_tail, (unsigned)desc->aes.mode, (unsigned)desc->aes.type, (unsigned)desc->aes.size); desc->length = seg->ss_len; desc->src_lo = (uint32_t)seg->ss_paddr; desc->src_hi = (seg->ss_paddr >> 32); desc->src_mem = CCP_MEMTYPE_SYSTEM; /* Crypt in-place */ desc->dst_lo = desc->src_lo; desc->dst_hi = desc->src_hi; desc->dst_mem = desc->src_mem; desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY); desc->key_hi = 0; desc->key_mem = CCP_MEMTYPE_SB; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV); qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); } return (0); } int __must_check ccp_blkcipher(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp) { struct ccp_completion_ctx ctx; ctx.callback_fn = ccp_blkcipher_done; ctx.session = s; ctx.callback_arg = crp; return (ccp_do_blkcipher(qp, s, crp, &ctx)); } static void ccp_authenc_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp, int error) { struct cryptop *crp; explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv)); crp = vcrp; ccp_do_hmac_done(qp, s, crp, error); } int __must_check ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp) { struct ccp_completion_ctx ctx; int error; ctx.callback_fn = ccp_authenc_done; ctx.session = s; ctx.callback_arg = crp; /* Perform first operation */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) error = ccp_do_blkcipher(qp, s, crp, NULL); else error = ccp_do_hmac(qp, s, crp, NULL); if (error != 0) return (error); /* Perform second operation */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) error = ccp_do_hmac(qp, s, crp, &ctx); else error = ccp_do_blkcipher(qp, s, crp, &ctx); return (error); } static int __must_check ccp_do_ghash_aad(struct ccp_queue *qp, struct ccp_session *s) { struct ccp_desc *desc; struct sglist_seg *seg; unsigned i; if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg) return (EAGAIN); for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) { seg = &qp->cq_sg_ulptx->sg_segs[i]; desc = &qp->desc_ring[qp->cq_tail]; desc->engine = CCP_ENGINE_AES; desc->aes.mode = CCP_AES_MODE_GHASH; desc->aes.type = s->blkcipher.cipher_type; desc->aes.encrypt = CCP_AES_MODE_GHASH_AAD; desc->som = (i == 0); desc->length = seg->ss_len; desc->src_lo = (uint32_t)seg->ss_paddr; desc->src_hi = (seg->ss_paddr >> 32); desc->src_mem = CCP_MEMTYPE_SYSTEM; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV); desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY); desc->key_mem = CCP_MEMTYPE_SB; qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); } return (0); } static int __must_check ccp_do_gctr(struct ccp_queue *qp, struct ccp_session *s, enum ccp_cipher_dir dir, struct sglist_seg *seg, bool som, bool eom) { struct ccp_desc *desc; if (ccp_queue_get_ring_space(qp) == 0) return (EAGAIN); desc = &qp->desc_ring[qp->cq_tail]; desc->engine = CCP_ENGINE_AES; desc->aes.mode = CCP_AES_MODE_GCTR; desc->aes.type = s->blkcipher.cipher_type; desc->aes.encrypt = dir; desc->aes.size = 8 * (seg->ss_len % GMAC_BLOCK_LEN) - 1; desc->som = som; desc->eom = eom; /* Trailing bytes will be masked off by aes.size above. */ desc->length = roundup2(seg->ss_len, GMAC_BLOCK_LEN); desc->dst_lo = desc->src_lo = (uint32_t)seg->ss_paddr; desc->dst_hi = desc->src_hi = seg->ss_paddr >> 32; desc->dst_mem = desc->src_mem = CCP_MEMTYPE_SYSTEM; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV); desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY); desc->key_mem = CCP_MEMTYPE_SB; qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); return (0); } static int __must_check ccp_do_ghash_final(struct ccp_queue *qp, struct ccp_session *s) { struct ccp_desc *desc; if (ccp_queue_get_ring_space(qp) == 0) return (EAGAIN); desc = &qp->desc_ring[qp->cq_tail]; desc->engine = CCP_ENGINE_AES; desc->aes.mode = CCP_AES_MODE_GHASH; desc->aes.type = s->blkcipher.cipher_type; desc->aes.encrypt = CCP_AES_MODE_GHASH_FINAL; desc->length = GMAC_BLOCK_LEN; desc->src_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN); desc->src_mem = CCP_MEMTYPE_SB; desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV); desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY); desc->key_mem = CCP_MEMTYPE_SB; desc->dst_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH); desc->dst_mem = CCP_MEMTYPE_SB; qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order); return (0); } static void ccp_gcm_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp, int error) { char tag[GMAC_DIGEST_LEN]; struct cryptop *crp; crp = vcrp; s->pending--; if (error != 0) { crp->crp_etype = error; goto out; } /* Encrypt is done. Decrypt needs to verify tag. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) goto out; /* Copy in message tag. */ crypto_copydata(crp, crp->crp_digest_start, s->gmac.hash_len, tag); /* Verify tag against computed GMAC */ if (timingsafe_bcmp(tag, s->gmac.final_block, s->gmac.hash_len) != 0) crp->crp_etype = EBADMSG; out: explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv)); explicit_bzero(&s->gmac.final_block, sizeof(s->gmac.final_block)); crypto_done(crp); } int __must_check ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp) { const struct crypto_session_params *csp; struct ccp_completion_ctx ctx; enum ccp_cipher_dir dir; device_t dev; unsigned i; int error; if (s->blkcipher.key_len == 0) return (EINVAL); dev = qp->cq_softc->dev; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) dir = CCP_CIPHER_DIR_ENCRYPT; else dir = CCP_CIPHER_DIR_DECRYPT; /* Zero initial GHASH portion of context */ memset(s->blkcipher.iv, 0, sizeof(s->blkcipher.iv)); /* Gather IV data */ csp = crypto_get_params(crp->crp_session); ccp_collect_iv(crp, csp, s->blkcipher.iv); /* Reverse order of key material for HW */ ccp_byteswap(s->blkcipher.enckey, s->blkcipher.key_len); /* Prepare input buffer of concatenated lengths for final GHASH */ be64enc(s->gmac.final_block, (uint64_t)crp->crp_aad_length * 8); be64enc(&s->gmac.final_block[8], (uint64_t)crp->crp_payload_length * 8); /* Send IV + initial zero GHASH, key data, and lengths buffer to LSB */ error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV), s->blkcipher.iv, 32); if (error != 0) return (error); error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY), s->blkcipher.enckey, s->blkcipher.key_len); if (error != 0) return (error); error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN), s->gmac.final_block, GMAC_BLOCK_LEN); if (error != 0) return (error); /* First step - compute GHASH over AAD */ if (crp->crp_aad_length != 0) { sglist_reset(qp->cq_sg_ulptx); error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_aad_start, crp->crp_aad_length); if (error != 0) return (error); /* This engine cannot process non-block multiple AAD data. */ for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) if ((qp->cq_sg_ulptx->sg_segs[i].ss_len % GMAC_BLOCK_LEN) != 0) { DPRINTF(dev, "%s: AD seg modulo: %zu\n", __func__, qp->cq_sg_ulptx->sg_segs[i].ss_len); return (EINVAL); } error = ccp_do_ghash_aad(qp, s); if (error != 0) return (error); } /* Feed data piece by piece into GCTR */ sglist_reset(qp->cq_sg_ulptx); error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_payload_start, crp->crp_payload_length); if (error != 0) return (error); /* * All segments except the last must be even multiples of AES block * size for the HW to process it. Non-compliant inputs aren't bogus, * just not doable on this hardware. * * XXX: Well, the hardware will produce a valid tag for shorter final * segment inputs, but it will still write out a block-sized plaintext * or ciphertext chunk. For a typical CRP this tramples trailing data, * including the provided message tag. So, reject such inputs for now. */ for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) if ((qp->cq_sg_ulptx->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) { DPRINTF(dev, "%s: seg modulo: %zu\n", __func__, qp->cq_sg_ulptx->sg_segs[i].ss_len); return (EINVAL); } for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) { struct sglist_seg *seg; seg = &qp->cq_sg_ulptx->sg_segs[i]; error = ccp_do_gctr(qp, s, dir, seg, (i == 0 && crp->crp_aad_length == 0), i == (qp->cq_sg_ulptx->sg_nseg - 1)); if (error != 0) return (error); } /* Send just initial IV (not GHASH!) to LSB again */ error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV), s->blkcipher.iv, AES_BLOCK_LEN); if (error != 0) return (error); ctx.callback_fn = ccp_gcm_done; ctx.session = s; ctx.callback_arg = crp; /* Compute final hash and copy result back */ error = ccp_do_ghash_final(qp, s); if (error != 0) return (error); /* When encrypting, copy computed tag out to caller buffer. */ sglist_reset(qp->cq_sg_ulptx); if (dir == CCP_CIPHER_DIR_ENCRYPT) error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp, crp->crp_digest_start, s->gmac.hash_len); else /* * For decrypting, copy the computed tag out to our session * buffer to verify in our callback. */ error = sglist_append(qp->cq_sg_ulptx, s->gmac.final_block, s->gmac.hash_len); if (error != 0) return (error); error = ccp_passthrough_sgl(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH), false, qp->cq_sg_ulptx, s->gmac.hash_len, true, &ctx); return (error); } #define MAX_TRNG_RETRIES 10 u_int random_ccp_read(void *v, u_int c) { uint32_t *buf; u_int i, j; KASSERT(c % sizeof(*buf) == 0, ("%u not multiple of u_long", c)); buf = v; for (i = c; i > 0; i -= sizeof(*buf)) { for (j = 0; j < MAX_TRNG_RETRIES; j++) { *buf = ccp_read_4(g_ccp_softc, TRNG_OUT_OFFSET); if (*buf != 0) break; } if (j == MAX_TRNG_RETRIES) return (0); buf++; } return (c); } #ifdef DDB void db_ccp_show_hw(struct ccp_softc *sc) { db_printf(" queue mask: 0x%x\n", ccp_read_4(sc, CMD_QUEUE_MASK_OFFSET)); db_printf(" queue prio: 0x%x\n", ccp_read_4(sc, CMD_QUEUE_PRIO_OFFSET)); db_printf(" reqid: 0x%x\n", ccp_read_4(sc, CMD_REQID_CONFIG_OFFSET)); db_printf(" trng output: 0x%x\n", ccp_read_4(sc, TRNG_OUT_OFFSET)); db_printf(" cmd timeout: 0x%x\n", ccp_read_4(sc, CMD_CMD_TIMEOUT_OFFSET)); db_printf(" lsb public mask lo: 0x%x\n", ccp_read_4(sc, LSB_PUBLIC_MASK_LO_OFFSET)); db_printf(" lsb public mask hi: 0x%x\n", ccp_read_4(sc, LSB_PUBLIC_MASK_HI_OFFSET)); db_printf(" lsb private mask lo: 0x%x\n", ccp_read_4(sc, LSB_PRIVATE_MASK_LO_OFFSET)); db_printf(" lsb private mask hi: 0x%x\n", ccp_read_4(sc, LSB_PRIVATE_MASK_HI_OFFSET)); db_printf(" version: 0x%x\n", ccp_read_4(sc, VERSION_REG)); } void db_ccp_show_queue_hw(struct ccp_queue *qp) { const struct ccp_error_code *ec; struct ccp_softc *sc; uint32_t status, error, esource, faultblock, headlo, qcontrol; unsigned q, i; sc = qp->cq_softc; q = qp->cq_qindex; qcontrol = ccp_read_queue_4(sc, q, CMD_Q_CONTROL_BASE); db_printf(" qcontrol: 0x%x%s%s\n", qcontrol, (qcontrol & CMD_Q_RUN) ? " RUN" : "", (qcontrol & CMD_Q_HALTED) ? " HALTED" : ""); db_printf(" tail_lo: 0x%x\n", ccp_read_queue_4(sc, q, CMD_Q_TAIL_LO_BASE)); headlo = ccp_read_queue_4(sc, q, CMD_Q_HEAD_LO_BASE); db_printf(" head_lo: 0x%x\n", headlo); db_printf(" int enable: 0x%x\n", ccp_read_queue_4(sc, q, CMD_Q_INT_ENABLE_BASE)); db_printf(" interrupt status: 0x%x\n", ccp_read_queue_4(sc, q, CMD_Q_INTERRUPT_STATUS_BASE)); status = ccp_read_queue_4(sc, q, CMD_Q_STATUS_BASE); db_printf(" status: 0x%x\n", status); db_printf(" int stats: 0x%x\n", ccp_read_queue_4(sc, q, CMD_Q_INT_STATUS_BASE)); error = status & STATUS_ERROR_MASK; if (error == 0) return; esource = (status >> STATUS_ERRORSOURCE_SHIFT) & STATUS_ERRORSOURCE_MASK; faultblock = (status >> STATUS_VLSB_FAULTBLOCK_SHIFT) & STATUS_VLSB_FAULTBLOCK_MASK; ec = NULL; for (i = 0; i < nitems(ccp_error_codes); i++) if (ccp_error_codes[i].ce_code == error) break; if (i < nitems(ccp_error_codes)) ec = &ccp_error_codes[i]; db_printf(" Error: %s (%u) Source: %u Faulting LSB block: %u\n", (ec != NULL) ? ec->ce_name : "(reserved)", error, esource, faultblock); if (ec != NULL) db_printf(" Error description: %s\n", ec->ce_desc); i = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE; db_printf(" Bad descriptor idx: %u contents:\n %32D\n", i, (void *)&qp->desc_ring[i], " "); } #endif diff --git a/sys/crypto/chacha20/chacha-sw.c b/sys/crypto/chacha20/chacha-sw.c index 2c28b9a0c459..b1bf0a106bfd 100644 --- a/sys/crypto/chacha20/chacha-sw.c +++ b/sys/crypto/chacha20/chacha-sw.c @@ -1,57 +1,57 @@ /* This file is in the public domain. */ #include __FBSDID("$FreeBSD$"); #include #include static int chacha20_xform_setkey(void *ctx, const uint8_t *key, int len) { if (len != CHACHA_MINKEYLEN && len != 32) return (EINVAL); chacha_keysetup(ctx, key, len * 8); return (0); } static void chacha20_xform_reinit(void *ctx, const uint8_t *iv) { chacha_ivsetup(ctx, iv + 8, iv); } static void chacha20_xform_crypt(void *ctx, const uint8_t *in, uint8_t *out) { chacha_encrypt_bytes(ctx, in, out, CHACHA_BLOCKLEN); } static void chacha20_xform_crypt_last(void *ctx, const uint8_t *in, uint8_t *out, size_t len) { chacha_encrypt_bytes(ctx, in, out, len); } -struct enc_xform enc_xform_chacha20 = { +const struct enc_xform enc_xform_chacha20 = { .type = CRYPTO_CHACHA20, .name = "chacha20", .ctxsize = sizeof(struct chacha_ctx), .blocksize = 1, .native_blocksize = CHACHA_BLOCKLEN, .ivsize = CHACHA_NONCELEN + CHACHA_CTRLEN, .minkey = CHACHA_MINKEYLEN, .maxkey = 32, .encrypt = chacha20_xform_crypt, .decrypt = chacha20_xform_crypt, .setkey = chacha20_xform_setkey, .reinit = chacha20_xform_reinit, .encrypt_last = chacha20_xform_crypt_last, .decrypt_last = chacha20_xform_crypt_last, }; diff --git a/sys/crypto/via/padlock.h b/sys/crypto/via/padlock.h index a22e88ba6ee9..da85771f3663 100644 --- a/sys/crypto/via/padlock.h +++ b/sys/crypto/via/padlock.h @@ -1,87 +1,87 @@ /*- * Copyright (c) 2005-2006 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _PADLOCK_H_ #define _PADLOCK_H_ #include #include #include union padlock_cw { uint64_t raw; struct { u_int round_count : 4; u_int algorithm_type : 3; u_int key_generation : 1; u_int intermediate : 1; u_int direction : 1; u_int key_size : 2; u_int filler0 : 20; u_int filler1 : 32; u_int filler2 : 32; u_int filler3 : 32; } __field; }; #define cw_round_count __field.round_count #define cw_algorithm_type __field.algorithm_type #define cw_key_generation __field.key_generation #define cw_intermediate __field.intermediate #define cw_direction __field.direction #define cw_key_size __field.key_size #define cw_filler0 __field.filler0 #define cw_filler1 __field.filler1 #define cw_filler2 __field.filler2 #define cw_filler3 __field.filler3 struct padlock_session { union padlock_cw ses_cw __aligned(16); uint32_t ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ uint32_t ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ - struct auth_hash *ses_axf; + const struct auth_hash *ses_axf; uint8_t *ses_ictx; uint8_t *ses_octx; int ses_mlen; struct fpu_kern_ctx *ses_fpu_ctx; }; #define PADLOCK_ALIGN(p) (void *)(roundup2((uintptr_t)(p), 16)) int padlock_cipher_setup(struct padlock_session *ses, const struct crypto_session_params *csp); int padlock_cipher_process(struct padlock_session *ses, struct cryptop *crp, const struct crypto_session_params *csp); bool padlock_hash_check(const struct crypto_session_params *csp); int padlock_hash_setup(struct padlock_session *ses, const struct crypto_session_params *csp); int padlock_hash_process(struct padlock_session *ses, struct cryptop *crp, const struct crypto_session_params *csp); void padlock_hash_free(struct padlock_session *ses); #endif /* !_PADLOCK_H_ */ diff --git a/sys/crypto/via/padlock_hash.c b/sys/crypto/via/padlock_hash.c index 1640bad4ea74..f09024af4ed5 100644 --- a/sys/crypto/via/padlock_hash.c +++ b/sys/crypto/via/padlock_hash.c @@ -1,425 +1,425 @@ /*- * Copyright (c) 2006 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #if defined(__amd64__) || defined(__i386__) #include #include #include #include #endif #include #include #include #include /* * Implementation notes. * * Some VIA CPUs provides SHA1 and SHA256 acceleration. * We implement all HMAC algorithms provided by crypto(9) framework, but we do * the crypto work in software unless this is HMAC/SHA1 or HMAC/SHA256 and * our CPU can accelerate it. * * Additional CPU instructions, which preform SHA1 and SHA256 are one-shot * functions - we have only one chance to give the data, CPU itself will add * the padding and calculate hash automatically. * This means, it is not possible to implement common init(), update(), final() * methods. * The way I've choosen is to keep adding data to the buffer on update() * (reallocating the buffer if necessary) and call XSHA{1,256} instruction on * final(). */ struct padlock_sha_ctx { uint8_t *psc_buf; int psc_offset; int psc_size; }; CTASSERT(sizeof(struct padlock_sha_ctx) <= sizeof(union authctx)); static void padlock_sha_init(void *vctx); static int padlock_sha_update(void *vctx, const void *buf, u_int bufsize); static void padlock_sha1_final(uint8_t *hash, void *vctx); static void padlock_sha256_final(uint8_t *hash, void *vctx); -static struct auth_hash padlock_hmac_sha1 = { +static const struct auth_hash padlock_hmac_sha1 = { .type = CRYPTO_SHA1_HMAC, .name = "HMAC-SHA1", .keysize = SHA1_BLOCK_LEN, .hashsize = SHA1_HASH_LEN, .ctxsize = sizeof(struct padlock_sha_ctx), .blocksize = SHA1_BLOCK_LEN, .Init = padlock_sha_init, .Update = padlock_sha_update, .Final = padlock_sha1_final, }; -static struct auth_hash padlock_hmac_sha256 = { +static const struct auth_hash padlock_hmac_sha256 = { .type = CRYPTO_SHA2_256_HMAC, .name = "HMAC-SHA2-256", .keysize = SHA2_256_BLOCK_LEN, .hashsize = SHA2_256_HASH_LEN, .ctxsize = sizeof(struct padlock_sha_ctx), .blocksize = SHA2_256_BLOCK_LEN, .Init = padlock_sha_init, .Update = padlock_sha_update, .Final = padlock_sha256_final, }; MALLOC_DECLARE(M_PADLOCK); static __inline void padlock_output_block(uint32_t *src, uint32_t *dst, size_t count) { while (count-- > 0) *dst++ = bswap32(*src++); } static void padlock_do_sha1(const u_char *in, u_char *out, int count) { u_char buf[128+16]; /* PadLock needs at least 128 bytes buffer. */ u_char *result = PADLOCK_ALIGN(buf); ((uint32_t *)result)[0] = 0x67452301; ((uint32_t *)result)[1] = 0xEFCDAB89; ((uint32_t *)result)[2] = 0x98BADCFE; ((uint32_t *)result)[3] = 0x10325476; ((uint32_t *)result)[4] = 0xC3D2E1F0; #ifdef __GNUCLIKE_ASM __asm __volatile( ".byte 0xf3, 0x0f, 0xa6, 0xc8" /* rep xsha1 */ : "+S"(in), "+D"(result) : "c"(count), "a"(0) ); #endif padlock_output_block((uint32_t *)result, (uint32_t *)out, SHA1_HASH_LEN / sizeof(uint32_t)); } static void padlock_do_sha256(const char *in, char *out, int count) { char buf[128+16]; /* PadLock needs at least 128 bytes buffer. */ char *result = PADLOCK_ALIGN(buf); ((uint32_t *)result)[0] = 0x6A09E667; ((uint32_t *)result)[1] = 0xBB67AE85; ((uint32_t *)result)[2] = 0x3C6EF372; ((uint32_t *)result)[3] = 0xA54FF53A; ((uint32_t *)result)[4] = 0x510E527F; ((uint32_t *)result)[5] = 0x9B05688C; ((uint32_t *)result)[6] = 0x1F83D9AB; ((uint32_t *)result)[7] = 0x5BE0CD19; #ifdef __GNUCLIKE_ASM __asm __volatile( ".byte 0xf3, 0x0f, 0xa6, 0xd0" /* rep xsha256 */ : "+S"(in), "+D"(result) : "c"(count), "a"(0) ); #endif padlock_output_block((uint32_t *)result, (uint32_t *)out, SHA2_256_HASH_LEN / sizeof(uint32_t)); } static void padlock_sha_init(void *vctx) { struct padlock_sha_ctx *ctx; ctx = vctx; ctx->psc_buf = NULL; ctx->psc_offset = 0; ctx->psc_size = 0; } static int padlock_sha_update(void *vctx, const void *buf, u_int bufsize) { struct padlock_sha_ctx *ctx; ctx = vctx; if (ctx->psc_size - ctx->psc_offset < bufsize) { ctx->psc_size = MAX(ctx->psc_size * 2, ctx->psc_size + bufsize); ctx->psc_buf = realloc(ctx->psc_buf, ctx->psc_size, M_PADLOCK, M_NOWAIT); if(ctx->psc_buf == NULL) return (ENOMEM); } bcopy(buf, ctx->psc_buf + ctx->psc_offset, bufsize); ctx->psc_offset += bufsize; return (0); } static void padlock_sha_free(void *vctx) { struct padlock_sha_ctx *ctx; ctx = vctx; if (ctx->psc_buf != NULL) { zfree(ctx->psc_buf, M_PADLOCK); ctx->psc_buf = NULL; ctx->psc_offset = 0; ctx->psc_size = 0; } } static void padlock_sha1_final(uint8_t *hash, void *vctx) { struct padlock_sha_ctx *ctx; ctx = vctx; padlock_do_sha1(ctx->psc_buf, hash, ctx->psc_offset); padlock_sha_free(ctx); } static void padlock_sha256_final(uint8_t *hash, void *vctx) { struct padlock_sha_ctx *ctx; ctx = vctx; padlock_do_sha256(ctx->psc_buf, hash, ctx->psc_offset); padlock_sha_free(ctx); } static void -padlock_copy_ctx(struct auth_hash *axf, void *sctx, void *dctx) +padlock_copy_ctx(const struct auth_hash *axf, void *sctx, void *dctx) { if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 && (axf->type == CRYPTO_SHA1_HMAC || axf->type == CRYPTO_SHA2_256_HMAC)) { struct padlock_sha_ctx *spctx = sctx, *dpctx = dctx; dpctx->psc_offset = spctx->psc_offset; dpctx->psc_size = spctx->psc_size; dpctx->psc_buf = malloc(dpctx->psc_size, M_PADLOCK, M_WAITOK); bcopy(spctx->psc_buf, dpctx->psc_buf, dpctx->psc_size); } else { bcopy(sctx, dctx, axf->ctxsize); } } static void -padlock_free_ctx(struct auth_hash *axf, void *ctx) +padlock_free_ctx(const struct auth_hash *axf, void *ctx) { if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 && (axf->type == CRYPTO_SHA1_HMAC || axf->type == CRYPTO_SHA2_256_HMAC)) { padlock_sha_free(ctx); } } static void padlock_hash_key_setup(struct padlock_session *ses, const uint8_t *key, int klen) { - struct auth_hash *axf; + const struct auth_hash *axf; axf = ses->ses_axf; /* * Try to free contexts before using them, because * padlock_hash_key_setup() can be called twice - once from * padlock_newsession() and again from padlock_process(). */ padlock_free_ctx(axf, ses->ses_ictx); padlock_free_ctx(axf, ses->ses_octx); hmac_init_ipad(axf, key, klen, ses->ses_ictx); hmac_init_opad(axf, key, klen, ses->ses_octx); } /* * Compute keyed-hash authenticator. */ static int padlock_authcompute(struct padlock_session *ses, struct cryptop *crp) { u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN]; - struct auth_hash *axf; + const struct auth_hash *axf; union authctx ctx; int error; axf = ses->ses_axf; padlock_copy_ctx(axf, ses->ses_ictx, &ctx); error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (error != 0) { padlock_free_ctx(axf, &ctx); return (error); } error = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); if (error != 0) { padlock_free_ctx(axf, &ctx); return (error); } axf->Final(hash, &ctx); padlock_copy_ctx(axf, ses->ses_octx, &ctx); axf->Update(&ctx, hash, axf->hashsize); axf->Final(hash, &ctx); if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen, hash2); if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0) return (EBADMSG); } else crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen, hash); return (0); } /* Find software structure which describes HMAC algorithm. */ -static struct auth_hash * +static const struct auth_hash * padlock_hash_lookup(int alg) { - struct auth_hash *axf; + const struct auth_hash *axf; switch (alg) { case CRYPTO_NULL_HMAC: axf = &auth_hash_null; break; case CRYPTO_SHA1_HMAC: if ((via_feature_xcrypt & VIA_HAS_SHA) != 0) axf = &padlock_hmac_sha1; else axf = &auth_hash_hmac_sha1; break; case CRYPTO_RIPEMD160_HMAC: axf = &auth_hash_hmac_ripemd_160; break; case CRYPTO_SHA2_256_HMAC: if ((via_feature_xcrypt & VIA_HAS_SHA) != 0) axf = &padlock_hmac_sha256; else axf = &auth_hash_hmac_sha2_256; break; case CRYPTO_SHA2_384_HMAC: axf = &auth_hash_hmac_sha2_384; break; case CRYPTO_SHA2_512_HMAC: axf = &auth_hash_hmac_sha2_512; break; default: axf = NULL; break; } return (axf); } bool padlock_hash_check(const struct crypto_session_params *csp) { return (padlock_hash_lookup(csp->csp_auth_alg) != NULL); } int padlock_hash_setup(struct padlock_session *ses, const struct crypto_session_params *csp) { ses->ses_axf = padlock_hash_lookup(csp->csp_auth_alg); if (csp->csp_auth_mlen == 0) ses->ses_mlen = ses->ses_axf->hashsize; else ses->ses_mlen = csp->csp_auth_mlen; /* Allocate memory for HMAC inner and outer contexts. */ ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_PADLOCK, M_ZERO | M_NOWAIT); ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_PADLOCK, M_ZERO | M_NOWAIT); if (ses->ses_ictx == NULL || ses->ses_octx == NULL) return (ENOMEM); /* Setup key if given. */ if (csp->csp_auth_key != NULL) { padlock_hash_key_setup(ses, csp->csp_auth_key, csp->csp_auth_klen); } return (0); } int padlock_hash_process(struct padlock_session *ses, struct cryptop *crp, const struct crypto_session_params *csp) { struct thread *td; int error; td = curthread; fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); if (crp->crp_auth_key != NULL) padlock_hash_key_setup(ses, crp->crp_auth_key, csp->csp_auth_klen); error = padlock_authcompute(ses, crp); fpu_kern_leave(td, ses->ses_fpu_ctx); return (error); } void padlock_hash_free(struct padlock_session *ses) { if (ses->ses_ictx != NULL) { padlock_free_ctx(ses->ses_axf, ses->ses_ictx); zfree(ses->ses_ictx, M_PADLOCK); ses->ses_ictx = NULL; } if (ses->ses_octx != NULL) { padlock_free_ctx(ses->ses_axf, ses->ses_octx); zfree(ses->ses_octx, M_PADLOCK); ses->ses_octx = NULL; } } diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 2adc902cc9ac..3eb39ef5e987 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -1,1486 +1,1487 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __T4_ADAPTER_H__ #define __T4_ADAPTER_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "offload.h" #include "t4_ioctl.h" #include "common/t4_msg.h" #include "firmware/t4fw_interface.h" #define KTR_CXGBE KTR_SPARE3 MALLOC_DECLARE(M_CXGBE); #define CXGBE_UNIMPLEMENTED(s) \ panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) /* * Same as LIST_HEAD from queue.h. This is to avoid conflict with LinuxKPI's * LIST_HEAD when building iw_cxgbe. */ #define CXGBE_LIST_HEAD(name, type) \ struct name { \ struct type *lh_first; /* first element */ \ } #ifndef SYSCTL_ADD_UQUAD #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD #define sysctl_handle_64 sysctl_handle_quad #define CTLTYPE_U64 CTLTYPE_QUAD #endif SYSCTL_DECL(_hw_cxgbe); struct adapter; typedef struct adapter adapter_t; enum { /* * All ingress queues use this entry size. Note that the firmware event * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to * be at least 64. */ IQ_ESIZE = 64, /* Default queue sizes for all kinds of ingress queues */ FW_IQ_QSIZE = 256, RX_IQ_QSIZE = 1024, /* All egress queues use this entry size */ EQ_ESIZE = 64, /* Default queue sizes for all kinds of egress queues */ CTRL_EQ_QSIZE = 1024, TX_EQ_QSIZE = 1024, #if MJUMPAGESIZE != MCLBYTES SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ #else SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ #endif CL_METADATA_SIZE = CACHE_LINE_SIZE, SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */ TX_SGL_SEGS = 39, TX_SGL_SEGS_TSO = 38, TX_SGL_SEGS_VM = 38, TX_SGL_SEGS_VM_TSO = 37, TX_SGL_SEGS_EO_TSO = 30, /* XXX: lower for IPv6. */ TX_SGL_SEGS_VXLAN_TSO = 37, TX_WR_FLITS = SGE_MAX_WR_LEN / 8 }; enum { /* adapter intr_type */ INTR_INTX = (1 << 0), INTR_MSI = (1 << 1), INTR_MSIX = (1 << 2) }; enum { XGMAC_MTU = (1 << 0), XGMAC_PROMISC = (1 << 1), XGMAC_ALLMULTI = (1 << 2), XGMAC_VLANEX = (1 << 3), XGMAC_UCADDR = (1 << 4), XGMAC_MCADDRS = (1 << 5), XGMAC_ALL = 0xffff }; enum { /* flags understood by begin_synchronized_op */ HOLD_LOCK = (1 << 0), SLEEP_OK = (1 << 1), INTR_OK = (1 << 2), /* flags understood by end_synchronized_op */ LOCK_HELD = HOLD_LOCK, }; enum { /* adapter flags */ FULL_INIT_DONE = (1 << 0), FW_OK = (1 << 1), CHK_MBOX_ACCESS = (1 << 2), MASTER_PF = (1 << 3), ADAP_SYSCTL_CTX = (1 << 4), ADAP_ERR = (1 << 5), BUF_PACKING_OK = (1 << 6), IS_VF = (1 << 7), KERN_TLS_ON = (1 << 8), /* HW is configured for KERN_TLS */ CXGBE_BUSY = (1 << 9), HW_OFF_LIMITS = (1 << 10), /* off limits to all except reset_thread */ /* port flags */ HAS_TRACEQ = (1 << 3), FIXED_IFMEDIA = (1 << 4), /* ifmedia list doesn't change. */ /* VI flags */ DOOMED = (1 << 0), VI_INIT_DONE = (1 << 1), VI_SYSCTL_CTX = (1 << 2), TX_USES_VM_WR = (1 << 3), VI_SKIP_STATS = (1 << 4), /* adapter debug_flags */ DF_DUMP_MBOX = (1 << 0), /* Log all mbox cmd/rpl. */ DF_LOAD_FW_ANYTIME = (1 << 1), /* Allow LOAD_FW after init */ DF_DISABLE_TCB_CACHE = (1 << 2), /* Disable TCB cache (T6+) */ DF_DISABLE_CFG_RETRY = (1 << 3), /* Disable fallback config */ DF_VERBOSE_SLOWINTR = (1 << 4), /* Chatty slow intr handler */ }; #define IS_DOOMED(vi) ((vi)->flags & DOOMED) #define SET_DOOMED(vi) do {(vi)->flags |= DOOMED;} while (0) #define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) #define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) #define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) struct vi_info { device_t dev; struct port_info *pi; struct adapter *adapter; struct ifnet *ifp; struct pfil_head *pfil; unsigned long flags; int if_flags; uint16_t *rss, *nm_rss; uint16_t viid; /* opaque VI identifier */ uint16_t smt_idx; uint16_t vin; uint8_t vfvld; int16_t xact_addr_filt;/* index of exact MAC address filter */ uint16_t rss_size; /* size of VI's RSS table slice */ uint16_t rss_base; /* start of VI's RSS table slice */ int hashen; int nintr; int first_intr; /* These need to be int as they are used in sysctl */ int ntxq; /* # of tx queues */ int first_txq; /* index of first tx queue */ int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ int nrxq; /* # of rx queues */ int first_rxq; /* index of first rx queue */ int nofldtxq; /* # of offload tx queues */ int first_ofld_txq; /* index of first offload tx queue */ int nofldrxq; /* # of offload rx queues */ int first_ofld_rxq; /* index of first offload rx queue */ int nnmtxq; int first_nm_txq; int nnmrxq; int first_nm_rxq; int tmr_idx; int ofld_tmr_idx; int pktc_idx; int ofld_pktc_idx; int qsize_rxq; int qsize_txq; struct timeval last_refreshed; struct fw_vi_stats_vf stats; struct mtx tick_mtx; struct callout tick; struct sysctl_ctx_list ctx; struct sysctl_oid *rxq_oid; struct sysctl_oid *txq_oid; struct sysctl_oid *nm_rxq_oid; struct sysctl_oid *nm_txq_oid; struct sysctl_oid *ofld_rxq_oid; struct sysctl_oid *ofld_txq_oid; uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ }; struct tx_ch_rl_params { enum fw_sched_params_rate ratemode; /* %port (REL) or kbps (ABS) */ uint32_t maxrate; }; /* CLRL state */ enum clrl_state { CS_UNINITIALIZED = 0, CS_PARAMS_SET, /* sw parameters have been set. */ CS_HW_UPDATE_REQUESTED, /* async HW update requested. */ CS_HW_UPDATE_IN_PROGRESS, /* sync hw update in progress. */ CS_HW_CONFIGURED /* configured in the hardware. */ }; /* CLRL flags */ enum { CF_USER = (1 << 0), /* was configured by driver ioctl. */ }; struct tx_cl_rl_params { enum clrl_state state; int refcount; uint8_t flags; enum fw_sched_params_rate ratemode; /* %port REL or ABS value */ enum fw_sched_params_unit rateunit; /* kbps or pps (when ABS) */ enum fw_sched_params_mode mode; /* aggr or per-flow */ uint32_t maxrate; uint16_t pktsize; uint16_t burstsize; }; /* Tx scheduler parameters for a channel/port */ struct tx_sched_params { /* Channel Rate Limiter */ struct tx_ch_rl_params ch_rl; /* Class WRR */ /* XXX */ /* Class Rate Limiter (including the default pktsize and burstsize). */ int pktsize; int burstsize; struct tx_cl_rl_params cl_rl[]; }; struct port_info { device_t dev; struct adapter *adapter; struct vi_info *vi; int nvi; int up_vis; int uld_vis; bool vxlan_tcam_entry; struct tx_sched_params *sched_params; struct mtx pi_lock; char lockname[16]; unsigned long flags; uint8_t lport; /* associated offload logical port */ int8_t mdio_addr; uint8_t port_type; uint8_t mod_type; uint8_t port_id; uint8_t tx_chan; uint8_t mps_bg_map; /* rx MPS buffer group bitmap */ uint8_t rx_e_chan_map; /* rx TP e-channel bitmap */ uint8_t rx_c_chan; /* rx TP c-channel */ struct link_config link_cfg; struct ifmedia media; struct port_stats stats; u_int tnl_cong_drops; u_int tx_parse_error; int fcs_reg; uint64_t fcs_base; }; #define IS_MAIN_VI(vi) ((vi) == &((vi)->pi->vi[0])) struct cluster_metadata { uma_zone_t zone; caddr_t cl; u_int refcount; }; struct fl_sdesc { caddr_t cl; uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ int16_t moff; /* offset of metadata from cl */ uint8_t zidx; }; struct tx_desc { __be64 flit[8]; }; struct tx_sdesc { struct mbuf *m; /* m_nextpkt linked chain of frames */ uint8_t desc_used; /* # of hardware descriptors used by the WR */ }; #define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header)) struct iq_desc { struct rss_header rss; uint8_t cpl[IQ_PAD]; struct rsp_ctrl rsp; }; #undef IQ_PAD CTASSERT(sizeof(struct iq_desc) == IQ_ESIZE); enum { /* iq flags */ IQ_SW_ALLOCATED = (1 << 0), /* sw resources allocated */ IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ IQ_RX_TIMESTAMP = (1 << 2), /* provide the SGE rx timestamp */ IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ IQ_ADJ_CREDIT = (1 << 4), /* hw is off by 1 credit for this iq */ IQ_HW_ALLOCATED = (1 << 5), /* fw/hw resources allocated */ /* iq state */ IQS_DISABLED = 0, IQS_BUSY = 1, IQS_IDLE = 2, /* netmap related flags */ NM_OFF = 0, NM_ON = 1, NM_BUSY = 2, }; enum { CPL_COOKIE_RESERVED = 0, CPL_COOKIE_FILTER, CPL_COOKIE_DDP0, CPL_COOKIE_DDP1, CPL_COOKIE_TOM, CPL_COOKIE_HASHFILTER, CPL_COOKIE_ETHOFLD, CPL_COOKIE_KERN_TLS, NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */ }; struct sge_iq; struct rss_header; typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, struct mbuf *); typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); /* * Ingress Queue: T4 is producer, driver is consumer. */ struct sge_iq { uint32_t flags; volatile int state; struct adapter *adapter; struct iq_desc *desc; /* KVA of descriptor ring */ int8_t intr_pktc_idx; /* packet count threshold index */ uint8_t gen; /* generation bit */ uint8_t intr_params; /* interrupt holdoff parameters */ int8_t cong; /* congestion settings */ uint16_t qsize; /* size (# of entries) of the queue */ uint16_t sidx; /* index of the entry with the status page */ uint16_t cidx; /* consumer index */ uint16_t cntxt_id; /* SGE context id for the iq */ uint16_t abs_id; /* absolute SGE id for the iq */ int16_t intr_idx; /* interrupt used by the queue */ STAILQ_ENTRY(sge_iq) link; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; /* bus address of descriptor ring */ }; enum { /* eq type */ EQ_CTRL = 1, EQ_ETH = 2, EQ_OFLD = 3, /* eq flags */ EQ_SW_ALLOCATED = (1 << 0), /* sw resources allocated */ EQ_HW_ALLOCATED = (1 << 1), /* hw/fw resources allocated */ EQ_ENABLED = (1 << 3), /* open for business */ EQ_QFLUSH = (1 << 4), /* if_qflush in progress */ }; /* Listed in order of preference. Update t4_sysctls too if you change these */ enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; /* * Egress Queue: driver is producer, T4 is consumer. * * Note: A free list is an egress queue (driver produces the buffers and T4 * consumes them) but it's special enough to have its own struct (see sge_fl). */ struct sge_eq { unsigned int flags; /* MUST be first */ unsigned int cntxt_id; /* SGE context id for the eq */ unsigned int abs_id; /* absolute SGE id for the eq */ uint8_t type; /* EQ_CTRL/EQ_ETH/EQ_OFLD */ uint8_t doorbells; uint8_t tx_chan; /* tx channel used by the eq */ struct mtx eq_lock; struct tx_desc *desc; /* KVA of descriptor ring */ volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ u_int udb_qid; /* relative qid within the doorbell page */ uint16_t sidx; /* index of the entry with the status page */ uint16_t cidx; /* consumer idx (desc idx) */ uint16_t pidx; /* producer idx (desc idx) */ uint16_t equeqidx; /* EQUEQ last requested at this pidx */ uint16_t dbidx; /* pidx of the most recent doorbell */ uint16_t iqid; /* cached iq->cntxt_id (see iq below) */ volatile u_int equiq; /* EQUIQ outstanding */ struct sge_iq *iq; /* iq that receives egr_update for the eq */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; /* bus address of descriptor ring */ char lockname[16]; }; struct rx_buf_info { uma_zone_t zone; /* zone that this cluster comes from */ uint16_t size1; /* same as size of cluster: 2K/4K/9K/16K. * hwsize[hwidx1] = size1. No spare. */ uint16_t size2; /* hwsize[hwidx2] = size2. * spare in cluster = size1 - size2. */ int8_t hwidx1; /* SGE bufsize idx for size1 */ int8_t hwidx2; /* SGE bufsize idx for size2 */ uint8_t type; /* EXT_xxx type of the cluster */ }; enum { NUM_MEMWIN = 3, MEMWIN0_APERTURE = 2048, MEMWIN0_BASE = 0x1b800, MEMWIN1_APERTURE = 32768, MEMWIN1_BASE = 0x28000, MEMWIN2_APERTURE_T4 = 65536, MEMWIN2_BASE_T4 = 0x30000, MEMWIN2_APERTURE_T5 = 128 * 1024, MEMWIN2_BASE_T5 = 0x60000, }; struct memwin { struct rwlock mw_lock __aligned(CACHE_LINE_SIZE); uint32_t mw_base; /* constant after setup_memwin */ uint32_t mw_aperture; /* ditto */ uint32_t mw_curpos; /* protected by mw_lock */ }; enum { FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ FL_DOOMED = (1 << 1), /* about to be destroyed */ FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ FL_BUF_RESUME = (1 << 3), /* resume from the middle of the frame */ }; #define FL_RUNNING_LOW(fl) \ (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat) #define FL_NOT_RUNNING_LOW(fl) \ (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat) struct sge_fl { struct mtx fl_lock; __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ uint16_t zidx; /* refill zone idx */ uint16_t safe_zidx; uint16_t lowat; /* # of buffers <= this means fl needs help */ int flags; uint16_t buf_boundary; /* The 16b idx all deal with hw descriptors */ uint16_t dbidx; /* hw pidx after last doorbell */ uint16_t sidx; /* index of status page */ volatile uint16_t hw_cidx; /* The 32b idx are all buffer idx, not hardware descriptor idx */ uint32_t cidx; /* consumer index */ uint32_t pidx; /* producer index */ uint32_t dbval; u_int rx_offset; /* offset in fl buf (when buffer packing) */ volatile uint32_t *udb; uint64_t cl_allocated; /* # of clusters allocated */ uint64_t cl_recycled; /* # of clusters recycled */ uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ /* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */ struct mbuf *m0; struct mbuf **pnext; u_int remaining; uint16_t qsize; /* # of hw descriptors (status page included) */ uint16_t cntxt_id; /* SGE context id for the freelist */ TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; char lockname[16]; bus_addr_t ba; /* bus address of descriptor ring */ }; struct mp_ring; struct txpkts { uint8_t wr_type; /* type 0 or type 1 */ uint8_t npkt; /* # of packets in this work request */ uint8_t len16; /* # of 16B pieces used by this work request */ uint8_t score; uint8_t max_npkt; /* maximum number of packets allowed */ uint16_t plen; /* total payload (sum of all packets) */ /* straight from fw_eth_tx_pkts_vm_wr. */ __u8 ethmacdst[6]; __u8 ethmacsrc[6]; __be16 ethtype; __be16 vlantci; struct mbuf *mb[15]; }; /* txq: SGE egress queue + what's needed for Ethernet NIC */ struct sge_txq { struct sge_eq eq; /* MUST be first */ struct ifnet *ifp; /* the interface this txq belongs to */ struct mp_ring *r; /* tx software ring */ struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ struct sglist *gl; __be32 cpl_ctrl0; /* for convenience */ int tc_idx; /* traffic class */ uint64_t last_tx; /* cycle count when eth_tx was last called */ struct txpkts txp; struct task tx_reclaim_task; /* stats for common events first */ uint64_t txcsum; /* # of times hardware assisted with checksum */ uint64_t tso_wrs; /* # of TSO work requests */ uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ uint64_t imm_wrs; /* # of work requests with immediate data */ uint64_t sgl_wrs; /* # of work requests with direct SGL */ uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ uint64_t txpkts0_wrs; /* # of type0 coalesced tx work requests */ uint64_t txpkts1_wrs; /* # of type1 coalesced tx work requests */ uint64_t txpkts0_pkts; /* # of frames in type0 coalesced tx WRs */ uint64_t txpkts1_pkts; /* # of frames in type1 coalesced tx WRs */ uint64_t txpkts_flush; /* # of times txp had to be sent by tx_update */ uint64_t raw_wrs; /* # of raw work requests (alloc_wr_mbuf) */ uint64_t vxlan_tso_wrs; /* # of VXLAN TSO work requests */ uint64_t vxlan_txcsum; uint64_t kern_tls_records; uint64_t kern_tls_short; uint64_t kern_tls_partial; uint64_t kern_tls_full; uint64_t kern_tls_octets; uint64_t kern_tls_waste; uint64_t kern_tls_options; uint64_t kern_tls_header; uint64_t kern_tls_fin; uint64_t kern_tls_fin_short; uint64_t kern_tls_cbc; uint64_t kern_tls_gcm; /* stats for not-that-common events */ /* Optional scratch space for constructing work requests. */ uint8_t ss[SGE_MAX_WR_LEN] __aligned(16); } __aligned(CACHE_LINE_SIZE); /* rxq: SGE ingress queue + SGE free list + miscellaneous items */ struct sge_rxq { struct sge_iq iq; /* MUST be first */ struct sge_fl fl; /* MUST follow iq */ struct ifnet *ifp; /* the interface this rxq belongs to */ struct lro_ctrl lro; /* LRO state */ /* stats for common events first */ uint64_t rxcsum; /* # of times hardware assisted with checksum */ uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ uint64_t vxlan_rxcsum; /* stats for not-that-common events */ } __aligned(CACHE_LINE_SIZE); static inline struct sge_rxq * iq_to_rxq(struct sge_iq *iq) { return (__containerof(iq, struct sge_rxq, iq)); } /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ struct sge_ofld_rxq { struct sge_iq iq; /* MUST be first */ struct sge_fl fl; /* MUST follow iq */ counter_u64_t rx_iscsi_ddp_setup_ok; counter_u64_t rx_iscsi_ddp_setup_error; uint64_t rx_iscsi_ddp_pdus; uint64_t rx_iscsi_ddp_octets; uint64_t rx_iscsi_fl_pdus; uint64_t rx_iscsi_fl_octets; u_long rx_toe_tls_records; u_long rx_toe_tls_octets; } __aligned(CACHE_LINE_SIZE); static inline struct sge_ofld_rxq * iq_to_ofld_rxq(struct sge_iq *iq) { return (__containerof(iq, struct sge_ofld_rxq, iq)); } struct wrqe { STAILQ_ENTRY(wrqe) link; struct sge_wrq *wrq; int wr_len; char wr[] __aligned(16); }; struct wrq_cookie { TAILQ_ENTRY(wrq_cookie) link; int ndesc; int pidx; }; /* * wrq: SGE egress queue that is given prebuilt work requests. Control queues * are of this type. */ struct sge_wrq { struct sge_eq eq; /* MUST be first */ struct adapter *adapter; struct task wrq_tx_task; /* Tx desc reserved but WR not "committed" yet. */ TAILQ_HEAD(wrq_incomplete_wrs , wrq_cookie) incomplete_wrs; /* List of WRs ready to go out as soon as descriptors are available. */ STAILQ_HEAD(, wrqe) wr_list; u_int nwr_pending; u_int ndesc_needed; /* stats for common events first */ uint64_t tx_wrs_direct; /* # of WRs written directly to desc ring. */ uint64_t tx_wrs_ss; /* # of WRs copied from scratch space. */ uint64_t tx_wrs_copied; /* # of WRs queued and copied to desc ring. */ /* stats for not-that-common events */ /* * Scratch space for work requests that wrap around after reaching the * status page, and some information about the last WR that used it. */ uint16_t ss_pidx; uint16_t ss_len; uint8_t ss[SGE_MAX_WR_LEN]; } __aligned(CACHE_LINE_SIZE); /* ofld_txq: SGE egress queue + miscellaneous items */ struct sge_ofld_txq { struct sge_wrq wrq; counter_u64_t tx_iscsi_pdus; counter_u64_t tx_iscsi_octets; counter_u64_t tx_toe_tls_records; counter_u64_t tx_toe_tls_octets; } __aligned(CACHE_LINE_SIZE); #define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1)) struct sge_nm_rxq { /* Items used by the driver rx ithread are in this cacheline. */ volatile int nm_state __aligned(CACHE_LINE_SIZE); /* NM_OFF, NM_ON, or NM_BUSY */ u_int nid; /* netmap ring # for this queue */ struct vi_info *vi; struct iq_desc *iq_desc; uint16_t iq_abs_id; uint16_t iq_cntxt_id; uint16_t iq_cidx; uint16_t iq_sidx; uint8_t iq_gen; uint32_t fl_sidx; /* Items used by netmap rxsync are in this cacheline. */ __be64 *fl_desc __aligned(CACHE_LINE_SIZE); uint16_t fl_cntxt_id; uint32_t fl_pidx; uint32_t fl_sidx2; /* copy of fl_sidx */ uint32_t fl_db_val; u_int fl_db_saved; u_int fl_db_threshold; /* in descriptors */ u_int fl_hwidx:4; /* * fl_cidx is used by both the ithread and rxsync, the rest are not used * in the rx fast path. */ uint32_t fl_cidx __aligned(CACHE_LINE_SIZE); bus_dma_tag_t iq_desc_tag; bus_dmamap_t iq_desc_map; bus_addr_t iq_ba; int intr_idx; bus_dma_tag_t fl_desc_tag; bus_dmamap_t fl_desc_map; bus_addr_t fl_ba; }; #define INVALID_NM_TXQ_CNTXT_ID ((u_int)(-1)) struct sge_nm_txq { struct tx_desc *desc; uint16_t cidx; uint16_t pidx; uint16_t sidx; uint16_t equiqidx; /* EQUIQ last requested at this pidx */ uint16_t equeqidx; /* EQUEQ last requested at this pidx */ uint16_t dbidx; /* pidx of the most recent doorbell */ uint8_t doorbells; volatile uint32_t *udb; u_int udb_qid; u_int cntxt_id; __be32 cpl_ctrl0; /* for convenience */ __be32 op_pkd; /* ditto */ u_int nid; /* netmap ring # for this queue */ /* infrequently used items after this */ bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; bus_addr_t ba; int iqidx; } __aligned(CACHE_LINE_SIZE); struct sge { int nrxq; /* total # of Ethernet rx queues */ int ntxq; /* total # of Ethernet tx queues */ int nofldrxq; /* total # of TOE rx queues */ int nofldtxq; /* total # of TOE tx queues */ int nnmrxq; /* total # of netmap rx queues */ int nnmtxq; /* total # of netmap tx queues */ int niq; /* total # of ingress queues */ int neq; /* total # of egress queues */ struct sge_iq fwq; /* Firmware event queue */ struct sge_wrq *ctrlq; /* Control queues */ struct sge_txq *txq; /* NIC tx queues */ struct sge_rxq *rxq; /* NIC rx queues */ struct sge_ofld_txq *ofld_txq; /* TOE tx queues */ struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ struct sge_nm_txq *nm_txq; /* netmap tx queues */ struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ uint16_t iq_start; /* first cntxt_id */ uint16_t iq_base; /* first abs_id */ int eq_start; /* first cntxt_id */ int eq_base; /* first abs_id */ int iqmap_sz; int eqmap_sz; struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ int8_t safe_zidx; struct rx_buf_info rx_buf_info[SW_ZONE_SIZES]; }; struct devnames { const char *nexus_name; const char *ifnet_name; const char *vi_ifnet_name; const char *pf03_drv_name; const char *vf_nexus_name; const char *vf_ifnet_name; }; struct clip_entry; struct adapter { SLIST_ENTRY(adapter) link; device_t dev; struct cdev *cdev; const struct devnames *names; /* PCIe register resources */ int regs_rid; struct resource *regs_res; int msix_rid; struct resource *msix_res; bus_space_handle_t bh; bus_space_tag_t bt; bus_size_t mmio_len; int udbs_rid; struct resource *udbs_res; volatile uint8_t *udbs_base; unsigned int pf; unsigned int mbox; unsigned int vpd_busy; unsigned int vpd_flag; /* Interrupt information */ int intr_type; int intr_count; struct irq { struct resource *res; int rid; void *tag; struct sge_rxq *rxq; struct sge_nm_rxq *nm_rxq; } __aligned(CACHE_LINE_SIZE) *irq; int sge_gts_reg; int sge_kdoorbell_reg; bus_dma_tag_t dmat; /* Parent DMA tag */ struct sge sge; int lro_timeout; int sc_do_rxcopy; int vxlan_port; u_int vxlan_refcount; int rawf_base; int nrawf; struct taskqueue *tq[MAX_NCHAN]; /* General purpose taskqueues */ struct task async_event_task; struct port_info *port[MAX_NPORTS]; uint8_t chan_map[MAX_NCHAN]; /* channel -> port */ CXGBE_LIST_HEAD(, clip_entry) *clip_table; TAILQ_HEAD(, clip_entry) clip_pending; /* these need hw update. */ u_long clip_mask; int clip_gen; struct timeout_task clip_task; void *tom_softc; /* (struct tom_data *) */ struct tom_tunables tt; struct t4_offload_policy *policy; struct rwlock policy_lock; void *iwarp_softc; /* (struct c4iw_dev *) */ struct iw_tunables iwt; void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ void *ccr_softc; /* (struct ccr_softc *) */ struct l2t_data *l2t; /* L2 table */ struct smt_data *smt; /* Source MAC Table */ struct tid_info tids; vmem_t *key_map; struct tls_tunables tlst; uint8_t doorbells; int offload_map; /* ports with IFCAP_TOE enabled */ int active_ulds; /* ULDs activated on this adapter */ int flags; int debug_flags; char ifp_lockname[16]; struct mtx ifp_lock; struct ifnet *ifp; /* tracer ifp */ struct ifmedia media; int traceq; /* iq used by all tracers, -1 if none */ int tracer_valid; /* bitmap of valid tracers */ int tracer_enabled; /* bitmap of enabled tracers */ char fw_version[16]; char tp_version[16]; char er_version[16]; char bs_version[16]; char cfg_file[32]; u_int cfcsum; struct adapter_params params; const struct chip_params *chip_params; struct t4_virt_res vres; uint16_t nbmcaps; uint16_t linkcaps; uint16_t switchcaps; uint16_t niccaps; uint16_t toecaps; uint16_t rdmacaps; uint16_t cryptocaps; uint16_t iscsicaps; uint16_t fcoecaps; struct sysctl_ctx_list ctx; struct sysctl_oid *ctrlq_oid; struct sysctl_oid *fwq_oid; struct mtx sc_lock; char lockname[16]; /* Starving free lists */ struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ TAILQ_HEAD(, sge_fl) sfl; struct callout sfl_callout; /* * Driver code that can run when the adapter is suspended must use this * lock or a synchronized_op and check for HW_OFF_LIMITS before * accessing hardware. * * XXX: could be changed to rwlock. wlock in suspend/resume and for * indirect register access, rlock everywhere else. */ struct mtx reg_lock; struct memwin memwin[NUM_MEMWIN]; /* memory windows */ struct mtx tc_lock; struct task tc_task; struct task reset_task; const void *reset_thread; int num_resets; int incarnation; const char *last_op; const void *last_op_thr; int last_op_flags; int swintr; int sensor_resets; struct callout ktls_tick; }; #define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) #define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) #define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) #define ASSERT_SYNCHRONIZED_OP(sc) \ KASSERT(IS_BUSY(sc) && \ (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ ("%s: operation not synchronized.", __func__)) #define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) #define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) #define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) #define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) #define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) #define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) #define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) #define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) #define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) #define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) #define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) #define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) #define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) #define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) #define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) #define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) #define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) #define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) #define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) #define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) #define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) #define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) #define for_each_txq(vi, iter, q) \ for (q = &vi->adapter->sge.txq[vi->first_txq], iter = 0; \ iter < vi->ntxq; ++iter, ++q) #define for_each_rxq(vi, iter, q) \ for (q = &vi->adapter->sge.rxq[vi->first_rxq], iter = 0; \ iter < vi->nrxq; ++iter, ++q) #define for_each_ofld_txq(vi, iter, q) \ for (q = &vi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \ iter < vi->nofldtxq; ++iter, ++q) #define for_each_ofld_rxq(vi, iter, q) \ for (q = &vi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \ iter < vi->nofldrxq; ++iter, ++q) #define for_each_nm_txq(vi, iter, q) \ for (q = &vi->adapter->sge.nm_txq[vi->first_nm_txq], iter = 0; \ iter < vi->nnmtxq; ++iter, ++q) #define for_each_nm_rxq(vi, iter, q) \ for (q = &vi->adapter->sge.nm_rxq[vi->first_nm_rxq], iter = 0; \ iter < vi->nnmrxq; ++iter, ++q) #define for_each_vi(_pi, _iter, _vi) \ for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \ ++(_iter), ++(_vi)) #define IDXINCR(idx, incr, wrap) do { \ idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \ } while (0) #define IDXDIFF(head, tail, wrap) \ ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) /* One for errors, one for firmware events */ #define T4_EXTRA_INTR 2 /* One for firmware events */ #define T4VF_EXTRA_INTR 1 static inline int forwarding_intr_to_fwq(struct adapter *sc) { return (sc->intr_count == 1); } /* Works reliably inside a sync_op or with reg_lock held. */ static inline bool hw_off_limits(struct adapter *sc) { return (__predict_false(sc->flags & HW_OFF_LIMITS)); } static inline uint32_t t4_read_reg(struct adapter *sc, uint32_t reg) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); return bus_space_read_4(sc->bt, sc->bh, reg); } static inline void t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); bus_space_write_4(sc->bt, sc->bh, reg, val); } static inline uint64_t t4_read_reg64(struct adapter *sc, uint32_t reg) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); #ifdef __LP64__ return bus_space_read_8(sc->bt, sc->bh, reg); #else return (uint64_t)bus_space_read_4(sc->bt, sc->bh, reg) + ((uint64_t)bus_space_read_4(sc->bt, sc->bh, reg + 4) << 32); #endif } static inline void t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); #ifdef __LP64__ bus_space_write_8(sc->bt, sc->bh, reg, val); #else bus_space_write_4(sc->bt, sc->bh, reg, val); bus_space_write_4(sc->bt, sc->bh, reg + 4, val>> 32); #endif } static inline void t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); *val = pci_read_config(sc->dev, reg, 1); } static inline void t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); pci_write_config(sc->dev, reg, val, 1); } static inline void t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); *val = pci_read_config(sc->dev, reg, 2); } static inline void t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); pci_write_config(sc->dev, reg, val, 2); } static inline void t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); *val = pci_read_config(sc->dev, reg, 4); } static inline void t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) { if (hw_off_limits(sc)) MPASS(curthread == sc->reset_thread); pci_write_config(sc->dev, reg, val, 4); } static inline struct port_info * adap2pinfo(struct adapter *sc, int idx) { return (sc->port[idx]); } static inline void t4_os_set_hw_addr(struct port_info *pi, uint8_t hw_addr[]) { bcopy(hw_addr, pi->vi[0].hw_addr, ETHER_ADDR_LEN); } static inline int tx_resume_threshold(struct sge_eq *eq) { /* not quite the same as qsize / 4, but this will do. */ return (eq->sidx / 4); } static inline int t4_use_ldst(struct adapter *sc) { #ifdef notyet return (sc->flags & FW_OK || !sc->use_bd); #else return (0); #endif } static inline void CH_DUMP_MBOX(struct adapter *sc, int mbox, const int reg, const char *msg, const __be64 *const p, const bool err) { if (!(sc->debug_flags & DF_DUMP_MBOX) && !err) return; if (p != NULL) { log(err ? LOG_ERR : LOG_DEBUG, "%s: mbox %u %s %016llx %016llx %016llx %016llx " "%016llx %016llx %016llx %016llx\n", device_get_nameunit(sc->dev), mbox, msg, (long long)be64_to_cpu(p[0]), (long long)be64_to_cpu(p[1]), (long long)be64_to_cpu(p[2]), (long long)be64_to_cpu(p[3]), (long long)be64_to_cpu(p[4]), (long long)be64_to_cpu(p[5]), (long long)be64_to_cpu(p[6]), (long long)be64_to_cpu(p[7])); } else { log(err ? LOG_ERR : LOG_DEBUG, "%s: mbox %u %s %016llx %016llx %016llx %016llx " "%016llx %016llx %016llx %016llx\n", device_get_nameunit(sc->dev), mbox, msg, (long long)t4_read_reg64(sc, reg), (long long)t4_read_reg64(sc, reg + 8), (long long)t4_read_reg64(sc, reg + 16), (long long)t4_read_reg64(sc, reg + 24), (long long)t4_read_reg64(sc, reg + 32), (long long)t4_read_reg64(sc, reg + 40), (long long)t4_read_reg64(sc, reg + 48), (long long)t4_read_reg64(sc, reg + 56)); } } /* t4_main.c */ extern int t4_ntxq; extern int t4_nrxq; extern int t4_intr_types; extern int t4_tmr_idx; extern int t4_pktc_idx; extern unsigned int t4_qsize_rxq; extern unsigned int t4_qsize_txq; extern device_method_t cxgbe_methods[]; int t4_os_find_pci_capability(struct adapter *, int); int t4_os_pci_save_state(struct adapter *); int t4_os_pci_restore_state(struct adapter *); void t4_os_portmod_changed(struct port_info *); void t4_os_link_changed(struct port_info *); void t4_iterate(void (*)(struct adapter *, void *), void *); void t4_init_devnames(struct adapter *); void t4_add_adapter(struct adapter *); int t4_detach_common(device_t); int t4_map_bars_0_and_4(struct adapter *); int t4_map_bar_2(struct adapter *); int t4_setup_intr_handlers(struct adapter *); void t4_sysctls(struct adapter *); int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *); void doom_vi(struct adapter *, struct vi_info *); void end_synchronized_op(struct adapter *, int); int update_mac_settings(struct ifnet *, int); int adapter_init(struct adapter *); int vi_init(struct vi_info *); void vi_sysctls(struct vi_info *); int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int); int alloc_atid(struct adapter *, void *); void *lookup_atid(struct adapter *, int); void free_atid(struct adapter *, int); void release_tid(struct adapter *, int, struct sge_wrq *); int cxgbe_media_change(struct ifnet *); void cxgbe_media_status(struct ifnet *, struct ifmediareq *); bool t4_os_dump_cimla(struct adapter *, int, bool); void t4_os_dump_devlog(struct adapter *); #ifdef KERN_TLS /* t4_kern_tls.c */ int cxgbe_tls_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, struct m_snd_tag **); void cxgbe_tls_tag_free(struct m_snd_tag *); void t6_ktls_modload(void); void t6_ktls_modunload(void); int t6_ktls_try(struct ifnet *, struct socket *, struct ktls_session *); int t6_ktls_parse_pkt(struct mbuf *, int *, int *); int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int, u_int); #endif /* t4_keyctx.c */ struct auth_hash; union authctx; #ifdef KERN_TLS struct ktls_session; struct tls_key_req; struct tls_keyctx; #endif void t4_aes_getdeckey(void *, const void *, unsigned int); void t4_copy_partial_hash(int, union authctx *, void *); void t4_init_gmac_hash(const char *, int, char *); -void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *); +void t4_init_hmac_digest(const struct auth_hash *, u_int, const char *, int, + char *); #ifdef KERN_TLS u_int t4_tls_key_info_size(const struct ktls_session *); int t4_tls_proto_ver(const struct ktls_session *); int t4_tls_cipher_mode(const struct ktls_session *); int t4_tls_auth_mode(const struct ktls_session *); int t4_tls_hmac_ctrl(const struct ktls_session *); void t4_tls_key_ctx(const struct ktls_session *, int, struct tls_keyctx *); int t4_alloc_tls_keyid(struct adapter *); void t4_free_tls_keyid(struct adapter *, int); void t4_write_tlskey_wr(const struct ktls_session *, int, int, int, int, struct tls_key_req *); #endif #ifdef DEV_NETMAP /* t4_netmap.c */ struct sge_nm_rxq; void cxgbe_nm_attach(struct vi_info *); void cxgbe_nm_detach(struct vi_info *); void service_nm_rxq(struct sge_nm_rxq *); int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int); int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int); int free_nm_txq(struct vi_info *, struct sge_nm_txq *); #endif /* t4_sge.c */ void t4_sge_modload(void); void t4_sge_modunload(void); uint64_t t4_sge_extfree_refs(void); void t4_tweak_chip_settings(struct adapter *); int t4_verify_chip_settings(struct adapter *); void t4_init_rx_buf_info(struct adapter *); int t4_create_dma_tag(struct adapter *); void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid_list *); int t4_destroy_dma_tag(struct adapter *); int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, bus_addr_t *, void **); int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, void *); void free_fl_buffers(struct adapter *, struct sge_fl *); int t4_setup_adapter_queues(struct adapter *); int t4_teardown_adapter_queues(struct adapter *); int t4_setup_vi_queues(struct vi_info *); int t4_teardown_vi_queues(struct vi_info *); void t4_intr_all(void *); void t4_intr(void *); #ifdef DEV_NETMAP void t4_nm_intr(void *); void t4_vi_intr(void *); #endif void t4_intr_err(void *); void t4_intr_evt(void *); void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); void t4_update_fl_bufsize(struct ifnet *); struct mbuf *alloc_wr_mbuf(int, int); int parse_pkt(struct mbuf **, bool); void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *); void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *); int tnl_cong(struct port_info *, int); void t4_register_an_handler(an_handler_t); void t4_register_fw_msg_handler(int, fw_msg_handler_t); void t4_register_cpl_handler(int, cpl_handler_t); void t4_register_shared_cpl_handler(int, cpl_handler_t, int); #ifdef RATELIMIT int ethofld_transmit(struct ifnet *, struct mbuf *); void send_etid_flush_wr(struct cxgbe_rate_tag *); #endif /* t4_tracer.c */ struct t4_tracer; void t4_tracer_modload(void); void t4_tracer_modunload(void); void t4_tracer_port_detach(struct adapter *); int t4_get_tracer(struct adapter *, struct t4_tracer *); int t4_set_tracer(struct adapter *, struct t4_tracer *); int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); /* t4_sched.c */ int t4_set_sched_class(struct adapter *, struct t4_sched_params *); int t4_set_sched_queue(struct adapter *, struct t4_sched_queue *); int t4_init_tx_sched(struct adapter *); int t4_free_tx_sched(struct adapter *); void t4_update_tx_sched(struct adapter *); int t4_reserve_cl_rl_kbps(struct adapter *, int, u_int, int *); void t4_release_cl_rl(struct adapter *, int, int); int sysctl_tc(SYSCTL_HANDLER_ARGS); int sysctl_tc_params(SYSCTL_HANDLER_ARGS); #ifdef RATELIMIT void t4_init_etid_table(struct adapter *); void t4_free_etid_table(struct adapter *); struct cxgbe_rate_tag *lookup_etid(struct adapter *, int); int cxgbe_rate_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, struct m_snd_tag **); int cxgbe_rate_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *); int cxgbe_rate_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *); void cxgbe_rate_tag_free(struct m_snd_tag *); void cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *); void cxgbe_ratelimit_query(struct ifnet *, struct if_ratelimit_query_results *); #endif /* t4_filter.c */ int get_filter_mode(struct adapter *, uint32_t *); int set_filter_mode(struct adapter *, uint32_t); int set_filter_mask(struct adapter *, uint32_t); int get_filter(struct adapter *, struct t4_filter *); int set_filter(struct adapter *, struct t4_filter *); int del_filter(struct adapter *, struct t4_filter *); int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_hashfilter_ao_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_hashfilter_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); int t4_del_hashfilter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); void free_hftid_hash(struct tid_info *); static inline struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq) { int len = offsetof(struct wrqe, wr) + wr_len; struct wrqe *wr; wr = malloc(len, M_CXGBE, M_NOWAIT); if (__predict_false(wr == NULL)) return (NULL); wr->wr_len = wr_len; wr->wrq = wrq; return (wr); } static inline void * wrtod(struct wrqe *wr) { return (&wr->wr[0]); } static inline void free_wrqe(struct wrqe *wr) { free(wr, M_CXGBE); } static inline void t4_wrq_tx(struct adapter *sc, struct wrqe *wr) { struct sge_wrq *wrq = wr->wrq; TXQ_LOCK(wrq); t4_wrq_tx_locked(sc, wrq, wr); TXQ_UNLOCK(wrq); } static inline int read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, val, len, 0)); } static inline int write_via_memwin(struct adapter *sc, int idx, uint32_t addr, const uint32_t *val, int len) { return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1)); } /* Number of len16 -> number of descriptors */ static inline int tx_len16_to_desc(int len16) { return (howmany(len16, EQ_ESIZE / 16)); } #endif diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c index 0fc806dc2eb8..3ce3e5c916db 100644 --- a/sys/dev/cxgbe/crypto/t4_crypto.c +++ b/sys/dev/cxgbe/crypto/t4_crypto.c @@ -1,2982 +1,2982 @@ /*- * Copyright (c) 2017 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "common/common.h" #include "crypto/t4_crypto.h" /* * Requests consist of: * * +-------------------------------+ * | struct fw_crypto_lookaside_wr | * +-------------------------------+ * | struct ulp_txpkt | * +-------------------------------+ * | struct ulptx_idata | * +-------------------------------+ * | struct cpl_tx_sec_pdu | * +-------------------------------+ * | struct cpl_tls_tx_scmd_fmt | * +-------------------------------+ * | key context header | * +-------------------------------+ * | AES key | ----- For requests with AES * +-------------------------------+ * | Hash state | ----- For hash-only requests * +-------------------------------+ - * | IPAD (16-byte aligned) | \ * +-------------------------------+ +---- For requests with HMAC * | OPAD (16-byte aligned) | / * +-------------------------------+ - * | GMAC H | ----- For AES-GCM * +-------------------------------+ - * | struct cpl_rx_phys_dsgl | \ * +-------------------------------+ +---- Destination buffer for * | PHYS_DSGL entries | / non-hash-only requests * +-------------------------------+ - * | 16 dummy bytes | ----- Only for HMAC/hash-only requests * +-------------------------------+ * | IV | ----- If immediate IV * +-------------------------------+ * | Payload | ----- If immediate Payload * +-------------------------------+ - * | struct ulptx_sgl | \ * +-------------------------------+ +---- If payload via SGL * | SGL entries | / * +-------------------------------+ - * * Note that the key context must be padded to ensure 16-byte alignment. * For HMAC requests, the key consists of the partial hash of the IPAD * followed by the partial hash of the OPAD. * * Replies consist of: * * +-------------------------------+ * | struct cpl_fw6_pld | * +-------------------------------+ * | hash digest | ----- For HMAC request with * +-------------------------------+ 'hash_size' set in work request * * A 32-bit big-endian error status word is supplied in the last 4 * bytes of data[0] in the CPL_FW6_PLD message. bit 0 indicates a * "MAC" error and bit 1 indicates a "PAD" error. * * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message * in the request is returned in data[1] of the CPL_FW6_PLD message. * * For block cipher replies, the updated IV is supplied in data[2] and * data[3] of the CPL_FW6_PLD message. * * For hash replies where the work request set 'hash_size' to request * a copy of the hash in the reply, the hash digest is supplied * immediately following the CPL_FW6_PLD message. */ /* * The crypto engine supports a maximum AAD size of 511 bytes. */ #define MAX_AAD_LEN 511 /* * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG * entries. While the CPL includes a 16-bit length field, the T6 can * sometimes hang if an error occurs while processing a request with a * single DSGL entry larger than 2k. */ #define MAX_RX_PHYS_DSGL_SGE 32 #define DSGL_SGE_MAXLEN 2048 /* * The adapter only supports requests with a total input or output * length of 64k-1 or smaller. Longer requests either result in hung * requests or incorrect results. */ #define MAX_REQUEST_SIZE 65535 static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto"); struct ccr_session_hmac { - struct auth_hash *auth_hash; + const struct auth_hash *auth_hash; int hash_len; unsigned int partial_digest_len; unsigned int auth_mode; unsigned int mk_size; char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2]; }; struct ccr_session_gmac { int hash_len; char ghash_h[GMAC_BLOCK_LEN]; }; struct ccr_session_ccm_mac { int hash_len; }; struct ccr_session_blkcipher { unsigned int cipher_mode; unsigned int key_len; unsigned int iv_len; __be32 key_ctx_hdr; char enckey[CHCR_AES_MAX_KEY_LEN]; char deckey[CHCR_AES_MAX_KEY_LEN]; }; struct ccr_port { struct sge_wrq *txq; struct sge_rxq *rxq; int rx_channel_id; int tx_channel_id; u_int active_sessions; counter_u64_t stats_queued; counter_u64_t stats_completed; }; struct ccr_session { #ifdef INVARIANTS int pending; #endif enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; struct ccr_port *port; union { struct ccr_session_hmac hmac; struct ccr_session_gmac gmac; struct ccr_session_ccm_mac ccm_mac; }; struct ccr_session_blkcipher blkcipher; struct mtx lock; /* * Pre-allocate S/G lists used when preparing a work request. * 'sg_input' contains an sglist describing the entire input * buffer for a 'struct cryptop'. 'sg_output' contains an * sglist describing the entire output buffer. 'sg_ulptx' is * used to describe the data the engine should DMA as input * via ULPTX_SGL. 'sg_dsgl' is used to describe the * destination that cipher text and a tag should be written * to. */ struct sglist *sg_input; struct sglist *sg_output; struct sglist *sg_ulptx; struct sglist *sg_dsgl; }; struct ccr_softc { struct adapter *adapter; device_t dev; uint32_t cid; struct mtx lock; bool detaching; struct ccr_port ports[MAX_NPORTS]; u_int port_mask; int first_rxq_id; /* * Pre-allocate a dummy output buffer for the IV and AAD for * AEAD requests. */ char *iv_aad_buf; struct sglist *sg_iv_aad; /* Statistics. */ counter_u64_t stats_blkcipher_encrypt; counter_u64_t stats_blkcipher_decrypt; counter_u64_t stats_hash; counter_u64_t stats_hmac; counter_u64_t stats_eta_encrypt; counter_u64_t stats_eta_decrypt; counter_u64_t stats_gcm_encrypt; counter_u64_t stats_gcm_decrypt; counter_u64_t stats_ccm_encrypt; counter_u64_t stats_ccm_decrypt; counter_u64_t stats_wr_nomem; counter_u64_t stats_inflight; counter_u64_t stats_mac_error; counter_u64_t stats_pad_error; counter_u64_t stats_sglist_error; counter_u64_t stats_process_error; counter_u64_t stats_sw_fallback; }; /* * Crypto requests involve two kind of scatter/gather lists. * * Non-hash-only requests require a PHYS_DSGL that describes the * location to store the results of the encryption or decryption * operation. This SGL uses a different format (PHYS_DSGL) and should * exclude the skip bytes at the start of the data as well as any AAD * or IV. For authenticated encryption requests it should include the * destination of the hash or tag. * * The input payload may either be supplied inline as immediate data, * or via a standard ULP_TX SGL. This SGL should include AAD, * ciphertext, and the hash or tag for authenticated decryption * requests. * * These scatter/gather lists can describe different subsets of the * buffers described by the crypto operation. ccr_populate_sglist() * generates a scatter/gather list that covers an entire crypto * operation buffer that is then used to construct the other * scatter/gather lists. */ static int ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) { int error; sglist_reset(sg); switch (cb->cb_type) { case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_SINGLE_MBUF: error = sglist_append_single_mbuf(sg, cb->cb_mbuf); break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; case CRYPTO_BUF_VMPAGE: error = sglist_append_vmpages(sg, cb->cb_vm_page, cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; } return (error); } /* * Segments in 'sg' larger than 'maxsegsize' are counted as multiple * segments. */ static int ccr_count_sgl(struct sglist *sg, int maxsegsize) { int i, nsegs; nsegs = 0; for (i = 0; i < sg->sg_nseg; i++) nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize); return (nsegs); } /* These functions deal with PHYS_DSGL for the reply buffer. */ static inline int ccr_phys_dsgl_len(int nsegs) { int len; len = (nsegs / 8) * sizeof(struct phys_sge_pairs); if ((nsegs % 8) != 0) { len += sizeof(uint16_t) * 8; len += roundup2(nsegs % 8, 2) * sizeof(uint64_t); } return (len); } static void ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs) { struct sglist *sg; struct cpl_rx_phys_dsgl *cpl; struct phys_sge_pairs *sgl; vm_paddr_t paddr; size_t seglen; u_int i, j; sg = s->sg_dsgl; cpl = dst; cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | V_CPL_RX_PHYS_DSGL_ISRDMA(0)); cpl->pcirlxorder_to_noofsgentr = htobe32( V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) | V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) | V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) | V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs)); cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; cpl->rss_hdr_int.qid = htobe16(s->port->rxq->iq.abs_id); cpl->rss_hdr_int.hash_val = 0; cpl->rss_hdr_int.channel = s->port->rx_channel_id; sgl = (struct phys_sge_pairs *)(cpl + 1); j = 0; for (i = 0; i < sg->sg_nseg; i++) { seglen = sg->sg_segs[i].ss_len; paddr = sg->sg_segs[i].ss_paddr; do { sgl->addr[j] = htobe64(paddr); if (seglen > DSGL_SGE_MAXLEN) { sgl->len[j] = htobe16(DSGL_SGE_MAXLEN); paddr += DSGL_SGE_MAXLEN; seglen -= DSGL_SGE_MAXLEN; } else { sgl->len[j] = htobe16(seglen); seglen = 0; } j++; if (j == 8) { sgl++; j = 0; } } while (seglen != 0); } MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs); } /* These functions deal with the ULPTX_SGL for input payload. */ static inline int ccr_ulptx_sgl_len(int nsegs) { u_int n; nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); return (roundup2(n, 16)); } static void ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs) { struct ulptx_sgl *usgl; struct sglist *sg; struct sglist_seg *ss; int i; sg = s->sg_ulptx; MPASS(nsegs == sg->sg_nseg); ss = &sg->sg_segs[0]; usgl = dst; usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); usgl->len0 = htobe32(ss->ss_len); usgl->addr0 = htobe64(ss->ss_paddr); ss++; for (i = 0; i < sg->sg_nseg - 1; i++) { usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr); ss++; } } static bool ccr_use_imm_data(u_int transhdr_len, u_int input_len) { if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN) return (false); if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) > SGE_MAX_WR_LEN) return (false); return (true); } static void ccr_populate_wreq(struct ccr_softc *sc, struct ccr_session *s, struct chcr_wr *crwr, u_int kctx_len, u_int wr_len, u_int imm_len, u_int sgl_len, u_int hash_size, struct cryptop *crp) { u_int cctx_size, idata_len; cctx_size = sizeof(struct _key_ctx) + kctx_len; crwr->wreq.op_to_cctx_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) | V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) | V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4)); crwr->wreq.len16_pkd = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16)); crwr->wreq.session_id = 0; crwr->wreq.rx_chid_to_rx_q_id = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(s->port->rx_channel_id) | V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) | V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) | V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) | V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) | V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) | /* unused in firmware */ V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(s->port->rxq->iq.abs_id)); crwr->wreq.key_addr = 0; crwr->wreq.pld_size_hash_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) | V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size)); crwr->wreq.cookie = htobe64((uintptr_t)crp); crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DATAMODIFY(0) | V_ULP_TXPKT_CHANNELID(s->port->tx_channel_id) | V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(sc->first_rxq_id) | V_ULP_TXPKT_RO(1)); crwr->ulptx.len = htobe32( ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16)); crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | V_ULP_TX_SC_MORE(sgl_len != 0 ? 1 : 0)); idata_len = wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len; if (imm_len % 16 != 0) idata_len -= 16 - imm_len % 16; crwr->sc_imm.len = htobe32(idata_len); } static int ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { struct chcr_wr *crwr; struct wrqe *wr; - struct auth_hash *axf; + const struct auth_hash *axf; char *dst; u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len; u_int hmac_ctrl, imm_len, iopad_size; int error, sgl_nsegs, sgl_len, use_opad; /* Reject requests with too large of an input buffer. */ if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); axf = s->hmac.auth_hash; if (s->mode == HMAC) { use_opad = 1; hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; } else { use_opad = 0; hmac_ctrl = SCMD_HMAC_CTRL_NOP; } /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the context includes the aligned IPAD and * OPAD. */ kctx_len = iopad_size; if (use_opad) kctx_len += iopad_size; hash_size_in_response = axf->hashsize; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); if (crp->crp_payload_length == 0) { imm_len = axf->blocksize; sgl_nsegs = 0; sgl_len = 0; } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) { imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, hash_size_in_response, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(0)); crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ? axf->blocksize : crp->crp_payload_length); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_LAST_FRAG(0) | V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1)); memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len); /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */ kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_OPAD_PRESENT(use_opad) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) | V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1)); dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES; if (crp->crp_payload_length == 0) { dst[0] = 0x80; if (s->mode == HMAC) *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = htobe64(axf->blocksize << 3); } else if (imm_len != 0) crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { uint8_t hash[HASH_MAX_LEN]; if (error) return (error); if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len, hash); if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0) return (EBADMSG); } else crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len, (cpl + 1)); return (0); } static int ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int imm_len, iv_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); /* Reject requests with too large of an input buffer. */ if (crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; sglist_reset(s->sg_dsgl); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* The 'key' must be 128-bit aligned. */ kctx_len = roundup2(s->blkcipher.key_len, 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* For AES-XTS we send a 16-byte IV in the work request. */ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) { imm_len = crp->crp_payload_length; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); crypto_read_iv(crp, iv); /* Zero the remainder of the IV for AES-XTS. */ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. */ return (error); } /* * 'hashsize' is the length of a full digest. 'authsize' is the * requested digest length for this operation which may be less * than 'hashsize'. */ static int ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize) { if (authsize == 10) return (SCMD_HMAC_CTRL_TRUNC_RFC4366); if (authsize == 12) return (SCMD_HMAC_CTRL_IPSEC_96BIT); if (authsize == hashsize / 2) return (SCMD_HMAC_CTRL_DIV2); return (SCMD_HMAC_CTRL_NO_TRUNC); } static int ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; - struct auth_hash *axf; + const struct auth_hash *axf; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len, iopad_size, iv_len; u_int aad_start, aad_stop; u_int auth_insert; u_int cipher_start, cipher_stop; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; /* * If there is a need in the future, requests with an empty * payload could be supported as HMAC-only requests. */ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) return (EINVAL); if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC && (crp->crp_payload_length % AES_BLOCK_LEN) != 0) return (EINVAL); /* For AES-XTS we send a 16-byte IV in the work request. */ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; if (crp->crp_aad_length + iv_len > MAX_AAD_LEN) return (EINVAL); axf = s->hmac.auth_hash; hash_size_in_response = s->hmac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The output buffer consists of the cipher text followed by * the hash when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + crp->crp_aad_length + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the key context consists of the key followed * by the IPAD and OPAD. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = crp->crp_aad_length + crp->crp_payload_length; /* * The firmware hangs if sent a request which is a * bit smaller than MAX_REQUEST_SIZE. In particular, the * firmware appears to require 512 - 16 bytes of spare room * along with the size of the hash even if the hash isn't * included in the input buffer. */ if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) > MAX_REQUEST_SIZE) return (EFBIG); if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } /* Any AAD comes after the IV. */ if (crp->crp_aad_length != 0) { aad_start = iv_len + 1; aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); crypto_read_iv(crp, iv); /* Zero the remainder of the IV for AES-XTS. */ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_CTR: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case SCMD_CIPH_MODE_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->hmac.pads, iopad_size * 2); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. */ return (error); } static int ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len; u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0) return (EINVAL); /* * The crypto engine doesn't handle GCM requests with an empty * payload, so handle those in software instead. */ if (crp->crp_payload_length == 0) return (EMSGSIZE); if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->gmac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The IV handling for GCM in OCF is a bit more complicated in * that IPSec provides a full 16-byte IV (including the * counter), whereas the /dev/crypto interface sometimes * provides a full 16-byte IV (if no IV is provided in the * ioctl) and sometimes a 12-byte IV (if the IV was explicit). * * When provided a 12-byte IV, assume the IV is really 16 bytes * with a counter in the last 4 bytes initialized to 1. * * While iv_len is checked below, the value is currently * always set to 12 when creating a GCM session in this driver * due to limitations in OCF (there is no way to know what the * IV length of a given request will be). This means that the * driver always assumes as 12-byte IV for now. */ if (s->blkcipher.iv_len == 12) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; /* * GCM requests should always provide an explicit IV. */ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + crp->crp_aad_length + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + crp->crp_aad_length + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* * The 'key' part of the key context consists of the key followed * by the Galois hash key. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = crp->crp_aad_length + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) return (EFBIG); if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } if (crp->crp_aad_length != 0) { aad_start = iv_len + 1; aad_stop = aad_start + crp->crp_aad_length - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = iv_len + crp->crp_aad_length + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); if (s->blkcipher.iv_len == 12) *(uint32_t *)&iv[12] = htobe32(1); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); /* * NB: cipherstop is explicitly set to 0. On encrypt it * should normally be set to 0 anyway. However, for decrypt * the cipher ends before the tag in the ETA case (and * authstop is set to stop before the tag), but for GCM the * cipher still runs to the end of the buffer. Not sure if * this is intentional or a firmware quirk, but it is required * for working tag validation with GCM decryption. */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * Note that the hardware should always verify the GMAC hash. */ return (error); } /* * Handle a GCM request that is not supported by the crypto engine by * performing the operation in software. Derived from swcr_authenc(). */ static void ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp) { - struct auth_hash *axf; - struct enc_xform *exf; + const struct auth_hash *axf; + const struct enc_xform *exf; void *auth_ctx, *kschedule; char block[GMAC_BLOCK_LEN]; char digest[GMAC_DIGEST_LEN]; char iv[AES_BLOCK_LEN]; int error, i, len; auth_ctx = NULL; kschedule = NULL; /* Initialize the MAC. */ switch (s->blkcipher.key_len) { case 16: axf = &auth_hash_nist_gmac_aes_128; break; case 24: axf = &auth_hash_nist_gmac_aes_192; break; case 32: axf = &auth_hash_nist_gmac_aes_256; break; default: error = EINVAL; goto out; } auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); if (auth_ctx == NULL) { error = ENOMEM; goto out; } axf->Init(auth_ctx); axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); /* Initialize the cipher. */ exf = &enc_xform_aes_nist_gcm; kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); if (kschedule == NULL) { error = ENOMEM; goto out; } error = exf->setkey(kschedule, s->blkcipher.enckey, s->blkcipher.key_len); if (error) goto out; /* * This assumes a 12-byte IV from the crp. See longer comment * above in ccr_gcm() for more details. */ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { error = EINVAL; goto out; } memcpy(iv, crp->crp_iv, 12); *(uint32_t *)&iv[12] = htobe32(1); axf->Reinit(auth_ctx, iv, sizeof(iv)); /* MAC the AAD. */ if (crp->crp_aad != NULL) { len = rounddown(crp->crp_aad_length, sizeof(block)); if (len != 0) axf->Update(auth_ctx, crp->crp_aad, len); if (crp->crp_aad_length != len) { memset(block, 0, sizeof(block)); memcpy(block, (char *)crp->crp_aad + len, crp->crp_aad_length - len); axf->Update(auth_ctx, block, sizeof(block)); } } else { for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { len = imin(crp->crp_aad_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_aad_start + i, len, block); bzero(block + len, sizeof(block) - len); axf->Update(auth_ctx, block, sizeof(block)); } } exf->reinit(kschedule, iv); /* Do encryption with MAC */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt(kschedule, block, block); axf->Update(auth_ctx, block, len); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } else { axf->Update(auth_ctx, block, len); } } /* Length block. */ bzero(block, sizeof(block)); ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8); ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8); axf->Update(auth_ctx, block, sizeof(block)); /* Finalize MAC. */ axf->Final(digest, auth_ctx); /* Inject or validate tag. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), digest); error = 0; } else { char digest2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } } else error = EBADMSG; explicit_bzero(digest2, sizeof(digest2)); } out: zfree(kschedule, M_CCR); zfree(auth_ctx, M_CCR); explicit_bzero(block, sizeof(block)); explicit_bzero(iv, sizeof(iv)); explicit_bzero(digest, sizeof(digest)); crp->crp_etype = error; crypto_done(crp); } static void generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response, const char *iv, char *b0) { u_int i, payload_len; /* NB: L is already set in the first byte of the IV. */ memcpy(b0, iv, CCM_B0_SIZE); /* Set length of hash in bits 3 - 5. */ b0[0] |= (((hash_size_in_response - 2) / 2) << 3); /* Store the payload length as a big-endian value. */ payload_len = crp->crp_payload_length; for (i = 0; i < iv[0]; i++) { b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len; payload_len >>= 8; } /* * If there is AAD in the request, set bit 6 in the flags * field and store the AAD length as a big-endian value at the * start of block 1. This only assumes a 16-bit AAD length * since T6 doesn't support large AAD sizes. */ if (crp->crp_aad_length != 0) { b0[0] |= (1 << 6); *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length); } } static int ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct ulptx_idata *idata; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; u_int aad_len, b0_len, hash_size_in_response, imm_len; u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0) return (EINVAL); /* * The crypto engine doesn't handle CCM requests with an empty * payload, so handle those in software instead. */ if (crp->crp_payload_length == 0) return (EMSGSIZE); /* * CCM always includes block 0 in the AAD before AAD from the * request. */ b0_len = CCM_B0_SIZE; if (crp->crp_aad_length != 0) b0_len += CCM_AAD_FIELD_SIZE; aad_len = b0_len + crp->crp_aad_length; /* * CCM requests should always provide an explicit IV (really * the nonce). */ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* * Always assume a 12 byte input nonce for now since that is * what OCF always generates. The full IV in the work request * is 16 bytes. */ iv_len = AES_BLOCK_LEN; if (iv_len + aad_len > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->ccm_mac.hash_len; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + aad_len + crp->crp_payload_length + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + aad_len + crp->crp_payload_length > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(s->sg_dsgl); error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + aad_len); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* * The 'key' part of the key context consists of two copies of * the AES key. */ kctx_len = roundup2(s->blkcipher.key_len, 16) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, AAD (including block * 0), and then the cipher/plain text. For decryption * requests the hash is appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = aad_len + crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) return (EFBIG); if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { /* Block 0 is passed as immediate data. */ imm_len = b0_len; sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } aad_start = iv_len + 1; aad_stop = aad_start + aad_len - 1; cipher_start = aad_stop + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); /* * Read the nonce from the request. Use the nonce to generate * the full IV with the counter set to 0. */ memset(iv, 0, iv_len); iv[0] = (15 - AES_CCM_IV_LEN) - 1; memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN); ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); /* * NB: cipherstop is explicitly set to 0. See comments above * in ccr_gcm(). */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(AES_CBC_MAC_HASH_LEN, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 0 : 1) | V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CCM) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_CBCMAC) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16), s->blkcipher.enckey, s->blkcipher.key_len); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; generate_ccm_b0(crp, hash_size_in_response, iv, dst); if (sgl_nsegs == 0) { dst += b0_len; if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) memcpy(dst, crp->crp_aad, crp->crp_aad_length); else crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, dst); dst += crp->crp_aad_length; } crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); dst += crp->crp_payload_length; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else { dst += CCM_B0_SIZE; if (b0_len > CCM_B0_SIZE) { /* * If there is AAD, insert padding including a * ULP_TX_SC_NOOP so that the ULP_TX_SC_DSGL * is 16-byte aligned. */ KASSERT(b0_len - CCM_B0_SIZE == CCM_AAD_FIELD_SIZE, ("b0_len mismatch")); memset(dst + CCM_AAD_FIELD_SIZE, 0, 8 - CCM_AAD_FIELD_SIZE); idata = (void *)(dst + 8); idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); idata->len = htobe32(0); dst = (void *)(idata + 1); } ccr_write_ulptx_sgl(s, dst, sgl_nsegs); } /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); explicit_bzero(iv, sizeof(iv)); return (0); } static int ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * Note that the hardware should always verify the CBC MAC * hash. */ return (error); } /* * Handle a CCM request that is not supported by the crypto engine by * performing the operation in software. Derived from swcr_authenc(). */ static void ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp) { - struct auth_hash *axf; - struct enc_xform *exf; + const struct auth_hash *axf; + const struct enc_xform *exf; union authctx *auth_ctx; void *kschedule; char block[CCM_CBC_BLOCK_LEN]; char digest[AES_CBC_MAC_HASH_LEN]; char iv[AES_CCM_IV_LEN]; int error, i, len; auth_ctx = NULL; kschedule = NULL; /* Initialize the MAC. */ switch (s->blkcipher.key_len) { case 16: axf = &auth_hash_ccm_cbc_mac_128; break; case 24: axf = &auth_hash_ccm_cbc_mac_192; break; case 32: axf = &auth_hash_ccm_cbc_mac_256; break; default: error = EINVAL; goto out; } auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); if (auth_ctx == NULL) { error = ENOMEM; goto out; } axf->Init(auth_ctx); axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); /* Initialize the cipher. */ exf = &enc_xform_ccm; kschedule = malloc(exf->ctxsize, M_CCR, M_NOWAIT); if (kschedule == NULL) { error = ENOMEM; goto out; } error = exf->setkey(kschedule, s->blkcipher.enckey, s->blkcipher.key_len); if (error) goto out; if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { error = EINVAL; goto out; } memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN); auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; axf->Reinit(auth_ctx, iv, sizeof(iv)); /* MAC the AAD. */ if (crp->crp_aad != NULL) error = axf->Update(auth_ctx, crp->crp_aad, crp->crp_aad_length); else error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, auth_ctx); if (error) goto out; exf->reinit(kschedule, iv); /* Do encryption/decryption with MAC */ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { axf->Update(auth_ctx, block, len); exf->encrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } else { exf->decrypt(kschedule, block, block); axf->Update(auth_ctx, block, len); } } /* Finalize MAC. */ axf->Final(digest, auth_ctx); /* Inject or validate tag. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), digest); error = 0; } else { char digest2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ exf->reinit(kschedule, iv); for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { len = imin(crp->crp_payload_length - i, sizeof(block)); crypto_copydata(crp, crp->crp_payload_start + i, len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block, block); crypto_copyback(crp, crp->crp_payload_start + i, len, block); } } else error = EBADMSG; explicit_bzero(digest2, sizeof(digest2)); } out: zfree(kschedule, M_CCR); zfree(auth_ctx, M_CCR); explicit_bzero(block, sizeof(block)); explicit_bzero(iv, sizeof(iv)); explicit_bzero(digest, sizeof(digest)); crp->crp_etype = error; crypto_done(crp); } static void ccr_identify(driver_t *driver, device_t parent) { struct adapter *sc; sc = device_get_softc(parent); if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE && device_find_child(parent, "ccr", -1) == NULL) device_add_child(parent, "ccr", -1); } static int ccr_probe(device_t dev) { device_set_desc(dev, "Chelsio Crypto Accelerator"); return (BUS_PROBE_DEFAULT); } static void ccr_sysctls(struct ccr_softc *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid, *port_oid; struct sysctl_oid_list *children; char buf[16]; int i; ctx = device_get_sysctl_ctx(sc->dev); /* * dev.ccr.X. */ oid = device_get_sysctl_tree(sc->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "port_mask", CTLFLAG_RW, &sc->port_mask, 0, "Mask of enabled ports"); /* * dev.ccr.X.stats. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, &sc->stats_hash, "Hash requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, &sc->stats_hmac, "HMAC requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, &sc->stats_blkcipher_encrypt, "Cipher encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, &sc->stats_blkcipher_decrypt, "Cipher decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, &sc->stats_eta_encrypt, "Combined AES+HMAC encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, &sc->stats_eta_decrypt, "Combined AES+HMAC decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, &sc->stats_gcm_encrypt, "AES-GCM encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, &sc->stats_gcm_decrypt, "AES-GCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD, &sc->stats_ccm_encrypt, "AES-CCM encryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD, &sc->stats_ccm_decrypt, "AES-CCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, &sc->stats_wr_nomem, "Work request memory allocation failures"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, &sc->stats_inflight, "Requests currently pending"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, &sc->stats_mac_error, "MAC errors"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, &sc->stats_pad_error, "Padding errors"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, &sc->stats_sglist_error, "Requests for which DMA mapping failed"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, &sc->stats_process_error, "Requests failed during queueing"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, &sc->stats_sw_fallback, "Requests processed by falling back to software"); /* * dev.ccr.X.stats.port */ port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics"); for (i = 0; i < nitems(sc->ports); i++) { if (sc->ports[i].rxq == NULL) continue; /* * dev.ccr.X.stats.port.Y */ snprintf(buf, sizeof(buf), "%d", i); oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(port_oid), OID_AUTO, buf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, buf); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "active_sessions", CTLFLAG_RD, &sc->ports[i].active_sessions, 0, "Count of active sessions"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "queued", CTLFLAG_RD, &sc->ports[i].stats_queued, "Requests queued"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "completed", CTLFLAG_RD, &sc->ports[i].stats_completed, "Requests completed"); } } static void ccr_init_port(struct ccr_softc *sc, int port) { struct port_info *pi; pi = sc->adapter->port[port]; sc->ports[port].txq = &sc->adapter->sge.ctrlq[port]; sc->ports[port].rxq = &sc->adapter->sge.rxq[pi->vi->first_rxq]; sc->ports[port].rx_channel_id = pi->rx_c_chan; sc->ports[port].tx_channel_id = pi->tx_chan; sc->ports[port].stats_queued = counter_u64_alloc(M_WAITOK); sc->ports[port].stats_completed = counter_u64_alloc(M_WAITOK); _Static_assert(sizeof(sc->port_mask) * NBBY >= MAX_NPORTS - 1, "Too many ports to fit in port_mask"); /* * Completions for crypto requests on port 1 can sometimes * return a stale cookie value due to a firmware bug. Disable * requests on port 1 by default on affected firmware. */ if (sc->adapter->params.fw_vers >= FW_VERSION32(1, 25, 4, 0) || port == 0) sc->port_mask |= 1u << port; } static int ccr_attach(device_t dev) { struct ccr_softc *sc; int32_t cid; int i; sc = device_get_softc(dev); sc->dev = dev; sc->adapter = device_get_softc(device_get_parent(dev)); for_each_port(sc->adapter, i) { ccr_init_port(sc, i); } cid = crypto_get_driverid(dev, sizeof(struct ccr_session), CRYPTOCAP_F_HARDWARE); if (cid < 0) { device_printf(dev, "could not get crypto driver id\n"); return (ENXIO); } sc->cid = cid; sc->adapter->ccr_softc = sc; /* * The FID must be the first RXQ for port 0 regardless of * which port is used to service the request. */ sc->first_rxq_id = sc->adapter->sge.rxq[0].iq.abs_id; mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_hash = counter_u64_alloc(M_WAITOK); sc->stats_hmac = counter_u64_alloc(M_WAITOK); sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK); sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK); sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK); sc->stats_inflight = counter_u64_alloc(M_WAITOK); sc->stats_mac_error = counter_u64_alloc(M_WAITOK); sc->stats_pad_error = counter_u64_alloc(M_WAITOK); sc->stats_sglist_error = counter_u64_alloc(M_WAITOK); sc->stats_process_error = counter_u64_alloc(M_WAITOK); sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK); ccr_sysctls(sc); return (0); } static void ccr_free_port(struct ccr_softc *sc, int port) { counter_u64_free(sc->ports[port].stats_queued); counter_u64_free(sc->ports[port].stats_completed); } static int ccr_detach(device_t dev) { struct ccr_softc *sc; int i; sc = device_get_softc(dev); mtx_lock(&sc->lock); sc->detaching = true; mtx_unlock(&sc->lock); crypto_unregister_all(sc->cid); mtx_destroy(&sc->lock); counter_u64_free(sc->stats_blkcipher_encrypt); counter_u64_free(sc->stats_blkcipher_decrypt); counter_u64_free(sc->stats_hash); counter_u64_free(sc->stats_hmac); counter_u64_free(sc->stats_eta_encrypt); counter_u64_free(sc->stats_eta_decrypt); counter_u64_free(sc->stats_gcm_encrypt); counter_u64_free(sc->stats_gcm_decrypt); counter_u64_free(sc->stats_ccm_encrypt); counter_u64_free(sc->stats_ccm_decrypt); counter_u64_free(sc->stats_wr_nomem); counter_u64_free(sc->stats_inflight); counter_u64_free(sc->stats_mac_error); counter_u64_free(sc->stats_pad_error); counter_u64_free(sc->stats_sglist_error); counter_u64_free(sc->stats_process_error); counter_u64_free(sc->stats_sw_fallback); for_each_port(sc->adapter, i) { ccr_free_port(sc, i); } sglist_free(sc->sg_iv_aad); free(sc->iv_aad_buf, M_CCR); sc->adapter->ccr_softc = NULL; return (0); } static void ccr_init_hash_digest(struct ccr_session *s) { union authctx auth_ctx; - struct auth_hash *axf; + const struct auth_hash *axf; axf = s->hmac.auth_hash; axf->Init(&auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads); } static bool ccr_aes_check_keylen(int alg, int klen) { switch (klen * 8) { case 128: case 192: if (alg == CRYPTO_AES_XTS) return (false); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) return (false); break; default: return (false); } return (true); } static void ccr_aes_setkey(struct ccr_session *s, const void *key, int klen) { unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size; unsigned int opad_present; if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) kbits = (klen / 2) * 8; else kbits = klen * 8; switch (kbits) { case 128: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; break; case 192: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; break; case 256: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; break; default: panic("should not get here"); } s->blkcipher.key_len = klen; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); switch (s->blkcipher.cipher_mode) { case SCMD_CIPH_MODE_AES_CBC: case SCMD_CIPH_MODE_AES_XTS: t4_aes_getdeckey(s->blkcipher.deckey, key, kbits); break; } kctx_len = roundup2(s->blkcipher.key_len, 16); switch (s->mode) { case ETA: mk_size = s->hmac.mk_size; opad_present = 1; iopad_size = roundup2(s->hmac.partial_digest_len, 16); kctx_len += iopad_size * 2; break; case GCM: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; opad_present = 0; kctx_len += GMAC_BLOCK_LEN; break; case CCM: switch (kbits) { case 128: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; break; case 192: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; break; case 256: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; break; default: panic("should not get here"); } opad_present = 0; kctx_len *= 2; break; default: mk_size = CHCR_KEYCTX_NO_KEY; opad_present = 0; break; } kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) | V_KEY_CONTEXT_OPAD_PRESENT(opad_present) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) | V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1)); } static bool ccr_auth_supported(const struct crypto_session_params *csp) { switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: break; default: return (false); } return (true); } static bool ccr_cipher_supported(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (ccr_aes_check_keylen(csp->csp_cipher_alg, csp->csp_cipher_klen)); } static int ccr_cipher_mode(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: return (SCMD_CIPH_MODE_AES_CBC); case CRYPTO_AES_ICM: return (SCMD_CIPH_MODE_AES_CTR); case CRYPTO_AES_NIST_GCM_16: return (SCMD_CIPH_MODE_AES_GCM); case CRYPTO_AES_XTS: return (SCMD_CIPH_MODE_AES_XTS); case CRYPTO_AES_CCM_16: return (SCMD_CIPH_MODE_AES_CCM); default: return (SCMD_CIPH_MODE_NOP); } } static int ccr_probesession(device_t dev, const struct crypto_session_params *csp) { unsigned int cipher_mode; if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_DIGEST: if (!ccr_auth_supported(csp)) return (EINVAL); break; case CSP_MODE_CIPHER: if (!ccr_cipher_supported(csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > AES_GMAC_HASH_LEN) return (EINVAL); break; case CRYPTO_AES_CCM_16: if (csp->csp_ivlen != AES_CCM_IV_LEN) return (EINVAL); if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN) return (EINVAL); break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp)) return (EINVAL); break; default: return (EINVAL); } if (csp->csp_cipher_klen != 0) { cipher_mode = ccr_cipher_mode(csp); if (cipher_mode == SCMD_CIPH_MODE_NOP) return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } /* * Select an available port with the lowest number of active sessions. */ static struct ccr_port * ccr_choose_port(struct ccr_softc *sc) { struct ccr_port *best, *p; int i; mtx_assert(&sc->lock, MA_OWNED); best = NULL; for (i = 0; i < nitems(sc->ports); i++) { p = &sc->ports[i]; /* Ignore non-existent ports. */ if (p->rxq == NULL) continue; /* * XXX: Ignore ports whose queues aren't initialized. * This is racy as the rxq can be destroyed by the * associated VI detaching. Eventually ccr should use * dedicated queues. */ if (p->rxq->iq.adapter == NULL || p->txq->adapter == NULL) continue; if ((sc->port_mask & (1u << i)) == 0) continue; if (best == NULL || p->active_sessions < best->active_sessions) best = p; } return (best); } static void ccr_delete_session(struct ccr_session *s) { sglist_free(s->sg_input); sglist_free(s->sg_output); sglist_free(s->sg_ulptx); sglist_free(s->sg_dsgl); mtx_destroy(&s->lock); } static int ccr_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ccr_softc *sc; struct ccr_session *s; - struct auth_hash *auth_hash; + const struct auth_hash *auth_hash; unsigned int auth_mode, cipher_mode, mk_size; unsigned int partial_digest_len; switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: auth_hash = &auth_hash_hmac_sha1; auth_mode = SCMD_AUTH_MODE_SHA1; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; partial_digest_len = SHA1_HASH_LEN; break; case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: auth_hash = &auth_hash_hmac_sha2_224; auth_mode = SCMD_AUTH_MODE_SHA224; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: auth_hash = &auth_hash_hmac_sha2_256; auth_mode = SCMD_AUTH_MODE_SHA256; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: auth_hash = &auth_hash_hmac_sha2_384; auth_mode = SCMD_AUTH_MODE_SHA512_384; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: auth_hash = &auth_hash_hmac_sha2_512; auth_mode = SCMD_AUTH_MODE_SHA512_512; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; default: auth_hash = NULL; auth_mode = SCMD_AUTH_MODE_NOP; mk_size = 0; partial_digest_len = 0; break; } cipher_mode = ccr_cipher_mode(csp); #ifdef INVARIANTS switch (csp->csp_mode) { case CSP_MODE_CIPHER: if (cipher_mode == SCMD_CIPH_MODE_NOP || cipher_mode == SCMD_CIPH_MODE_AES_GCM || cipher_mode == SCMD_CIPH_MODE_AES_CCM) panic("invalid cipher algo"); break; case CSP_MODE_DIGEST: if (auth_mode == SCMD_AUTH_MODE_NOP) panic("invalid auth algo"); break; case CSP_MODE_AEAD: if (cipher_mode != SCMD_CIPH_MODE_AES_GCM && cipher_mode != SCMD_CIPH_MODE_AES_CCM) panic("invalid aead cipher algo"); if (auth_mode != SCMD_AUTH_MODE_NOP) panic("invalid aead auth aglo"); break; case CSP_MODE_ETA: if (cipher_mode == SCMD_CIPH_MODE_NOP || cipher_mode == SCMD_CIPH_MODE_AES_GCM || cipher_mode == SCMD_CIPH_MODE_AES_CCM) panic("invalid cipher algo"); if (auth_mode == SCMD_AUTH_MODE_NOP) panic("invalid auth algo"); break; default: panic("invalid csp mode"); } #endif s = crypto_get_driver_session(cses); mtx_init(&s->lock, "ccr session", NULL, MTX_DEF); s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT); if (s->sg_input == NULL || s->sg_output == NULL || s->sg_ulptx == NULL || s->sg_dsgl == NULL) { ccr_delete_session(s); return (ENOMEM); } sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); ccr_delete_session(s); return (ENXIO); } s->port = ccr_choose_port(sc); if (s->port == NULL) { mtx_unlock(&sc->lock); ccr_delete_session(s); return (ENXIO); } switch (csp->csp_mode) { case CSP_MODE_AEAD: if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) s->mode = CCM; else s->mode = GCM; break; case CSP_MODE_ETA: s->mode = ETA; break; case CSP_MODE_DIGEST: if (csp->csp_auth_klen != 0) s->mode = HMAC; else s->mode = HASH; break; case CSP_MODE_CIPHER: s->mode = BLKCIPHER; break; } if (s->mode == GCM) { if (csp->csp_auth_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = csp->csp_auth_mlen; t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen, s->gmac.ghash_h); } else if (s->mode == CCM) { if (csp->csp_auth_mlen == 0) s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN; else s->ccm_mac.hash_len = csp->csp_auth_mlen; } else if (auth_mode != SCMD_AUTH_MODE_NOP) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; s->hmac.mk_size = mk_size; s->hmac.partial_digest_len = partial_digest_len; if (csp->csp_auth_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else s->hmac.hash_len = csp->csp_auth_mlen; if (csp->csp_auth_key != NULL) t4_init_hmac_digest(auth_hash, partial_digest_len, csp->csp_auth_key, csp->csp_auth_klen, s->hmac.pads); else ccr_init_hash_digest(s); } if (cipher_mode != SCMD_CIPH_MODE_NOP) { s->blkcipher.cipher_mode = cipher_mode; s->blkcipher.iv_len = csp->csp_ivlen; if (csp->csp_cipher_key != NULL) ccr_aes_setkey(s, csp->csp_cipher_key, csp->csp_cipher_klen); } s->port->active_sessions++; mtx_unlock(&sc->lock); return (0); } static void ccr_freesession(device_t dev, crypto_session_t cses) { struct ccr_softc *sc; struct ccr_session *s; sc = device_get_softc(dev); s = crypto_get_driver_session(cses); #ifdef INVARIANTS if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); #endif mtx_lock(&sc->lock); s->port->active_sessions--; mtx_unlock(&sc->lock); ccr_delete_session(s); } static int ccr_process(device_t dev, struct cryptop *crp, int hint) { const struct crypto_session_params *csp; struct ccr_softc *sc; struct ccr_session *s; int error; csp = crypto_get_params(crp->crp_session); s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); mtx_lock(&s->lock); error = ccr_populate_sglist(s->sg_input, &crp->crp_buf); if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf); if (error) { counter_u64_add(sc->stats_sglist_error, 1); goto out; } switch (s->mode) { case HASH: error = ccr_hash(sc, s, crp); if (error == 0) counter_u64_add(sc->stats_hash, 1); break; case HMAC: if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, s->hmac.partial_digest_len, crp->crp_auth_key, csp->csp_auth_klen, s->hmac.pads); error = ccr_hash(sc, s, crp); if (error == 0) counter_u64_add(sc->stats_hmac, 1); break; case BLKCIPHER: if (crp->crp_cipher_key != NULL) ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); error = ccr_blkcipher(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_blkcipher_encrypt, 1); else counter_u64_add(sc->stats_blkcipher_decrypt, 1); } break; case ETA: if (crp->crp_auth_key != NULL) t4_init_hmac_digest(s->hmac.auth_hash, s->hmac.partial_digest_len, crp->crp_auth_key, csp->csp_auth_klen, s->hmac.pads); if (crp->crp_cipher_key != NULL) ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); error = ccr_eta(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_eta_encrypt, 1); else counter_u64_add(sc->stats_eta_decrypt, 1); } break; case GCM: if (crp->crp_cipher_key != NULL) { t4_init_gmac_hash(crp->crp_cipher_key, csp->csp_cipher_klen, s->gmac.ghash_h); ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); } if (crp->crp_payload_length == 0) { mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } error = ccr_gcm(sc, s, crp); if (error == EMSGSIZE || error == EFBIG) { counter_u64_add(sc->stats_sw_fallback, 1); mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_gcm_encrypt, 1); else counter_u64_add(sc->stats_gcm_decrypt, 1); } break; case CCM: if (crp->crp_cipher_key != NULL) { ccr_aes_setkey(s, crp->crp_cipher_key, csp->csp_cipher_klen); } error = ccr_ccm(sc, s, crp); if (error == EMSGSIZE || error == EFBIG) { counter_u64_add(sc->stats_sw_fallback, 1); mtx_unlock(&s->lock); ccr_ccm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) counter_u64_add(sc->stats_ccm_encrypt, 1); else counter_u64_add(sc->stats_ccm_decrypt, 1); } break; } if (error == 0) { #ifdef INVARIANTS s->pending++; #endif counter_u64_add(sc->stats_inflight, 1); counter_u64_add(s->port->stats_queued, 1); } else counter_u64_add(sc->stats_process_error, 1); out: mtx_unlock(&s->lock); if (error) { crp->crp_etype = error; crypto_done(crp); } return (0); } static int do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct ccr_softc *sc = iq->adapter->ccr_softc; struct ccr_session *s; const struct cpl_fw6_pld *cpl; struct cryptop *crp; uint32_t status; int error; if (m != NULL) cpl = mtod(m, const void *); else cpl = (const void *)(rss + 1); crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]); s = crypto_get_driver_session(crp->crp_session); status = be64toh(cpl->data[0]); if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status)) error = EBADMSG; else error = 0; #ifdef INVARIANTS mtx_lock(&s->lock); s->pending--; mtx_unlock(&s->lock); #endif counter_u64_add(sc->stats_inflight, -1); counter_u64_add(s->port->stats_completed, 1); switch (s->mode) { case HASH: case HMAC: error = ccr_hash_done(sc, s, crp, cpl, error); break; case BLKCIPHER: error = ccr_blkcipher_done(sc, s, crp, cpl, error); break; case ETA: error = ccr_eta_done(sc, s, crp, cpl, error); break; case GCM: error = ccr_gcm_done(sc, s, crp, cpl, error); break; case CCM: error = ccr_ccm_done(sc, s, crp, cpl, error); break; } if (error == EBADMSG) { if (CHK_MAC_ERR_BIT(status)) counter_u64_add(sc->stats_mac_error, 1); if (CHK_PAD_ERR_BIT(status)) counter_u64_add(sc->stats_pad_error, 1); } crp->crp_etype = error; crypto_done(crp); m_freem(m); return (0); } static int ccr_modevent(module_t mod, int cmd, void *arg) { switch (cmd) { case MOD_LOAD: t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld); return (0); case MOD_UNLOAD: t4_register_cpl_handler(CPL_FW6_PLD, NULL); return (0); default: return (EOPNOTSUPP); } } static device_method_t ccr_methods[] = { DEVMETHOD(device_identify, ccr_identify), DEVMETHOD(device_probe, ccr_probe), DEVMETHOD(device_attach, ccr_attach), DEVMETHOD(device_detach, ccr_detach), DEVMETHOD(cryptodev_probesession, ccr_probesession), DEVMETHOD(cryptodev_newsession, ccr_newsession), DEVMETHOD(cryptodev_freesession, ccr_freesession), DEVMETHOD(cryptodev_process, ccr_process), DEVMETHOD_END }; static driver_t ccr_driver = { "ccr", ccr_methods, sizeof(struct ccr_softc) }; static devclass_t ccr_devclass; DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL); MODULE_VERSION(ccr, 1); MODULE_DEPEND(ccr, crypto, 1, 1, 1); MODULE_DEPEND(ccr, t6nex, 1, 1, 1); diff --git a/sys/dev/cxgbe/crypto/t4_keyctx.c b/sys/dev/cxgbe/crypto/t4_keyctx.c index b64eb4ff23d7..136aba759a08 100644 --- a/sys/dev/cxgbe/crypto/t4_keyctx.c +++ b/sys/dev/cxgbe/crypto/t4_keyctx.c @@ -1,680 +1,680 @@ /*- * Copyright (c) 2017-2019 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_kern_tls.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "common/common.h" #include "crypto/t4_crypto.h" /* * Crypto operations use a key context to store cipher keys and * partial hash digests. They can either be passed inline as part of * a work request using crypto or they can be stored in card RAM. For * the latter case, work requests must replace the inline key context * with a request to read the context from card RAM. * * The format of a key context: * * +-------------------------------+ * | key context header | * +-------------------------------+ * | AES key | ----- For requests with AES * +-------------------------------+ * | Hash state | ----- For hash-only requests * +-------------------------------+ - * | IPAD (16-byte aligned) | \ * +-------------------------------+ +---- For requests with HMAC * | OPAD (16-byte aligned) | / * +-------------------------------+ - * | GMAC H | ----- For AES-GCM * +-------------------------------+ - */ /* Fields in the key context header. */ #define S_TLS_KEYCTX_TX_WR_DUALCK 12 #define M_TLS_KEYCTX_TX_WR_DUALCK 0x1 #define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK) #define G_TLS_KEYCTX_TX_WR_DUALCK(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK) #define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U) #define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 #define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) #define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) #define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 #define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) #define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ M_TLS_KEYCTX_TX_WR_SALT_PRESENT) #define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 #define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf #define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) #define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ M_TLS_KEYCTX_TX_WR_TXCK_SIZE) #define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 #define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf #define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) #define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ M_TLS_KEYCTX_TX_WR_TXMK_SIZE) #define S_TLS_KEYCTX_TX_WR_TXVALID 0 #define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 #define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) #define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) #define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) #define S_TLS_KEYCTX_TX_WR_FLITCNT 3 #define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f #define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT) #define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT) #define S_TLS_KEYCTX_TX_WR_HMACCTRL 0 #define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7 #define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL) #define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL) #define S_TLS_KEYCTX_TX_WR_PROTOVER 4 #define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf #define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \ ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER) #define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER) #define S_TLS_KEYCTX_TX_WR_CIPHMODE 0 #define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf #define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE) #define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE) #define S_TLS_KEYCTX_TX_WR_AUTHMODE 4 #define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf #define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE) #define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE) #define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3 #define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1 #define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) #define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \ M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) #define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \ V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U) #define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1 #define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3 #define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) #define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \ M_TLS_KEYCTX_TX_WR_SEQNUMCTRL) #define S_TLS_KEYCTX_TX_WR_RXVALID 0 #define M_TLS_KEYCTX_TX_WR_RXVALID 0x1 #define V_TLS_KEYCTX_TX_WR_RXVALID(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXVALID) #define G_TLS_KEYCTX_TX_WR_RXVALID(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID) #define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U) #define S_TLS_KEYCTX_TX_WR_IVPRESENT 7 #define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT) #define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \ M_TLS_KEYCTX_TX_WR_IVPRESENT) #define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U) #define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6 #define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) #define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \ M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) #define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \ V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3 #define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7 #define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE) #define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \ M_TLS_KEYCTX_TX_WR_RXCK_SIZE) #define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0 #define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7 #define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE) #define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \ M_TLS_KEYCTX_TX_WR_RXMK_SIZE) #define S_TLS_KEYCTX_TX_WR_IVINSERT 55 #define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL #define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT) #define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT) #define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47 #define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL #define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST) #define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \ M_TLS_KEYCTX_TX_WR_AADSTRTOFST) #define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39 #define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL #define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST) #define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_AADSTOPOFST) #define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30 #define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL #define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) #define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \ M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) #define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23 #define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f #define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) #define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) #define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14 #define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff #define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) #define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \ M_TLS_KEYCTX_TX_WR_AUTHSRTOFST) #define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7 #define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f #define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) #define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) #define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0 #define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f #define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT) #define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \ M_TLS_KEYCTX_TX_WR_AUTHINSRT) /* Key Context Programming Operation type */ #define KEY_WRITE_RX 0x1 #define KEY_WRITE_TX 0x2 #define KEY_DELETE_RX 0x4 #define KEY_DELETE_TX 0x8 #define S_KEY_CLR_LOC 4 #define M_KEY_CLR_LOC 0xf #define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) #define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) #define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) #define S_KEY_GET_LOC 0 #define M_KEY_GET_LOC 0xf #define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) #define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) /* * Generate the initial GMAC hash state for a AES-GCM key. * * Borrowed from AES_GMAC_Setkey(). */ void t4_init_gmac_hash(const char *key, int klen, char *ghash) { static char zeroes[GMAC_BLOCK_LEN]; uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; int rounds; rounds = rijndaelKeySetupEnc(keysched, key, klen * 8); rijndaelEncrypt(keysched, rounds, zeroes, ghash); explicit_bzero(keysched, sizeof(keysched)); } /* Copy out the partial hash state from a software hash implementation. */ void t4_copy_partial_hash(int alg, union authctx *auth_ctx, void *dst) { uint32_t *u32; uint64_t *u64; u_int i; u32 = (uint32_t *)dst; u64 = (uint64_t *)dst; switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: for (i = 0; i < SHA1_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); break; case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha224ctx.state[i]); break; case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); break; case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); break; case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) u64[i] = htobe64(auth_ctx->sha512ctx.state[i]); break; } } void -t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len, +t4_init_hmac_digest(const struct auth_hash *axf, u_int partial_digest_len, const char *key, int klen, char *dst) { union authctx auth_ctx; hmac_init_ipad(axf, key, klen, &auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, dst); dst += roundup2(partial_digest_len, 16); hmac_init_opad(axf, key, klen, &auth_ctx); t4_copy_partial_hash(axf->type, &auth_ctx, dst); explicit_bzero(&auth_ctx, sizeof(auth_ctx)); } /* * Borrowed from cesa_prep_aes_key(). * * NB: The crypto engine wants the words in the decryption key in reverse * order. */ void t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits) { uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; uint32_t *dkey; int i; rijndaelKeySetupEnc(ek, enc_key, kbits); dkey = dec_key; dkey += (kbits / 8) / 4; switch (kbits) { case 128: for (i = 0; i < 4; i++) *--dkey = htobe32(ek[4 * 10 + i]); break; case 192: for (i = 0; i < 2; i++) *--dkey = htobe32(ek[4 * 11 + 2 + i]); for (i = 0; i < 4; i++) *--dkey = htobe32(ek[4 * 12 + i]); break; case 256: for (i = 0; i < 4; i++) *--dkey = htobe32(ek[4 * 13 + i]); for (i = 0; i < 4; i++) *--dkey = htobe32(ek[4 * 14 + i]); break; } MPASS(dkey == dec_key); explicit_bzero(ek, sizeof(ek)); } #ifdef KERN_TLS /* * - keyid management * - request to program key? */ u_int t4_tls_key_info_size(const struct ktls_session *tls) { u_int key_info_size, mac_key_size; key_info_size = sizeof(struct tx_keyctx_hdr) + tls->params.cipher_key_len; if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { key_info_size += GMAC_BLOCK_LEN; } else { switch (tls->params.auth_algorithm) { case CRYPTO_SHA1_HMAC: mac_key_size = SHA1_HASH_LEN; break; case CRYPTO_SHA2_256_HMAC: mac_key_size = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384_HMAC: mac_key_size = SHA2_512_HASH_LEN; break; default: __assert_unreachable(); } key_info_size += roundup2(mac_key_size, 16) * 2; } return (key_info_size); } int t4_tls_proto_ver(const struct ktls_session *tls) { if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) return (SCMD_PROTO_VERSION_TLS_1_1); else return (SCMD_PROTO_VERSION_TLS_1_2); } int t4_tls_cipher_mode(const struct ktls_session *tls) { switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: return (SCMD_CIPH_MODE_AES_CBC); case CRYPTO_AES_NIST_GCM_16: return (SCMD_CIPH_MODE_AES_GCM); default: return (SCMD_CIPH_MODE_NOP); } } int t4_tls_auth_mode(const struct ktls_session *tls) { switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: switch (tls->params.auth_algorithm) { case CRYPTO_SHA1_HMAC: return (SCMD_AUTH_MODE_SHA1); case CRYPTO_SHA2_256_HMAC: return (SCMD_AUTH_MODE_SHA256); case CRYPTO_SHA2_384_HMAC: return (SCMD_AUTH_MODE_SHA512_384); default: return (SCMD_AUTH_MODE_NOP); } case CRYPTO_AES_NIST_GCM_16: return (SCMD_AUTH_MODE_GHASH); default: return (SCMD_AUTH_MODE_NOP); } } int t4_tls_hmac_ctrl(const struct ktls_session *tls) { switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: return (SCMD_HMAC_CTRL_NO_TRUNC); case CRYPTO_AES_NIST_GCM_16: return (SCMD_HMAC_CTRL_NOP); default: return (SCMD_HMAC_CTRL_NOP); } } static int tls_cipher_key_size(const struct ktls_session *tls) { switch (tls->params.cipher_key_len) { case 128 / 8: return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128); case 192 / 8: return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192); case 256 / 8: return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256); default: __assert_unreachable(); } } static int tls_mac_key_size(const struct ktls_session *tls) { if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) return (CHCR_KEYCTX_MAC_KEY_SIZE_512); else { switch (tls->params.auth_algorithm) { case CRYPTO_SHA1_HMAC: return (CHCR_KEYCTX_MAC_KEY_SIZE_160); case CRYPTO_SHA2_256_HMAC: return (CHCR_KEYCTX_MAC_KEY_SIZE_256); case CRYPTO_SHA2_384_HMAC: return (CHCR_KEYCTX_MAC_KEY_SIZE_512); default: __assert_unreachable(); } } } void t4_tls_key_ctx(const struct ktls_session *tls, int direction, struct tls_keyctx *kctx) { - struct auth_hash *axf; + const struct auth_hash *axf; u_int mac_key_size; char *hash; /* Key context header. */ if (direction == KTLS_TX) { kctx->u.txhdr.ctxlen = t4_tls_key_info_size(tls) / 16; kctx->u.txhdr.dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) | V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) | V_TLS_KEYCTX_TX_WR_TXVALID(1); if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) kctx->u.txhdr.dualck_to_txvalid |= V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); kctx->u.txhdr.dualck_to_txvalid = htobe16(kctx->u.txhdr.dualck_to_txvalid); } else { kctx->u.rxhdr.flitcnt_hmacctrl = V_TLS_KEYCTX_TX_WR_FLITCNT(t4_tls_key_info_size(tls) / 16) | V_TLS_KEYCTX_TX_WR_HMACCTRL(t4_tls_hmac_ctrl(tls)); kctx->u.rxhdr.protover_ciphmode = V_TLS_KEYCTX_TX_WR_PROTOVER(t4_tls_proto_ver(tls)) | V_TLS_KEYCTX_TX_WR_CIPHMODE(t4_tls_cipher_mode(tls)); kctx->u.rxhdr.authmode_to_rxvalid = V_TLS_KEYCTX_TX_WR_AUTHMODE(t4_tls_auth_mode(tls)) | V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | V_TLS_KEYCTX_TX_WR_RXVALID(1); kctx->u.rxhdr.ivpresent_to_rxmk_size = V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) | V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls)); if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { kctx->u.rxhdr.ivinsert_to_authinsrt = htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); } else { kctx->u.rxhdr.authmode_to_rxvalid |= V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1); kctx->u.rxhdr.ivpresent_to_rxmk_size |= V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1); kctx->u.rxhdr.ivinsert_to_authinsrt = htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); } } /* Key. */ if (direction == KTLS_RX && tls->params.cipher_algorithm == CRYPTO_AES_CBC) t4_aes_getdeckey(kctx->keys.edkey, tls->params.cipher_key, tls->params.cipher_key_len * 8); else memcpy(kctx->keys.edkey, tls->params.cipher_key, tls->params.cipher_key_len); /* Auth state and implicit IV (salt). */ hash = kctx->keys.edkey + tls->params.cipher_key_len; if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { _Static_assert(offsetof(struct tx_keyctx_hdr, txsalt) == offsetof(struct rx_keyctx_hdr, rxsalt), "salt offset mismatch"); memcpy(kctx->u.txhdr.txsalt, tls->params.iv, SALT_SIZE); t4_init_gmac_hash(tls->params.cipher_key, tls->params.cipher_key_len, hash); } else { switch (tls->params.auth_algorithm) { case CRYPTO_SHA1_HMAC: axf = &auth_hash_hmac_sha1; mac_key_size = SHA1_HASH_LEN; break; case CRYPTO_SHA2_256_HMAC: axf = &auth_hash_hmac_sha2_256; mac_key_size = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384_HMAC: axf = &auth_hash_hmac_sha2_384; mac_key_size = SHA2_512_HASH_LEN; break; default: __assert_unreachable(); } t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, tls->params.auth_key_len, hash); } } int t4_alloc_tls_keyid(struct adapter *sc) { vmem_addr_t addr; if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, &addr) != 0) return (-1); return (addr); } void t4_free_tls_keyid(struct adapter *sc, int keyid) { vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); } void t4_write_tlskey_wr(const struct ktls_session *tls, int direction, int tid, int flags, int keyid, struct tls_key_req *kwr) { kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_ATOMIC | flags); kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(TLS_KEY_WR_SZ, 16)) | V_FW_WR_FLOWID(tid)); kwr->protocol = t4_tls_proto_ver(tls); kwr->mfs = htobe16(tls->params.max_frame_len); kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ? KEY_WRITE_TX : KEY_WRITE_RX); /* master command */ kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(TLS_KEY_CONTEXT_SZ >> 5)); kwr->len16 = htobe32((tid << 8) | DIV_ROUND_UP(TLS_KEY_WR_SZ - sizeof(struct work_request_hdr), 16)); kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); /* sub command */ kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); kwr->sc_len = htobe32(TLS_KEY_CONTEXT_SZ); } #endif diff --git a/sys/dev/glxsb/glxsb.h b/sys/dev/glxsb/glxsb.h index 27e5bb44709c..50b30d718618 100644 --- a/sys/dev/glxsb/glxsb.h +++ b/sys/dev/glxsb/glxsb.h @@ -1,54 +1,54 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2006 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GLXSB_H_ #define _GLXSB_H_ #include #define SB_AES_BLOCK_SIZE 0x0010 struct glxsb_session { uint32_t ses_key[4]; /* key */ - struct auth_hash *ses_axf; + const struct auth_hash *ses_axf; uint8_t *ses_ictx; uint8_t *ses_octx; int ses_mlen; }; int glxsb_hash_setup(struct glxsb_session *ses, const struct crypto_session_params *csp); int glxsb_hash_process(struct glxsb_session *ses, const struct crypto_session_params *csp, struct cryptop *crp); void glxsb_hash_free(struct glxsb_session *ses); #endif /* !_GLXSB_H_ */ diff --git a/sys/dev/glxsb/glxsb_hash.c b/sys/dev/glxsb/glxsb_hash.c index b9ceb27deb4d..320ffd66a81e 100644 --- a/sys/dev/glxsb/glxsb_hash.c +++ b/sys/dev/glxsb/glxsb_hash.c @@ -1,160 +1,160 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include "glxsb.h" /* * Implementation notes. * * The Geode LX Security Block provides AES-128-CBC acceleration. * We implement all HMAC algorithms provided by crypto(9) framework so glxsb can work * with ipsec(4) * * This code was stolen from crypto/via/padlock_hash.c */ MALLOC_DECLARE(M_GLXSB); static void glxsb_hash_key_setup(struct glxsb_session *ses, const char *key, int klen) { - struct auth_hash *axf; + const struct auth_hash *axf; axf = ses->ses_axf; hmac_init_ipad(axf, key, klen, ses->ses_ictx); hmac_init_opad(axf, key, klen, ses->ses_octx); } /* * Compute keyed-hash authenticator. */ static int glxsb_authcompute(struct glxsb_session *ses, struct cryptop *crp) { u_char hash[HASH_MAX_LEN]; - struct auth_hash *axf; + const struct auth_hash *axf; union authctx ctx; int error; axf = ses->ses_axf; bcopy(ses->ses_ictx, &ctx, axf->ctxsize); error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (error != 0) return (error); error = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); if (error != 0) return (error); axf->Final(hash, &ctx); bcopy(ses->ses_octx, &ctx, axf->ctxsize); axf->Update(&ctx, hash, axf->hashsize); axf->Final(hash, &ctx); explicit_bzero(&ctx, sizeof(ctx)); /* Verify or inject the authentication data */ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char hash2[HASH_MAX_LEN]; crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen, hash2); if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0) error = EBADMSG; explicit_bzero(hash2, sizeof(hash2)); } else crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen, hash); explicit_bzero(hash, sizeof(hash)); return (error); } int glxsb_hash_setup(struct glxsb_session *ses, const struct crypto_session_params *csp) { ses->ses_axf = crypto_auth_hash(csp); if (csp->csp_auth_mlen == 0) ses->ses_mlen = ses->ses_axf->hashsize; else ses->ses_mlen = csp->csp_auth_mlen; /* Allocate memory for HMAC inner and outer contexts. */ ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_GLXSB, M_ZERO | M_NOWAIT); ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_GLXSB, M_ZERO | M_NOWAIT); if (ses->ses_ictx == NULL || ses->ses_octx == NULL) return (ENOMEM); /* Setup key if given. */ if (csp->csp_auth_key != NULL) { glxsb_hash_key_setup(ses, csp->csp_auth_key, csp->csp_auth_klen); } return (0); } int glxsb_hash_process(struct glxsb_session *ses, const struct crypto_session_params *csp, struct cryptop *crp) { int error; if (crp->crp_auth_key != NULL) glxsb_hash_key_setup(ses, crp->crp_auth_key, csp->csp_auth_klen); error = glxsb_authcompute(ses, crp); return (error); } void glxsb_hash_free(struct glxsb_session *ses) { if (ses->ses_ictx != NULL) { zfree(ses->ses_ictx, M_GLXSB); ses->ses_ictx = NULL; } if (ses->ses_octx != NULL) { zfree(ses->ses_octx, M_GLXSB); ses->ses_octx = NULL; } } diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c index 71300dcb0393..3940361561d0 100644 --- a/sys/dev/safexcel/safexcel.c +++ b/sys/dev/safexcel/safexcel.c @@ -1,2627 +1,2627 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020, 2021 Rubicon Communications, LLC (Netgate) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "safexcel_reg.h" #include "safexcel_var.h" /* * We only support the EIP97 for now. */ static struct ofw_compat_data safexcel_compat[] = { { "inside-secure,safexcel-eip97ies", (uintptr_t)97 }, { "inside-secure,safexcel-eip97", (uintptr_t)97 }, { NULL, 0 } }; const struct safexcel_reg_offsets eip97_regs_offset = { .hia_aic = SAFEXCEL_EIP97_HIA_AIC_BASE, .hia_aic_g = SAFEXCEL_EIP97_HIA_AIC_G_BASE, .hia_aic_r = SAFEXCEL_EIP97_HIA_AIC_R_BASE, .hia_aic_xdr = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE, .hia_dfe = SAFEXCEL_EIP97_HIA_DFE_BASE, .hia_dfe_thr = SAFEXCEL_EIP97_HIA_DFE_THR_BASE, .hia_dse = SAFEXCEL_EIP97_HIA_DSE_BASE, .hia_dse_thr = SAFEXCEL_EIP97_HIA_DSE_THR_BASE, .hia_gen_cfg = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE, .pe = SAFEXCEL_EIP97_PE_BASE, }; const struct safexcel_reg_offsets eip197_regs_offset = { .hia_aic = SAFEXCEL_EIP197_HIA_AIC_BASE, .hia_aic_g = SAFEXCEL_EIP197_HIA_AIC_G_BASE, .hia_aic_r = SAFEXCEL_EIP197_HIA_AIC_R_BASE, .hia_aic_xdr = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE, .hia_dfe = SAFEXCEL_EIP197_HIA_DFE_BASE, .hia_dfe_thr = SAFEXCEL_EIP197_HIA_DFE_THR_BASE, .hia_dse = SAFEXCEL_EIP197_HIA_DSE_BASE, .hia_dse_thr = SAFEXCEL_EIP197_HIA_DSE_THR_BASE, .hia_gen_cfg = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE, .pe = SAFEXCEL_EIP197_PE_BASE, }; static struct safexcel_request * safexcel_next_request(struct safexcel_ring *ring) { int i; i = ring->cdr.read; KASSERT(i >= 0 && i < SAFEXCEL_RING_SIZE, ("%s: out of bounds request index %d", __func__, i)); return (&ring->requests[i]); } static struct safexcel_cmd_descr * safexcel_cmd_descr_next(struct safexcel_cmd_descr_ring *ring) { struct safexcel_cmd_descr *cdesc; if (ring->write == ring->read) return (NULL); cdesc = &ring->desc[ring->read]; ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; return (cdesc); } static struct safexcel_res_descr * safexcel_res_descr_next(struct safexcel_res_descr_ring *ring) { struct safexcel_res_descr *rdesc; if (ring->write == ring->read) return (NULL); rdesc = &ring->desc[ring->read]; ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; return (rdesc); } static struct safexcel_request * safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring) { int i; mtx_assert(&ring->mtx, MA_OWNED); i = ring->cdr.write; if ((i + 1) % SAFEXCEL_RING_SIZE == ring->cdr.read) return (NULL); return (&ring->requests[i]); } static void safexcel_free_request(struct safexcel_ring *ring, struct safexcel_request *req) { struct safexcel_context_record *ctx; mtx_assert(&ring->mtx, MA_OWNED); if (req->dmap_loaded) { bus_dmamap_unload(ring->data_dtag, req->dmap); req->dmap_loaded = false; } ctx = (struct safexcel_context_record *)req->ctx.vaddr; explicit_bzero(ctx->data, sizeof(ctx->data)); explicit_bzero(req->iv, sizeof(req->iv)); } static void safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) { TAILQ_HEAD(, cryptop) cq; struct cryptop *crp, *tmp; struct safexcel_cmd_descr *cdesc; struct safexcel_res_descr *rdesc; struct safexcel_request *req; struct safexcel_ring *ring; uint32_t blocked, error, i, ncdescs, nrdescs, nreqs; blocked = 0; ring = &sc->sc_ring[ringidx]; nreqs = SAFEXCEL_READ(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT); nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET; nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK; if (nreqs == 0) { SAFEXCEL_DPRINTF(sc, 1, "zero pending requests on ring %d\n", ringidx); mtx_lock(&ring->mtx); goto out; } TAILQ_INIT(&cq); ring = &sc->sc_ring[ringidx]; bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); ncdescs = nrdescs = 0; for (i = 0; i < nreqs; i++) { req = safexcel_next_request(ring); bus_dmamap_sync(req->ctx.tag, req->ctx.map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(ring->data_dtag, req->dmap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); ncdescs += req->cdescs; while (req->cdescs-- > 0) { cdesc = safexcel_cmd_descr_next(&ring->cdr); KASSERT(cdesc != NULL, ("%s: missing control descriptor", __func__)); if (req->cdescs == 0) KASSERT(cdesc->last_seg, ("%s: chain is not terminated", __func__)); } nrdescs += req->rdescs; while (req->rdescs-- > 0) { rdesc = safexcel_res_descr_next(&ring->rdr); error = rdesc->result_data.error_code; if (error != 0) { if (error == SAFEXCEL_RESULT_ERR_AUTH_FAILED && req->crp->crp_etype == 0) { req->crp->crp_etype = EBADMSG; } else { SAFEXCEL_DPRINTF(sc, 1, "error code %#x\n", error); req->crp->crp_etype = EIO; } } } TAILQ_INSERT_TAIL(&cq, req->crp, crp_next); } mtx_lock(&ring->mtx); if (nreqs != 0) { KASSERT(ring->queued >= nreqs, ("%s: request count underflow, %d queued %d completed", __func__, ring->queued, nreqs)); ring->queued -= nreqs; SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); blocked = ring->blocked; ring->blocked = 0; } out: if (ring->queued != 0) { SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | imin(ring->queued, 16)); } mtx_unlock(&ring->mtx); if (blocked) crypto_unblock(sc->sc_cid, blocked); TAILQ_FOREACH_SAFE(crp, &cq, crp_next, tmp) crypto_done(crp); } static void safexcel_ring_intr(void *arg) { struct safexcel_softc *sc; struct safexcel_intr_handle *ih; uint32_t status, stat; int ring; bool rdrpending; ih = arg; sc = ih->sc; ring = ih->ring; status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring)); /* CDR interrupts */ if (status & SAFEXCEL_CDR_IRQ(ring)) { stat = SAFEXCEL_READ(sc, SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, stat & SAFEXCEL_CDR_INTR_MASK); } /* RDR interrupts */ rdrpending = false; if (status & SAFEXCEL_RDR_IRQ(ring)) { stat = SAFEXCEL_READ(sc, SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); if ((stat & SAFEXCEL_xDR_ERR) == 0) rdrpending = true; SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, stat & SAFEXCEL_RDR_INTR_MASK); } SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring), status); if (rdrpending) safexcel_rdr_intr(sc, ring); } static int safexcel_configure(struct safexcel_softc *sc) { uint32_t i, mask, pemask, reg; device_t dev; if (sc->sc_type == 197) { sc->sc_offsets = eip197_regs_offset; pemask = SAFEXCEL_N_PES_MASK; } else { sc->sc_offsets = eip97_regs_offset; pemask = EIP97_N_PES_MASK; } dev = sc->sc_dev; /* Scan for valid ring interrupt controllers. */ for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) { reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_VERSION(i)); if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE) break; } sc->sc_config.aic_rings = i; if (sc->sc_config.aic_rings == 0) return (-1); reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS); /* Check for 64bit addressing. */ if ((reg & SAFEXCEL_OPT_ADDR_64) == 0) return (-1); /* Check alignment constraints (which we do not support). */ if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >> SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0) return (-1); sc->sc_config.hdw = (reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET; mask = (1 << sc->sc_config.hdw) - 1; sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; /* Limit the number of rings to the number of the AIC Rings. */ sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; sc->sc_config.cd_size = sizeof(struct safexcel_cmd_descr) / sizeof(uint32_t); sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; sc->sc_config.rd_size = sizeof(struct safexcel_res_descr) / sizeof(uint32_t); sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; sc->sc_config.atok_offset = (SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) & ~mask; return (0); } static void safexcel_init_hia_bus_access(struct safexcel_softc *sc) { uint32_t version, val; /* Determine endianness and configure byte swap. */ version = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION); val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) { val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL, val); } /* Configure wr/rd cache values. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL, SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS)); } static void safexcel_disable_global_interrupts(struct safexcel_softc *sc) { /* Disable and clear pending interrupts. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, SAFEXCEL_AIC_G_ACK_ALL_MASK); } /* * Configure the data fetch engine. This component parses command descriptors * and sets up DMA transfers from host memory to the corresponding processing * engine. */ static void safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe) { /* Reset all DFE threads. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_RESET_PE); /* Deassert the DFE reset. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0); /* DMA transfer size to use. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe), SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG | SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) | SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) | SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) | SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) | SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS)); /* Configure the PE DMA transfer thresholds. */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe), SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | SAFEXCEL_PE_IN_xBUF_THRES_MAX(9)); SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe), SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | SAFEXCEL_PE_IN_xBUF_THRES_MAX(7)); } /* * Configure the data store engine. This component parses result descriptors * and sets up DMA transfers from the processing engine to host memory. */ static int safexcel_configure_dse(struct safexcel_softc *sc, int pe) { uint32_t val; int count; /* Disable and reset all DSE threads. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_RESET_PE); /* Wait for a second for threads to go idle. */ for (count = 0;;) { val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe)); if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) == SAFEXCEL_DSE_THR_RDR_ID_MASK) break; if (count++ > 10000) { device_printf(sc->sc_dev, "DSE reset timeout\n"); return (-1); } DELAY(100); } /* Exit the reset state. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0); /* DMA transfer size to use */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe), SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG | SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) | SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) | SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) | SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE); /* Configure the procesing engine thresholds */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe), SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) | SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8)); return (0); } static void safexcel_hw_prepare_rings(struct safexcel_softc *sc) { int i; for (i = 0; i < sc->sc_config.rings; i++) { /* * Command descriptors. */ /* Clear interrupts for this ring. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK); /* Disable external triggering. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * sizeof(uint32_t)); /* * Result descriptors. */ /* Disable external triggering. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); /* Ring size. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * sizeof(uint32_t)); } } static void safexcel_hw_setup_rings(struct safexcel_softc *sc) { struct safexcel_ring *ring; uint32_t cd_size_rnd, mask, rd_size_rnd, val; int i; mask = (1 << sc->sc_config.hdw) - 1; cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; val = (sizeof(struct safexcel_res_descr) - sizeof(struct safexcel_res_data)) / sizeof(uint32_t); rd_size_rnd = (val + mask) >> sc->sc_config.hdw; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; /* * Command descriptors. */ /* Ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP | (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | sc->sc_config.cd_size); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << SAFEXCEL_xDR_xD_FETCH_THRESH) | (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); /* Configure DMA tx control. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS)); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_CDR_INTR_MASK); /* * Result descriptors. */ /* Ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, SAFEXCEL_xDR_DESC_MODE_64BIT | (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | sc->sc_config.rd_size); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << SAFEXCEL_xDR_xD_FETCH_THRESH) | (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); /* Configure DMA tx control. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) | SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_RDR_INTR_MASK); /* Enable ring interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i), SAFEXCEL_RDR_IRQ(i)); } } /* Reset the command and result descriptor rings. */ static void safexcel_hw_reset_rings(struct safexcel_softc *sc) { int i; for (i = 0; i < sc->sc_config.rings; i++) { /* * Result descriptor ring operations. */ /* Reset ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_RDR_INTR_MASK); /* Disable ring interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), SAFEXCEL_RDR_IRQ(i)); /* * Command descriptor ring operations. */ /* Reset ring base address. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); /* Clear the pending prepared counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, SAFEXCEL_xDR_PREP_CLR_COUNT); /* Clear the pending processed counter. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, SAFEXCEL_xDR_PROC_CLR_COUNT); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); /* Clear any pending interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, SAFEXCEL_CDR_INTR_MASK); } } static void safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe) { int i, ring_mask; for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { ring_mask <<= 1; ring_mask |= 1; } /* Enable command descriptor rings. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); /* Enable result descriptor rings. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); /* Clear any HIA interrupt. */ SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, SAFEXCEL_AIC_G_ACK_HIA_MASK); } static void safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, struct safexcel_request *req, int hint) { int ringidx, ncdesc, nrdesc; bool busy; mtx_assert(&ring->mtx, MA_OWNED); if ((hint & CRYPTO_HINT_MORE) != 0) { ring->pending++; ring->pending_cdesc += req->cdescs; ring->pending_rdesc += req->rdescs; return; } ringidx = req->ringidx; busy = ring->queued != 0; ncdesc = ring->pending_cdesc + req->cdescs; nrdesc = ring->pending_rdesc + req->rdescs; ring->queued += ring->pending + 1; if (!busy) { SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | ring->queued); } SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, nrdesc * sc->sc_config.rd_offset * sizeof(uint32_t)); SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, ncdesc * sc->sc_config.cd_offset * sizeof(uint32_t)); ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; } static void safexcel_init_rings(struct safexcel_softc *sc) { struct safexcel_cmd_descr *cdesc; struct safexcel_ring *ring; uint64_t atok; int i, j; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; snprintf(ring->lockname, sizeof(ring->lockname), "safexcel_ring%d", i); mtx_init(&ring->mtx, ring->lockname, NULL, MTX_DEF); ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; ring->queued = 0; ring->cdr.read = ring->cdr.write = 0; ring->rdr.read = ring->rdr.write = 0; for (j = 0; j < SAFEXCEL_RING_SIZE; j++) { cdesc = &ring->cdr.desc[j]; atok = ring->dma_atok.paddr + sc->sc_config.atok_offset * j; cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); } } } static void safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct safexcel_dma_mem *sdm; if (error != 0) return; KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); sdm = arg; sdm->paddr = segs->ds_addr; } static int safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm, bus_size_t size) { int error; KASSERT(sdm->vaddr == NULL, ("%s: DMA memory descriptor in use.", __func__)); error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ PAGE_SIZE, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ size, 1, /* maxsize, nsegments */ size, BUS_DMA_COHERENT, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sdm->tag); /* dmat */ if (error != 0) { device_printf(sc->sc_dev, "failed to allocate busdma tag, error %d\n", error); goto err1; } error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate DMA safe memory, error %d\n", error); goto err2; } error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "cannot get address of the DMA memory, error %d\n", error); goto err3; } return (0); err3: bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); err2: bus_dma_tag_destroy(sdm->tag); err1: sdm->vaddr = NULL; return (error); } static void safexcel_dma_free_mem(struct safexcel_dma_mem *sdm) { bus_dmamap_unload(sdm->tag, sdm->map); bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); bus_dma_tag_destroy(sdm->tag); } static void safexcel_dma_free_rings(struct safexcel_softc *sc) { struct safexcel_ring *ring; int i; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; safexcel_dma_free_mem(&ring->cdr.dma); safexcel_dma_free_mem(&ring->dma_atok); safexcel_dma_free_mem(&ring->rdr.dma); bus_dma_tag_destroy(ring->data_dtag); mtx_destroy(&ring->mtx); } } static int safexcel_dma_init(struct safexcel_softc *sc) { struct safexcel_ring *ring; bus_size_t size; int error, i; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; error = bus_dma_tag_create( bus_get_dma_tag(sc->sc_dev),/* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ SAFEXCEL_MAX_REQUEST_SIZE, /* maxsize */ SAFEXCEL_MAX_FRAGMENTS, /* nsegments */ SAFEXCEL_MAX_REQUEST_SIZE, /* maxsegsz */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &ring->data_dtag); /* dmat */ if (error != 0) { device_printf(sc->sc_dev, "bus_dma_tag_create main failed; error %d\n", error); return (error); } size = sizeof(uint32_t) * sc->sc_config.cd_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate CDR DMA memory, error %d\n", error); goto err; } ring->cdr.desc = (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; /* Allocate additional CDR token memory. */ size = (bus_size_t)sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate atoken DMA memory, error %d\n", error); goto err; } size = sizeof(uint32_t) * sc->sc_config.rd_offset * SAFEXCEL_RING_SIZE; error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); if (error) { device_printf(sc->sc_dev, "failed to allocate RDR DMA memory, error %d\n", error); goto err; } ring->rdr.desc = (struct safexcel_res_descr *)ring->rdr.dma.vaddr; } return (0); err: safexcel_dma_free_rings(sc); return (error); } static void safexcel_deinit_hw(struct safexcel_softc *sc) { safexcel_hw_reset_rings(sc); safexcel_dma_free_rings(sc); } static int safexcel_init_hw(struct safexcel_softc *sc) { int pe; /* 23.3.7 Initialization */ if (safexcel_configure(sc) != 0) return (EINVAL); if (safexcel_dma_init(sc) != 0) return (ENOMEM); safexcel_init_rings(sc); safexcel_init_hia_bus_access(sc); /* 23.3.7.2 Disable EIP-97 global Interrupts */ safexcel_disable_global_interrupts(sc); for (pe = 0; pe < sc->sc_config.pes; pe++) { /* 23.3.7.3 Configure Data Fetch Engine */ safexcel_configure_dfe_engine(sc, pe); /* 23.3.7.4 Configure Data Store Engine */ if (safexcel_configure_dse(sc, pe)) { safexcel_deinit_hw(sc); return (-1); } /* 23.3.7.5 1. Protocol enables */ SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe), 0xffffffff); SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe), 0xffffffff); } safexcel_hw_prepare_rings(sc); /* 23.3.7.5 Configure the Processing Engine(s). */ for (pe = 0; pe < sc->sc_config.pes; pe++) safexcel_enable_pe_engine(sc, pe); safexcel_hw_setup_rings(sc); return (0); } static int safexcel_setup_dev_interrupts(struct safexcel_softc *sc) { int error, i, j; for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { sc->sc_ih[i].sc = sc; sc->sc_ih[i].ring = i; if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr, &sc->sc_ih[i], &sc->sc_ih[i].handle)) { device_printf(sc->sc_dev, "couldn't setup interrupt %d\n", i); goto err; } error = bus_bind_intr(sc->sc_dev, sc->sc_intr[i], i % mp_ncpus); if (error != 0) device_printf(sc->sc_dev, "failed to bind ring %d\n", error); } return (0); err: for (j = 0; j < i; j++) bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], sc->sc_ih[j].handle); return (ENXIO); } static void safexcel_teardown_dev_interrupts(struct safexcel_softc *sc) { int i; for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], sc->sc_ih[i].handle); } static int safexcel_alloc_dev_resources(struct safexcel_softc *sc) { char name[16]; device_t dev; phandle_t node; int error, i, rid; dev = sc->sc_dev; node = ofw_bus_get_node(dev); rid = 0; sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_res == NULL) { device_printf(dev, "couldn't allocate memory resources\n"); return (ENXIO); } for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) { (void)snprintf(name, sizeof(name), "ring%d", i); error = ofw_bus_find_string_index(node, "interrupt-names", name, &rid); if (error != 0) break; sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_intr[i] == NULL) { error = ENXIO; goto out; } } if (i == 0) { device_printf(dev, "couldn't allocate interrupt resources\n"); error = ENXIO; goto out; } return (0); out: for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), sc->sc_res); return (error); } static void safexcel_free_dev_resources(struct safexcel_softc *sc) { int i; for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) bus_release_resource(sc->sc_dev, SYS_RES_IRQ, rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); if (sc->sc_res != NULL) bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), sc->sc_res); } static int safexcel_probe(device_t dev) { struct safexcel_softc *sc; if (!ofw_bus_status_okay(dev)) return (ENXIO); sc = device_get_softc(dev); sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; if (sc->sc_type == 0) return (ENXIO); device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); return (BUS_PROBE_DEFAULT); } static int safexcel_attach(device_t dev) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; struct safexcel_softc *sc; struct safexcel_request *req; struct safexcel_ring *ring; int i, j, ringidx; sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_cid = -1; if (safexcel_alloc_dev_resources(sc)) goto err; if (safexcel_setup_dev_interrupts(sc)) goto err1; if (safexcel_init_hw(sc)) goto err2; for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { ring = &sc->sc_ring[ringidx]; ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); for (i = 0; i < SAFEXCEL_RING_SIZE; i++) { req = &ring->requests[i]; req->sc = sc; req->ringidx = ringidx; if (bus_dmamap_create(ring->data_dtag, BUS_DMA_COHERENT, &req->dmap) != 0) { for (j = 0; j < i; j++) bus_dmamap_destroy(ring->data_dtag, ring->requests[j].dmap); goto err2; } if (safexcel_dma_alloc_mem(sc, &req->ctx, sizeof(struct safexcel_context_record)) != 0) { for (j = 0; j < i; j++) { bus_dmamap_destroy(ring->data_dtag, ring->requests[j].dmap); safexcel_dma_free_mem( &ring->requests[j].ctx); } goto err2; } } } ctx = device_get_sysctl_ctx(dev); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, "Debug message verbosity"); oid = device_get_sysctl_tree(sc->sc_dev); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); sc->sc_req_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "req_alloc_failures", CTLFLAG_RD, &sc->sc_req_alloc_failures, "Number of request allocation failures"); sc->sc_cdesc_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cdesc_alloc_failures", CTLFLAG_RD, &sc->sc_cdesc_alloc_failures, "Number of command descriptor ring overflows"); sc->sc_rdesc_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "rdesc_alloc_failures", CTLFLAG_RD, &sc->sc_rdesc_alloc_failures, "Number of result descriptor ring overflows"); sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), CRYPTOCAP_F_HARDWARE); if (sc->sc_cid < 0) goto err2; return (0); err2: safexcel_teardown_dev_interrupts(sc); err1: safexcel_free_dev_resources(sc); err: return (ENXIO); } static int safexcel_detach(device_t dev) { struct safexcel_ring *ring; struct safexcel_softc *sc; int i, ringidx; sc = device_get_softc(dev); if (sc->sc_cid >= 0) crypto_unregister_all(sc->sc_cid); counter_u64_free(sc->sc_req_alloc_failures); counter_u64_free(sc->sc_cdesc_alloc_failures); counter_u64_free(sc->sc_rdesc_alloc_failures); for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { ring = &sc->sc_ring[ringidx]; for (i = 0; i < SAFEXCEL_RING_SIZE; i++) { bus_dmamap_destroy(ring->data_dtag, ring->requests[i].dmap); safexcel_dma_free_mem(&ring->requests[i].ctx); } sglist_free(ring->cmd_data); sglist_free(ring->res_data); } safexcel_deinit_hw(sc); safexcel_teardown_dev_interrupts(sc); safexcel_free_dev_resources(sc); return (0); } /* * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted * using the cipher key. */ static void safexcel_setkey_ghash(const uint8_t *key, int klen, uint32_t *hashkey) { uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; uint8_t zeros[AES_BLOCK_LEN]; int i, rounds; memset(zeros, 0, sizeof(zeros)); rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)hashkey); for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) hashkey[i] = htobe32(hashkey[i]); explicit_bzero(ks, sizeof(ks)); } /* * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 * in the hardware implementation. K1 is the cipher key and comes last in the * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC * is not implemented so K2 and K3 are fixed. */ static void safexcel_setkey_xcbcmac(const uint8_t *key, int klen, uint32_t *hashkey) { int i, off; memset(hashkey, 0, 2 * AES_BLOCK_LEN); off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) hashkey[i + off] = htobe32(le32dec(key)); } static void -safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, +safexcel_setkey_hmac_digest(const struct auth_hash *ahash, union authctx *ctx, char *buf) { int hashwords, i; switch (ahash->type) { case CRYPTO_SHA1_HMAC: hashwords = ahash->hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); break; case CRYPTO_SHA2_224_HMAC: hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); break; case CRYPTO_SHA2_256_HMAC: hashwords = ahash->hashsize / sizeof(uint32_t); for (i = 0; i < hashwords; i++) ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); break; case CRYPTO_SHA2_384_HMAC: hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); for (i = 0; i < hashwords; i++) ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); break; case CRYPTO_SHA2_512_HMAC: hashwords = ahash->hashsize / sizeof(uint64_t); for (i = 0; i < hashwords; i++) ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); break; } } /* * Pre-compute the inner and outer digests used in the HMAC algorithm. */ static void safexcel_setkey_hmac(const struct crypto_session_params *csp, const uint8_t *key, int klen, uint8_t *ipad, uint8_t *opad) { union authctx ctx; - struct auth_hash *ahash; + const struct auth_hash *ahash; ahash = crypto_auth_hash(csp); hmac_init_ipad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, ipad); hmac_init_opad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, opad); explicit_bzero(&ctx, ahash->ctxsize); } static void safexcel_setkey_xts(const uint8_t *key, int klen, uint8_t *tweakkey) { memcpy(tweakkey, key + klen, klen); } /* * Populate a context record with paramters from a session. Some consumers * specify per-request keys, in which case the context must be re-initialized * for each request. */ static int safexcel_set_context(struct safexcel_context_record *ctx, int op, const uint8_t *ckey, const uint8_t *akey, struct safexcel_session *sess) { const struct crypto_session_params *csp; uint8_t *data; uint32_t ctrl0, ctrl1; int aklen, alg, cklen, off; csp = crypto_get_params(sess->cses); aklen = csp->csp_auth_klen; cklen = csp->csp_cipher_klen; if (csp->csp_cipher_alg == CRYPTO_AES_XTS) cklen /= 2; ctrl0 = sess->alg | sess->digest | sess->hash; ctrl1 = sess->mode; data = (uint8_t *)ctx->data; if (csp->csp_cipher_alg != 0) { memcpy(data, ckey, cklen); off = cklen; } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { memcpy(data, akey, aklen); off = aklen; } else { off = 0; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: safexcel_setkey_ghash(ckey, cklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_AES_CCM_16: safexcel_setkey_xcbcmac(ckey, cklen, (uint32_t *)(data + off)); off += AES_BLOCK_LEN * 2 + cklen; break; case CRYPTO_AES_XTS: safexcel_setkey_xts(ckey, cklen, data + off); off += cklen; break; } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: safexcel_setkey_ghash(akey, aklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: safexcel_setkey_hmac(csp, akey, aklen, data + off, data + off + sess->statelen); off += sess->statelen * 2; break; } ctrl0 |= SAFEXCEL_CONTROL0_SIZE(off / sizeof(uint32_t)); alg = csp->csp_cipher_alg; if (alg == 0) alg = csp->csp_auth_alg; switch (alg) { case CRYPTO_AES_CCM_16: if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | SAFEXCEL_CONTROL0_KEY_EN; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN | SAFEXCEL_CONTROL0_KEY_EN; } ctrl1 |= SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | SAFEXCEL_CONTROL1_IV2 | SAFEXCEL_CONTROL1_IV3; break; case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN; if (csp->csp_auth_alg != 0) ctrl0 |= SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | SAFEXCEL_CONTROL0_KEY_EN; if (csp->csp_auth_alg != 0) ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; } break; case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_NIST_GMAC: if (CRYPTO_OP_IS_ENCRYPT(op) || csp->csp_auth_alg != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_OUT; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; } if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE | SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | SAFEXCEL_CONTROL1_IV2; } break; case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: ctrl0 |= SAFEXCEL_CONTROL0_RESTART_HASH; /* FALLTHROUGH */ case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_OUT; break; } ctx->control0 = ctrl0; ctx->control1 = ctrl1; return (off); } /* * Construct a no-op instruction, used to pad input tokens. */ static void safexcel_instr_nop(struct safexcel_instr **instrp) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = (1 << 2); instr->status = 0; instr->instructions = 0; *instrp = instr + 1; } /* * Insert the digest of the input payload. This is typically the last * instruction of a sequence. */ static void safexcel_instr_insert_digest(struct safexcel_instr **instrp, int len) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_INSERT_HASH_DIGEST; *instrp = instr + 1; } /* * Retrieve and verify a digest. */ static void safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE; instr->length = len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS; instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | SAFEXCEL_INSTR_STATUS_LAST_PACKET; instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING; *instrp = instr + 1; } static void safexcel_instr_temp_aes_block(struct safexcel_instr **instrp) { struct safexcel_instr *instr; instr = *instrp; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT; instr->length = 0; instr->status = 0; instr->instructions = AES_BLOCK_LEN; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = AES_BLOCK_LEN; instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO; *instrp = instr + 1; } /* * Handle a request for an unauthenticated block cipher. */ static void safexcel_instr_cipher(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; crp = req->crp; /* Insert the payload. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT; cdesc->additional_cdata_size = 1; } static void safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_instr *start; crp = req->crp; csp = crypto_get_params(crp->crp_session); start = instr; /* Insert the AAD. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; /* Encrypt any data left in the request. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_DEST_OUTPUT; instr++; } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_sha_hash(struct safexcel_request *req, struct safexcel_instr *instr) { struct cryptop *crp; struct safexcel_instr *start; crp = req->crp; start = instr; /* Pass the input data to the hash engine. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; /* Insert the hash result into the output stream. */ safexcel_instr_insert_digest(&instr, req->sess->digestlen); /* Pad the rest of the inline instruction space. */ while (instr != start + SAFEXCEL_MAX_ITOKENS) safexcel_instr_nop(&instr); } static void safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; uint8_t *a0, *b0, *alenp, L; int aalign, blen; crp = req->crp; start = instr; /* * Construct two blocks, A0 and B0, used in encryption and * authentication, respectively. A0 is embedded in the token * descriptor, and B0 is inserted directly into the data stream using * instructions below. * * OCF seems to assume a 12-byte IV, fixing L (the payload length size) * at 3 bytes due to the layout of B0. This is fine since the driver * has a maximum of 65535 bytes anyway. */ blen = AES_BLOCK_LEN; L = 3; a0 = (uint8_t *)&cdesc->control_data.token[0]; memset(a0, 0, blen); a0[0] = L - 1; memcpy(&a0[1], req->iv, AES_CCM_IV_LEN); /* * Insert B0 and the AAD length into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INSERT_IMMEDIATE; instr++; b0 = (uint8_t *)instr; memset(b0, 0, blen); b0[0] = (L - 1) | /* payload length size */ ((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ | (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; memcpy(&b0[1], req->iv, AES_CCM_IV_LEN); b0[14] = crp->crp_payload_length >> 8; b0[15] = crp->crp_payload_length & 0xff; instr += blen / sizeof(*instr); /* Insert the AAD length and data into the input stream. */ if (crp->crp_aad_length > 0) { alenp = (uint8_t *)instr; alenp[0] = crp->crp_aad_length >> 8; alenp[1] = crp->crp_aad_length & 0xff; alenp[2] = 0; alenp[3] = 0; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; /* Insert zero padding. */ aalign = (crp->crp_aad_length + 2) & (blen - 1); instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = aalign == 0 ? 0 : blen - ((crp->crp_aad_length + 2) & (blen - 1)); instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; } safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INS_LAST; instr++; /* Insert zero padding. */ if (crp->crp_payload_length & (blen - 1)) { instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = blen - (crp->crp_payload_length & (blen - 1)); instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; } } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); cdesc->control_data.token[3] = htobe32(1); crp = req->crp; start = instr; /* Insert the AAD into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_aad_length; instr->status = crp->crp_payload_length == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ if (crp->crp_payload_length > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INS_LAST; instr++; } /* * Compute the digest, or extract it and place it in the output stream. */ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { struct cryptop *crp; struct safexcel_instr *start; memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); cdesc->control_data.token[3] = htobe32(1); crp = req->crp; start = instr; instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; instr->length = crp->crp_payload_length; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; instr++; safexcel_instr_temp_aes_block(&instr); safexcel_instr_insert_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void safexcel_set_token(struct safexcel_request *req) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_cmd_descr *cdesc; struct safexcel_context_record *ctx; struct safexcel_context_template *ctxtmp; struct safexcel_instr *instr; struct safexcel_softc *sc; const uint8_t *akey, *ckey; int ringidx; crp = req->crp; csp = crypto_get_params(crp->crp_session); cdesc = req->cdesc; sc = req->sc; ringidx = req->ringidx; akey = crp->crp_auth_key; ckey = crp->crp_cipher_key; if (akey != NULL || ckey != NULL) { /* * If we have a per-request key we have to generate the context * record on the fly. */ if (akey == NULL) akey = csp->csp_auth_key; if (ckey == NULL) ckey = csp->csp_cipher_key; ctx = (struct safexcel_context_record *)req->ctx.vaddr; (void)safexcel_set_context(ctx, crp->crp_op, ckey, akey, req->sess); } else { /* * Use the context record template computed at session * initialization time. */ ctxtmp = CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ? &req->sess->encctx : &req->sess->decctx; ctx = &ctxtmp->ctx; memcpy(req->ctx.vaddr + 2 * sizeof(uint32_t), ctx->data, ctxtmp->len); } cdesc->control_data.control0 = ctx->control0; cdesc->control_data.control1 = ctx->control1; /* * For keyless hash operations, the token instructions can be embedded * in the token itself. Otherwise we use an additional token descriptor * and the embedded instruction space is used to store the IV. */ if (csp->csp_cipher_alg == 0 && csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { instr = (void *)cdesc->control_data.token; } else { instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + sc->sc_config.atok_offset * (cdesc - sc->sc_ring[ringidx].cdr.desc)); cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: safexcel_instr_gcm(req, instr, cdesc); break; case CRYPTO_AES_CCM_16: safexcel_instr_ccm(req, instr, cdesc); break; case CRYPTO_AES_XTS: memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); memset(cdesc->control_data.token + AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); safexcel_instr_cipher(req, instr, cdesc); break; case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); if (csp->csp_auth_alg != 0) safexcel_instr_eta(req, instr, cdesc); else safexcel_instr_cipher(req, instr, cdesc); break; default: switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: safexcel_instr_sha_hash(req, instr); break; case CRYPTO_AES_NIST_GMAC: safexcel_instr_gmac(req, instr, cdesc); break; default: panic("unhandled auth request %d", csp->csp_auth_alg); } break; } } static struct safexcel_res_descr * safexcel_res_descr_add(struct safexcel_ring *ring, bool first, bool last, bus_addr_t data, uint32_t len) { struct safexcel_res_descr *rdesc; struct safexcel_res_descr_ring *rring; mtx_assert(&ring->mtx, MA_OWNED); rring = &ring->rdr; if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) return (NULL); rdesc = &rring->desc[rring->write]; rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; rdesc->particle_size = len; rdesc->rsvd0 = 0; rdesc->descriptor_overflow = 0; rdesc->buffer_overflow = 0; rdesc->last_seg = last; rdesc->first_seg = first; rdesc->result_size = sizeof(struct safexcel_res_data) / sizeof(uint32_t); rdesc->rsvd1 = 0; rdesc->data_lo = SAFEXCEL_ADDR_LO(data); rdesc->data_hi = SAFEXCEL_ADDR_HI(data); if (first) { rdesc->result_data.packet_length = 0; rdesc->result_data.error_code = 0; } return (rdesc); } static struct safexcel_cmd_descr * safexcel_cmd_descr_add(struct safexcel_ring *ring, bool first, bool last, bus_addr_t data, uint32_t seglen, uint32_t reqlen, bus_addr_t context) { struct safexcel_cmd_descr *cdesc; struct safexcel_cmd_descr_ring *cring; KASSERT(reqlen <= SAFEXCEL_MAX_REQUEST_SIZE, ("%s: request length %u too long", __func__, reqlen)); mtx_assert(&ring->mtx, MA_OWNED); cring = &ring->cdr; if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) return (NULL); cdesc = &cring->desc[cring->write]; cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; cdesc->particle_size = seglen; cdesc->rsvd0 = 0; cdesc->last_seg = last; cdesc->first_seg = first; cdesc->additional_cdata_size = 0; cdesc->rsvd1 = 0; cdesc->data_lo = SAFEXCEL_ADDR_LO(data); cdesc->data_hi = SAFEXCEL_ADDR_HI(data); if (first) { cdesc->control_data.packet_length = reqlen; cdesc->control_data.options = SAFEXCEL_OPTION_IP | SAFEXCEL_OPTION_CP | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD | SAFEXCEL_OPTION_RC_AUTO; cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | SAFEXCEL_CONTEXT_SMALL; cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); } return (cdesc); } static void safexcel_cmd_descr_rollback(struct safexcel_ring *ring, int count) { struct safexcel_cmd_descr_ring *cring; mtx_assert(&ring->mtx, MA_OWNED); cring = &ring->cdr; cring->write -= count; if (cring->write < 0) cring->write += SAFEXCEL_RING_SIZE; } static void safexcel_res_descr_rollback(struct safexcel_ring *ring, int count) { struct safexcel_res_descr_ring *rring; mtx_assert(&ring->mtx, MA_OWNED); rring = &ring->rdr; rring->write -= count; if (rring->write < 0) rring->write += SAFEXCEL_RING_SIZE; } static void safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg, int start, int len) { bus_dma_segment_t *seg; size_t seglen; int error, i; for (i = 0; i < nseg && len > 0; i++) { seg = &segs[i]; if (seg->ds_len <= start) { start -= seg->ds_len; continue; } seglen = MIN(len, seg->ds_len - start); error = sglist_append_phys(sg, seg->ds_addr + start, seglen); if (error != 0) panic("%s: ran out of segments: %d", __func__, error); len -= seglen; start = 0; } } static void safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_cmd_descr *cdesc; struct safexcel_request *req; struct safexcel_ring *ring; struct safexcel_session *sess; struct sglist *sg; size_t inlen; int i; bool first, last; req = arg; if (error != 0) { req->error = error; return; } crp = req->crp; csp = crypto_get_params(crp->crp_session); sess = req->sess; ring = &req->sc->sc_ring[req->ringidx]; mtx_assert(&ring->mtx, MA_OWNED); /* * Set up descriptors for input and output data. * * The processing engine programs require that any AAD comes first, * followed by the cipher plaintext, followed by the digest. Some * consumers place the digest first in the input buffer, in which case * we have to create an extra descriptor. * * As an optimization, unmodified data is not passed to the output * stream. */ sglist_reset(ring->cmd_data); sglist_reset(ring->res_data); if (crp->crp_aad_length != 0) { safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_aad_start, crp->crp_aad_length); } safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_payload_start, crp->crp_payload_length); if (csp->csp_cipher_alg != 0) { safexcel_append_segs(segs, nseg, ring->res_data, crp->crp_payload_start, crp->crp_payload_length); } if (sess->digestlen > 0) { if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { safexcel_append_segs(segs, nseg, ring->cmd_data, crp->crp_digest_start, sess->digestlen); } else { safexcel_append_segs(segs, nseg, ring->res_data, crp->crp_digest_start, sess->digestlen); } } sg = ring->cmd_data; if (sg->sg_nseg == 0) { /* * Fake a segment for the command descriptor if the input has * length zero. The EIP97 apparently does not handle * zero-length packets properly since subsequent requests return * bogus errors, so provide a dummy segment using the context * descriptor. Also, we must allocate at least one command ring * entry per request to keep the request shadow ring in sync. */ (void)sglist_append_phys(sg, req->ctx.paddr, 1); } for (i = 0, inlen = 0; i < sg->sg_nseg; i++) inlen += sg->sg_segs[i].ss_len; for (i = 0; i < sg->sg_nseg; i++) { first = i == 0; last = i == sg->sg_nseg - 1; cdesc = safexcel_cmd_descr_add(ring, first, last, sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, (uint32_t)inlen, req->ctx.paddr); if (cdesc == NULL) { safexcel_cmd_descr_rollback(ring, i); counter_u64_add(req->sc->sc_cdesc_alloc_failures, 1); req->error = ERESTART; return; } if (i == 0) req->cdesc = cdesc; } req->cdescs = sg->sg_nseg; sg = ring->res_data; if (sg->sg_nseg == 0) { /* * We need a result descriptor even if the output stream will be * empty, for example when verifying an AAD digest. */ sg->sg_segs[0].ss_paddr = 0; sg->sg_segs[0].ss_len = 0; sg->sg_nseg = 1; } for (i = 0; i < sg->sg_nseg; i++) { first = i == 0; last = i == sg->sg_nseg - 1; if (safexcel_res_descr_add(ring, first, last, sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { safexcel_cmd_descr_rollback(ring, ring->cmd_data->sg_nseg); safexcel_res_descr_rollback(ring, i); counter_u64_add(req->sc->sc_rdesc_alloc_failures, 1); req->error = ERESTART; return; } } req->rdescs = sg->sg_nseg; } static int safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req) { int error; req->error = 0; req->cdescs = req->rdescs = 0; error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); if (error == 0) req->dmap_loaded = true; if (req->error != 0) error = req->error; return (error); } static bool safexcel_probe_cipher(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return (false); break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (false); break; default: return (false); } return (true); } /* * Determine whether the driver can implement a session with the requested * parameters. */ static int safexcel_probesession(device_t dev, const struct crypto_session_params *csp) { if (csp->csp_flags != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_CIPHER: if (!safexcel_probe_cipher(csp)) return (EINVAL); break; case CSP_MODE_DIGEST: switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); break; case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: break; default: return (EINVAL); } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); break; case CRYPTO_AES_CCM_16: if (csp->csp_ivlen != AES_CCM_IV_LEN) return (EINVAL); break; default: return (EINVAL); } break; case CSP_MODE_ETA: if (!safexcel_probe_cipher(csp)) return (EINVAL); switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: /* * The EIP-97 does not support combining AES-XTS with * hash operations. */ if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) return (EINVAL); break; default: return (EINVAL); } break; default: return (EINVAL); } return (CRYPTODEV_PROBE_HARDWARE); } static uint32_t safexcel_aes_algid(int keylen) { switch (keylen) { case 16: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128); case 24: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192); case 32: return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256); default: panic("invalid AES key length %d", keylen); } } static uint32_t safexcel_aes_ccm_hashid(int keylen) { switch (keylen) { case 16: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC128); case 24: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC192); case 32: return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC256); default: panic("invalid AES key length %d", keylen); } } static uint32_t safexcel_sha_hashid(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA1); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA224); case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA256); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA384); case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SAFEXCEL_CONTROL0_HASH_ALG_SHA512); default: __assert_unreachable(); } } static int safexcel_sha_hashlen(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SHA1_HASH_LEN); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: return (SHA2_224_HASH_LEN); case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SHA2_256_HASH_LEN); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: return (SHA2_384_HASH_LEN); case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SHA2_512_HASH_LEN); default: __assert_unreachable(); } } static int safexcel_sha_statelen(int alg) { switch (alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: return (SHA1_HASH_LEN); case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: return (SHA2_256_HASH_LEN); case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: return (SHA2_512_HASH_LEN); default: __assert_unreachable(); } } static int safexcel_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct safexcel_session *sess; struct safexcel_softc *sc; sc = device_get_softc(dev); sess = crypto_get_driver_session(cses); sess->cses = cses; switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED; sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); break; case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC; sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); break; case CRYPTO_AES_NIST_GMAC: sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; sess->digestlen = GMAC_DIGEST_LEN; sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; sess->alg = safexcel_aes_algid(csp->csp_auth_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; break; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; sess->digestlen = GMAC_DIGEST_LEN; sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; break; case CRYPTO_AES_CCM_16: sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM; sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM; break; case CRYPTO_AES_CBC: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC; break; case CRYPTO_AES_ICM: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR; break; case CRYPTO_AES_XTS: sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS; break; } if (csp->csp_auth_mlen != 0) sess->digestlen = csp->csp_auth_mlen; if ((csp->csp_cipher_alg == 0 || csp->csp_cipher_key != NULL) && (csp->csp_auth_alg == 0 || csp->csp_auth_key != NULL)) { sess->encctx.len = safexcel_set_context(&sess->encctx.ctx, CRYPTO_OP_ENCRYPT, csp->csp_cipher_key, csp->csp_auth_key, sess); sess->decctx.len = safexcel_set_context(&sess->decctx.ctx, CRYPTO_OP_DECRYPT, csp->csp_cipher_key, csp->csp_auth_key, sess); } return (0); } static int safexcel_process(device_t dev, struct cryptop *crp, int hint) { const struct crypto_session_params *csp; struct safexcel_request *req; struct safexcel_ring *ring; struct safexcel_session *sess; struct safexcel_softc *sc; int error; sc = device_get_softc(dev); sess = crypto_get_driver_session(crp->crp_session); csp = crypto_get_params(crp->crp_session); if (__predict_false(crypto_buffer_len(&crp->crp_buf) > SAFEXCEL_MAX_REQUEST_SIZE)) { crp->crp_etype = E2BIG; crypto_done(crp); return (0); } ring = &sc->sc_ring[curcpu % sc->sc_config.rings]; mtx_lock(&ring->mtx); req = safexcel_alloc_request(sc, ring); if (__predict_false(req == NULL)) { ring->blocked = CRYPTO_SYMQ; mtx_unlock(&ring->mtx); counter_u64_add(sc->sc_req_alloc_failures, 1); return (ERESTART); } req->crp = crp; req->sess = sess; crypto_read_iv(crp, req->iv); error = safexcel_create_chain(ring, req); if (__predict_false(error != 0)) { safexcel_free_request(ring, req); if (error == ERESTART) ring->blocked = CRYPTO_SYMQ; mtx_unlock(&ring->mtx); if (error != ERESTART) { crp->crp_etype = error; crypto_done(crp); return (0); } else { return (ERESTART); } } safexcel_set_token(req); bus_dmamap_sync(ring->data_dtag, req->dmap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(req->ctx.tag, req->ctx.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); safexcel_execute(sc, ring, req, hint); mtx_unlock(&ring->mtx); return (0); } static device_method_t safexcel_methods[] = { /* Device interface */ DEVMETHOD(device_probe, safexcel_probe), DEVMETHOD(device_attach, safexcel_attach), DEVMETHOD(device_detach, safexcel_detach), /* Cryptodev interface */ DEVMETHOD(cryptodev_probesession, safexcel_probesession), DEVMETHOD(cryptodev_newsession, safexcel_newsession), DEVMETHOD(cryptodev_process, safexcel_process), DEVMETHOD_END }; static devclass_t safexcel_devclass; static driver_t safexcel_driver = { .name = "safexcel", .methods = safexcel_methods, .size = sizeof(struct safexcel_softc), }; DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0); MODULE_VERSION(safexcel, 1); MODULE_DEPEND(safexcel, crypto, 1, 1, 1); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index b6c4441b4284..9a992100f222 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -1,1918 +1,1918 @@ /*- * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Cryptographic Subsystem. * * This code is derived from the Openbsd Cryptographic Framework (OCF) * that has the copyright shown below. Very little of the original * code remains. */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000, 2001 Angelos D. Keromytis * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include "opt_compat.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) #include #endif SDT_PROVIDER_DEFINE(opencrypto); /* * Crypto drivers register themselves by allocating a slot in the * crypto_drivers table with crypto_get_driverid(). */ static struct mtx crypto_drivers_mtx; /* lock on driver table */ #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED) /* * Crypto device/driver capabilities structure. * * Synchronization: * (d) - protected by CRYPTO_DRIVER_LOCK() * (q) - protected by CRYPTO_Q_LOCK() * Not tagged fields are read-only. */ struct cryptocap { device_t cc_dev; uint32_t cc_hid; uint32_t cc_sessions; /* (d) # of sessions */ int cc_flags; /* (d) flags */ #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ int cc_qblocked; /* (q) symmetric q blocked */ size_t cc_session_size; volatile int cc_refs; }; static struct cryptocap **crypto_drivers = NULL; static int crypto_drivers_size = 0; struct crypto_session { struct cryptocap *cap; struct crypto_session_params csp; uint64_t id; /* Driver softc follows. */ }; static int crp_sleep = 0; static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */ static struct mtx crypto_q_mtx; #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0, "In-kernel cryptography"); /* * Taskqueue used to dispatch the crypto requests * that have the CRYPTO_F_ASYNC flag */ static struct taskqueue *crypto_tq; /* * Crypto seq numbers are operated on with modular arithmetic */ #define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0) struct crypto_ret_worker { struct mtx crypto_ret_mtx; TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */ TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */ uint32_t reorder_ops; /* total ordered sym jobs received */ uint32_t reorder_cur_seq; /* current sym job dispatched */ struct proc *cryptoretproc; }; static struct crypto_ret_worker *crypto_ret_workers = NULL; #define CRYPTO_RETW(i) (&crypto_ret_workers[i]) #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers) #define FOREACH_CRYPTO_RETW(w) \ for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w) #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx) #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx) static int crypto_workers_num = 0; SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, &crypto_workers_num, 0, "Number of crypto workers used to dispatch crypto jobs"); #ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, &crypto_workers_num, 0, "Number of crypto workers used to dispatch crypto jobs"); #endif static uma_zone_t cryptop_zone; int crypto_devallowsoft = 0; SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW, &crypto_devallowsoft, 0, "Enable use of software crypto by /dev/crypto"); #ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, &crypto_devallowsoft, 0, "Enable/disable use of software crypto by /dev/crypto"); #endif MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); static void crypto_proc(void); static struct proc *cryptoproc; static void crypto_ret_proc(struct crypto_ret_worker *ret_worker); static void crypto_destroy(void); static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); static void crypto_task_invoke(void *ctx, int pending); static void crypto_batch_enqueue(struct cryptop *crp); static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)]; SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, cryptostats, nitems(cryptostats), "Crypto system statistics"); #define CRYPTOSTAT_INC(stat) do { \ counter_u64_add( \ cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\ 1); \ } while (0) static void cryptostats_init(void *arg __unused) { COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK); } SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL); static void cryptostats_fini(void *arg __unused) { COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats)); } SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini, NULL); /* Try to avoid directly exposing the key buffer as a symbol */ static struct keybuf *keybuf; static struct keybuf empty_keybuf = { .kb_nents = 0 }; /* Obtain the key buffer from boot metadata */ static void keybuf_init(void) { caddr_t kmdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); keybuf = (struct keybuf *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_KEYBUF); if (keybuf == NULL) keybuf = &empty_keybuf; } /* It'd be nice if we could store these in some kind of secure memory... */ struct keybuf * get_keybuf(void) { return (keybuf); } static struct cryptocap * cap_ref(struct cryptocap *cap) { refcount_acquire(&cap->cc_refs); return (cap); } static void cap_rele(struct cryptocap *cap) { if (refcount_release(&cap->cc_refs) == 0) return; KASSERT(cap->cc_sessions == 0, ("freeing crypto driver with active sessions")); free(cap, M_CRYPTO_DATA); } static int crypto_init(void) { struct crypto_ret_worker *ret_worker; int error; mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", MTX_DEF|MTX_QUIET); TAILQ_INIT(&crp_q); mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); cryptop_zone = uma_zcreate("cryptop", sizeof(struct cryptop), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); crypto_drivers_size = CRYPTO_DRIVERS_INITIAL; crypto_drivers = malloc(crypto_drivers_size * sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO); if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus) crypto_workers_num = mp_ncpus; crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &crypto_tq); taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN, "crypto"); error = kproc_create((void (*)(void *)) crypto_proc, NULL, &cryptoproc, 0, 0, "crypto"); if (error) { printf("crypto_init: cannot start crypto thread; error %d", error); goto bad; } crypto_ret_workers = mallocarray(crypto_workers_num, sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO); FOREACH_CRYPTO_RETW(ret_worker) { TAILQ_INIT(&ret_worker->crp_ordered_ret_q); TAILQ_INIT(&ret_worker->crp_ret_q); ret_worker->reorder_ops = 0; ret_worker->reorder_cur_seq = 0; mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF); error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker, &ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker)); if (error) { printf("crypto_init: cannot start cryptoret thread; error %d", error); goto bad; } } keybuf_init(); return 0; bad: crypto_destroy(); return error; } /* * Signal a crypto thread to terminate. We use the driver * table lock to synchronize the sleep/wakeups so that we * are sure the threads have terminated before we release * the data structures they use. See crypto_finis below * for the other half of this song-and-dance. */ static void crypto_terminate(struct proc **pp, void *q) { struct proc *p; mtx_assert(&crypto_drivers_mtx, MA_OWNED); p = *pp; *pp = NULL; if (p) { wakeup_one(q); PROC_LOCK(p); /* NB: insure we don't miss wakeup */ CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); PROC_UNLOCK(p); CRYPTO_DRIVER_LOCK(); } } static void hmac_init_pad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx, uint8_t padval) { uint8_t hmac_key[HMAC_MAX_BLOCK_LEN]; u_int i; KASSERT(axf->blocksize <= sizeof(hmac_key), ("Invalid HMAC block size %d", axf->blocksize)); /* * If the key is larger than the block size, use the digest of * the key as the key instead. */ memset(hmac_key, 0, sizeof(hmac_key)); if (klen > axf->blocksize) { axf->Init(auth_ctx); axf->Update(auth_ctx, key, klen); axf->Final(hmac_key, auth_ctx); klen = axf->hashsize; } else memcpy(hmac_key, key, klen); for (i = 0; i < axf->blocksize; i++) hmac_key[i] ^= padval; axf->Init(auth_ctx); axf->Update(auth_ctx, hmac_key, axf->blocksize); explicit_bzero(hmac_key, sizeof(hmac_key)); } void hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx) { hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL); } void hmac_init_opad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx) { hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL); } static void crypto_destroy(void) { struct crypto_ret_worker *ret_worker; int i; /* * Terminate any crypto threads. */ if (crypto_tq != NULL) taskqueue_drain_all(crypto_tq); CRYPTO_DRIVER_LOCK(); crypto_terminate(&cryptoproc, &crp_q); FOREACH_CRYPTO_RETW(ret_worker) crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q); CRYPTO_DRIVER_UNLOCK(); /* XXX flush queues??? */ /* * Reclaim dynamically allocated resources. */ for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] != NULL) cap_rele(crypto_drivers[i]); } free(crypto_drivers, M_CRYPTO_DATA); if (cryptop_zone != NULL) uma_zdestroy(cryptop_zone); mtx_destroy(&crypto_q_mtx); FOREACH_CRYPTO_RETW(ret_worker) mtx_destroy(&ret_worker->crypto_ret_mtx); free(crypto_ret_workers, M_CRYPTO_DATA); if (crypto_tq != NULL) taskqueue_free(crypto_tq); mtx_destroy(&crypto_drivers_mtx); } uint32_t crypto_ses2hid(crypto_session_t crypto_session) { return (crypto_session->cap->cc_hid); } uint32_t crypto_ses2caps(crypto_session_t crypto_session) { return (crypto_session->cap->cc_flags & 0xff000000); } void * crypto_get_driver_session(crypto_session_t crypto_session) { return (crypto_session + 1); } const struct crypto_session_params * crypto_get_params(crypto_session_t crypto_session) { return (&crypto_session->csp); } -struct auth_hash * +const struct auth_hash * crypto_auth_hash(const struct crypto_session_params *csp) { switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: return (&auth_hash_hmac_sha1); case CRYPTO_SHA2_224_HMAC: return (&auth_hash_hmac_sha2_224); case CRYPTO_SHA2_256_HMAC: return (&auth_hash_hmac_sha2_256); case CRYPTO_SHA2_384_HMAC: return (&auth_hash_hmac_sha2_384); case CRYPTO_SHA2_512_HMAC: return (&auth_hash_hmac_sha2_512); case CRYPTO_NULL_HMAC: return (&auth_hash_null); case CRYPTO_RIPEMD160_HMAC: return (&auth_hash_hmac_ripemd_160); case CRYPTO_SHA1: return (&auth_hash_sha1); case CRYPTO_SHA2_224: return (&auth_hash_sha2_224); case CRYPTO_SHA2_256: return (&auth_hash_sha2_256); case CRYPTO_SHA2_384: return (&auth_hash_sha2_384); case CRYPTO_SHA2_512: return (&auth_hash_sha2_512); case CRYPTO_AES_NIST_GMAC: switch (csp->csp_auth_klen) { case 128 / 8: return (&auth_hash_nist_gmac_aes_128); case 192 / 8: return (&auth_hash_nist_gmac_aes_192); case 256 / 8: return (&auth_hash_nist_gmac_aes_256); default: return (NULL); } case CRYPTO_BLAKE2B: return (&auth_hash_blake2b); case CRYPTO_BLAKE2S: return (&auth_hash_blake2s); case CRYPTO_POLY1305: return (&auth_hash_poly1305); case CRYPTO_AES_CCM_CBC_MAC: switch (csp->csp_auth_klen) { case 128 / 8: return (&auth_hash_ccm_cbc_mac_128); case 192 / 8: return (&auth_hash_ccm_cbc_mac_192); case 256 / 8: return (&auth_hash_ccm_cbc_mac_256); default: return (NULL); } default: return (NULL); } } -struct enc_xform * +const struct enc_xform * crypto_cipher(const struct crypto_session_params *csp) { switch (csp->csp_cipher_alg) { case CRYPTO_RIJNDAEL128_CBC: return (&enc_xform_rijndael128); case CRYPTO_AES_XTS: return (&enc_xform_aes_xts); case CRYPTO_AES_ICM: return (&enc_xform_aes_icm); case CRYPTO_AES_NIST_GCM_16: return (&enc_xform_aes_nist_gcm); case CRYPTO_CAMELLIA_CBC: return (&enc_xform_camellia); case CRYPTO_NULL_CBC: return (&enc_xform_null); case CRYPTO_CHACHA20: return (&enc_xform_chacha20); case CRYPTO_AES_CCM_16: return (&enc_xform_ccm); case CRYPTO_CHACHA20_POLY1305: return (&enc_xform_chacha20_poly1305); default: return (NULL); } } static struct cryptocap * crypto_checkdriver(uint32_t hid) { return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]); } /* * Select a driver for a new session that supports the specified * algorithms and, optionally, is constrained according to the flags. */ static struct cryptocap * crypto_select_driver(const struct crypto_session_params *csp, int flags) { struct cryptocap *cap, *best; int best_match, error, hid; CRYPTO_DRIVER_ASSERT(); best = NULL; for (hid = 0; hid < crypto_drivers_size; hid++) { /* * If there is no driver for this slot, or the driver * is not appropriate (hardware or software based on * match), then skip. */ cap = crypto_drivers[hid]; if (cap == NULL || (cap->cc_flags & flags) == 0) continue; error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp); if (error >= 0) continue; /* * Use the driver with the highest probe value. * Hardware drivers use a higher probe value than * software. In case of a tie, prefer the driver with * the fewest active sessions. */ if (best == NULL || error > best_match || (error == best_match && cap->cc_sessions < best->cc_sessions)) { best = cap; best_match = error; } } return best; } static enum alg_type { ALG_NONE = 0, ALG_CIPHER, ALG_DIGEST, ALG_KEYED_DIGEST, ALG_COMPRESSION, ALG_AEAD } alg_types[] = { [CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_CBC] = ALG_CIPHER, [CRYPTO_SHA1] = ALG_DIGEST, [CRYPTO_NULL_HMAC] = ALG_DIGEST, [CRYPTO_NULL_CBC] = ALG_CIPHER, [CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION, [CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_CAMELLIA_CBC] = ALG_CIPHER, [CRYPTO_AES_XTS] = ALG_CIPHER, [CRYPTO_AES_ICM] = ALG_CIPHER, [CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_NIST_GCM_16] = ALG_AEAD, [CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST, [CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST, [CRYPTO_CHACHA20] = ALG_CIPHER, [CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST, [CRYPTO_RIPEMD160] = ALG_DIGEST, [CRYPTO_SHA2_224] = ALG_DIGEST, [CRYPTO_SHA2_256] = ALG_DIGEST, [CRYPTO_SHA2_384] = ALG_DIGEST, [CRYPTO_SHA2_512] = ALG_DIGEST, [CRYPTO_POLY1305] = ALG_KEYED_DIGEST, [CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST, [CRYPTO_AES_CCM_16] = ALG_AEAD, [CRYPTO_CHACHA20_POLY1305] = ALG_AEAD, }; static enum alg_type alg_type(int alg) { if (alg < nitems(alg_types)) return (alg_types[alg]); return (ALG_NONE); } static bool alg_is_compression(int alg) { return (alg_type(alg) == ALG_COMPRESSION); } static bool alg_is_cipher(int alg) { return (alg_type(alg) == ALG_CIPHER); } static bool alg_is_digest(int alg) { return (alg_type(alg) == ALG_DIGEST || alg_type(alg) == ALG_KEYED_DIGEST); } static bool alg_is_keyed_digest(int alg) { return (alg_type(alg) == ALG_KEYED_DIGEST); } static bool alg_is_aead(int alg) { return (alg_type(alg) == ALG_AEAD); } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) /* Various sanity checks on crypto session parameters. */ static bool check_csp(const struct crypto_session_params *csp) { - struct auth_hash *axf; + const struct auth_hash *axf; /* Mode-independent checks. */ if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (false); if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 || csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0) return (false); if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0) return (false); if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0) return (false); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: if (!alg_is_compression(csp->csp_cipher_alg)) return (false); if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 || csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || csp->csp_auth_mlen != 0) return (false); break; case CSP_MODE_CIPHER: if (!alg_is_cipher(csp->csp_cipher_alg)) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0) return (false); } if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || csp->csp_auth_mlen != 0) return (false); break; case CSP_MODE_DIGEST: if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0) return (false); if (csp->csp_flags & CSP_F_SEPARATE_AAD) return (false); /* IV is optional for digests (e.g. GMAC). */ if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (!alg_is_digest(csp->csp_auth_alg)) return (false); /* Key is optional for BLAKE2 digests. */ if (csp->csp_auth_alg == CRYPTO_BLAKE2B || csp->csp_auth_alg == CRYPTO_BLAKE2S) ; else if (alg_is_keyed_digest(csp->csp_auth_alg)) { if (csp->csp_auth_klen == 0) return (false); } else { if (csp->csp_auth_klen != 0) return (false); } if (csp->csp_auth_mlen != 0) { axf = crypto_auth_hash(csp); if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) return (false); } break; case CSP_MODE_AEAD: if (!alg_is_aead(csp->csp_cipher_alg)) return (false); if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0 || csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0) return (false); /* * XXX: Would be nice to have a better way to get this * value. */ switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: if (csp->csp_auth_mlen > 16) return (false); break; } break; case CSP_MODE_ETA: if (!alg_is_cipher(csp->csp_cipher_alg)) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { if (csp->csp_cipher_klen == 0) return (false); if (csp->csp_ivlen == 0) return (false); } if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) return (false); if (!alg_is_digest(csp->csp_auth_alg)) return (false); /* Key is optional for BLAKE2 digests. */ if (csp->csp_auth_alg == CRYPTO_BLAKE2B || csp->csp_auth_alg == CRYPTO_BLAKE2S) ; else if (alg_is_keyed_digest(csp->csp_auth_alg)) { if (csp->csp_auth_klen == 0) return (false); } else { if (csp->csp_auth_klen != 0) return (false); } if (csp->csp_auth_mlen != 0) { axf = crypto_auth_hash(csp); if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) return (false); } break; default: return (false); } return (true); } /* * Delete a session after it has been detached from its driver. */ static void crypto_deletesession(crypto_session_t cses) { struct cryptocap *cap; cap = cses->cap; zfree(cses, M_CRYPTO_DATA); CRYPTO_DRIVER_LOCK(); cap->cc_sessions--; if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP) wakeup(cap); CRYPTO_DRIVER_UNLOCK(); cap_rele(cap); } /* * Create a new session. The crid argument specifies a crypto * driver to use or constraints on a driver to select (hardware * only, software only, either). Whatever driver is selected * must be capable of the requested crypto algorithms. */ int crypto_newsession(crypto_session_t *cses, const struct crypto_session_params *csp, int crid) { static uint64_t sessid = 0; crypto_session_t res; struct cryptocap *cap; int err; if (!check_csp(csp)) return (EINVAL); res = NULL; CRYPTO_DRIVER_LOCK(); if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { /* * Use specified driver; verify it is capable. */ cap = crypto_checkdriver(crid); if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0) cap = NULL; } else { /* * No requested driver; select based on crid flags. */ cap = crypto_select_driver(csp, crid); } if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); CRYPTDEB("no driver"); return (EOPNOTSUPP); } cap_ref(cap); cap->cc_sessions++; CRYPTO_DRIVER_UNLOCK(); /* Allocate a single block for the generic session and driver softc. */ res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO); res->cap = cap; res->csp = *csp; res->id = atomic_fetchadd_64(&sessid, 1); /* Call the driver initialization routine. */ err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp); if (err != 0) { CRYPTDEB("dev newsession failed: %d", err); crypto_deletesession(res); return (err); } *cses = res; return (0); } /* * Delete an existing session (or a reserved session on an unregistered * driver). */ void crypto_freesession(crypto_session_t cses) { struct cryptocap *cap; if (cses == NULL) return; cap = cses->cap; /* Call the driver cleanup routine, if available. */ CRYPTODEV_FREESESSION(cap->cc_dev, cses); crypto_deletesession(cses); } /* * Return a new driver id. Registers a driver with the system so that * it can be probed by subsequent sessions. */ int32_t crypto_get_driverid(device_t dev, size_t sessionsize, int flags) { struct cryptocap *cap, **newdrv; int i; if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { device_printf(dev, "no flags specified when registering driver\n"); return -1; } cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO); cap->cc_dev = dev; cap->cc_session_size = sessionsize; cap->cc_flags = flags; refcount_init(&cap->cc_refs, 1); CRYPTO_DRIVER_LOCK(); for (;;) { for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] == NULL) break; } if (i < crypto_drivers_size) break; /* Out of entries, allocate some more. */ if (2 * crypto_drivers_size <= crypto_drivers_size) { CRYPTO_DRIVER_UNLOCK(); printf("crypto: driver count wraparound!\n"); cap_rele(cap); return (-1); } CRYPTO_DRIVER_UNLOCK(); newdrv = malloc(2 * crypto_drivers_size * sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO); CRYPTO_DRIVER_LOCK(); memcpy(newdrv, crypto_drivers, crypto_drivers_size * sizeof(*crypto_drivers)); crypto_drivers_size *= 2; free(crypto_drivers, M_CRYPTO_DATA); crypto_drivers = newdrv; } cap->cc_hid = i; crypto_drivers[i] = cap; CRYPTO_DRIVER_UNLOCK(); if (bootverbose) printf("crypto: assign %s driver id %u, flags 0x%x\n", device_get_nameunit(dev), i, flags); return i; } /* * Lookup a driver by name. We match against the full device * name and unit, and against just the name. The latter gives * us a simple widlcarding by device name. On success return the * driver/hardware identifier; otherwise return -1. */ int crypto_find_driver(const char *match) { struct cryptocap *cap; int i, len = strlen(match); CRYPTO_DRIVER_LOCK(); for (i = 0; i < crypto_drivers_size; i++) { if (crypto_drivers[i] == NULL) continue; cap = crypto_drivers[i]; if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 || strncmp(match, device_get_name(cap->cc_dev), len) == 0) { CRYPTO_DRIVER_UNLOCK(); return (i); } } CRYPTO_DRIVER_UNLOCK(); return (-1); } /* * Return the device_t for the specified driver or NULL * if the driver identifier is invalid. */ device_t crypto_find_device_byhid(int hid) { struct cryptocap *cap; device_t dev; dev = NULL; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(hid); if (cap != NULL) dev = cap->cc_dev; CRYPTO_DRIVER_UNLOCK(); return (dev); } /* * Return the device/driver capabilities. */ int crypto_getcaps(int hid) { struct cryptocap *cap; int flags; flags = 0; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(hid); if (cap != NULL) flags = cap->cc_flags; CRYPTO_DRIVER_UNLOCK(); return (flags); } /* * Unregister all algorithms associated with a crypto driver. * If there are pending sessions using it, leave enough information * around so that subsequent calls using those sessions will * correctly detect the driver has been unregistered and reroute * requests. */ int crypto_unregister_all(uint32_t driverid) { struct cryptocap *cap; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(driverid); if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); return (EINVAL); } cap->cc_flags |= CRYPTOCAP_F_CLEANUP; crypto_drivers[driverid] = NULL; /* * XXX: This doesn't do anything to kick sessions that * have no pending operations. */ while (cap->cc_sessions != 0) mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0); CRYPTO_DRIVER_UNLOCK(); cap_rele(cap); return (0); } /* * Clear blockage on a driver. The what parameter indicates whether * the driver is now ready for cryptop's and/or cryptokop's. */ int crypto_unblock(uint32_t driverid, int what) { struct cryptocap *cap; int err; CRYPTO_Q_LOCK(); cap = crypto_checkdriver(driverid); if (cap != NULL) { if (what & CRYPTO_SYMQ) cap->cc_qblocked = 0; if (crp_sleep) wakeup_one(&crp_q); err = 0; } else err = EINVAL; CRYPTO_Q_UNLOCK(); return err; } size_t crypto_buffer_len(struct crypto_buffer *cb) { switch (cb->cb_type) { case CRYPTO_BUF_CONTIG: return (cb->cb_buf_len); case CRYPTO_BUF_MBUF: if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); case CRYPTO_BUF_SINGLE_MBUF: return (cb->cb_mbuf->m_len); case CRYPTO_BUF_VMPAGE: return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: return (cb->cb_uio->uio_resid); default: return (0); } } #ifdef INVARIANTS /* Various sanity checks on crypto requests. */ static void cb_sanity(struct crypto_buffer *cb, const char *name) { KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid %s buffer type", name)); switch (cb->cb_type) { case CRYPTO_BUF_CONTIG: KASSERT(cb->cb_buf_len >= 0, ("incoming crp with -ve %s buffer length", name)); break; case CRYPTO_BUF_VMPAGE: KASSERT(CRYPTO_HAS_VMPAGE, ("incoming crp uses dmap on supported arch")); KASSERT(cb->cb_vm_page_len >= 0, ("incoming crp with -ve %s buffer length", name)); KASSERT(cb->cb_vm_page_offset >= 0, ("incoming crp with -ve %s buffer offset", name)); KASSERT(cb->cb_vm_page_offset < PAGE_SIZE, ("incoming crp with %s buffer offset greater than page size" , name)); break; default: break; } } static void crp_sanity(struct cryptop *crp) { struct crypto_session_params *csp; struct crypto_buffer *out; size_t ilen, len, olen; KASSERT(crp->crp_session != NULL, ("incoming crp without a session")); KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE && crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid output buffer type")); KASSERT(crp->crp_etype == 0, ("incoming crp with error")); KASSERT(!(crp->crp_flags & CRYPTO_F_DONE), ("incoming crp already done")); csp = &crp->crp_session->csp; cb_sanity(&crp->crp_buf, "input"); ilen = crypto_buffer_len(&crp->crp_buf); olen = ilen; out = NULL; if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) { if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) { cb_sanity(&crp->crp_obuf, "output"); out = &crp->crp_obuf; olen = crypto_buffer_len(out); } } else KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE, ("incoming crp with separate output buffer " "but no session support")); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS || crp->crp_op == CRYPTO_OP_DECOMPRESS, ("invalid compression op %x", crp->crp_op)); break; case CSP_MODE_CIPHER: KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT || crp->crp_op == CRYPTO_OP_DECRYPT, ("invalid cipher op %x", crp->crp_op)); break; case CSP_MODE_DIGEST: KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST || crp->crp_op == CRYPTO_OP_VERIFY_DIGEST, ("invalid digest op %x", crp->crp_op)); break; case CSP_MODE_AEAD: KASSERT(crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || crp->crp_op == (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), ("invalid AEAD op %x", crp->crp_op)); KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE, ("AEAD without a separate IV")); break; case CSP_MODE_ETA: KASSERT(crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || crp->crp_op == (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), ("invalid ETA op %x", crp->crp_op)); break; } if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { if (crp->crp_aad == NULL) { KASSERT(crp->crp_aad_start == 0 || crp->crp_aad_start < ilen, ("invalid AAD start")); KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0, ("AAD with zero length and non-zero start")); KASSERT(crp->crp_aad_length == 0 || crp->crp_aad_start + crp->crp_aad_length <= ilen, ("AAD outside input length")); } else { KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD, ("session doesn't support separate AAD buffer")); KASSERT(crp->crp_aad_start == 0, ("separate AAD buffer with non-zero AAD start")); KASSERT(crp->crp_aad_length != 0, ("separate AAD buffer with zero length")); } } else { KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 && crp->crp_aad_length == 0, ("AAD region in request not supporting AAD")); } if (csp->csp_ivlen == 0) { KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0, ("IV_SEPARATE set when IV isn't used")); KASSERT(crp->crp_iv_start == 0, ("crp_iv_start set when IV isn't used")); } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { KASSERT(crp->crp_iv_start == 0, ("IV_SEPARATE used with non-zero IV start")); } else { KASSERT(crp->crp_iv_start < ilen, ("invalid IV start")); KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen, ("IV outside buffer length")); } /* XXX: payload_start of 0 should always be < ilen? */ KASSERT(crp->crp_payload_start == 0 || crp->crp_payload_start < ilen, ("invalid payload start")); KASSERT(crp->crp_payload_start + crp->crp_payload_length <= ilen, ("payload outside input buffer")); if (out == NULL) { KASSERT(crp->crp_payload_output_start == 0, ("payload output start non-zero without output buffer")); } else { KASSERT(crp->crp_payload_output_start < olen, ("invalid payload output start")); KASSERT(crp->crp_payload_output_start + crp->crp_payload_length <= olen, ("payload outside output buffer")); } if (csp->csp_mode == CSP_MODE_DIGEST || csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) len = ilen; else len = olen; KASSERT(crp->crp_digest_start == 0 || crp->crp_digest_start < len, ("invalid digest start")); /* XXX: For the mlen == 0 case this check isn't perfect. */ KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len, ("digest outside buffer")); } else { KASSERT(crp->crp_digest_start == 0, ("non-zero digest start for request without a digest")); } if (csp->csp_cipher_klen != 0) KASSERT(csp->csp_cipher_key != NULL || crp->crp_cipher_key != NULL, ("cipher request without a key")); if (csp->csp_auth_klen != 0) KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL, ("auth request without a key")); KASSERT(crp->crp_callback != NULL, ("incoming crp without callback")); } #endif static int crypto_dispatch_one(struct cryptop *crp, int hint) { struct cryptocap *cap; int result; #ifdef INVARIANTS crp_sanity(crp); #endif CRYPTOSTAT_INC(cs_ops); crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; /* * Caller marked the request to be processed immediately; dispatch it * directly to the driver unless the driver is currently blocked, in * which case it is queued for deferred dispatch. */ cap = crp->crp_session->cap; if (!atomic_load_int(&cap->cc_qblocked)) { result = crypto_invoke(cap, crp, hint); if (result != ERESTART) return (result); /* * The driver ran out of resources, put the request on the * queue. */ } crypto_batch_enqueue(crp); return (0); } int crypto_dispatch(struct cryptop *crp) { return (crypto_dispatch_one(crp, 0)); } int crypto_dispatch_async(struct cryptop *crp, int flags) { struct crypto_ret_worker *ret_worker; if (!CRYPTO_SESS_SYNC(crp->crp_session)) { /* * The driver issues completions asynchonously, don't bother * deferring dispatch to a worker thread. */ return (crypto_dispatch(crp)); } #ifdef INVARIANTS crp_sanity(crp); #endif CRYPTOSTAT_INC(cs_ops); crp->crp_retw_id = crp->crp_session->id % crypto_workers_num; if ((flags & CRYPTO_ASYNC_ORDERED) != 0) { crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED; ret_worker = CRYPTO_RETW(crp->crp_retw_id); CRYPTO_RETW_LOCK(ret_worker); crp->crp_seq = ret_worker->reorder_ops++; CRYPTO_RETW_UNLOCK(ret_worker); } TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp); taskqueue_enqueue(crypto_tq, &crp->crp_task); return (0); } void crypto_dispatch_batch(struct cryptopq *crpq, int flags) { struct cryptop *crp; int hint; while ((crp = TAILQ_FIRST(crpq)) != NULL) { hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0; TAILQ_REMOVE(crpq, crp, crp_next); if (crypto_dispatch_one(crp, hint) != 0) crypto_batch_enqueue(crp); } } static void crypto_batch_enqueue(struct cryptop *crp) { CRYPTO_Q_LOCK(); TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); if (crp_sleep) wakeup_one(&crp_q); CRYPTO_Q_UNLOCK(); } static void crypto_task_invoke(void *ctx, int pending) { struct cryptocap *cap; struct cryptop *crp; int result; crp = (struct cryptop *)ctx; cap = crp->crp_session->cap; result = crypto_invoke(cap, crp, 0); if (result == ERESTART) crypto_batch_enqueue(crp); } /* * Dispatch a crypto request to the appropriate crypto devices. */ static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) { KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); KASSERT(crp->crp_callback != NULL, ("%s: crp->crp_callback == NULL", __func__)); KASSERT(crp->crp_session != NULL, ("%s: crp->crp_session == NULL", __func__)); if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { struct crypto_session_params csp; crypto_session_t nses; /* * Driver has unregistered; migrate the session and return * an error to the caller so they'll resubmit the op. * * XXX: What if there are more already queued requests for this * session? * * XXX: Real solution is to make sessions refcounted * and force callers to hold a reference when * assigning to crp_session. Could maybe change * crypto_getreq to accept a session pointer to make * that work. Alternatively, we could abandon the * notion of rewriting crp_session in requests forcing * the caller to deal with allocating a new session. * Perhaps provide a method to allow a crp's session to * be swapped that callers could use. */ csp = crp->crp_session->csp; crypto_freesession(crp->crp_session); /* * XXX: Key pointers may no longer be valid. If we * really want to support this we need to define the * KPI such that 'csp' is required to be valid for the * duration of a session by the caller perhaps. * * XXX: If the keys have been changed this will reuse * the old keys. This probably suggests making * rekeying more explicit and updating the key * pointers in 'csp' when the keys change. */ if (crypto_newsession(&nses, &csp, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) crp->crp_session = nses; crp->crp_etype = EAGAIN; crypto_done(crp); return 0; } else { /* * Invoke the driver to process the request. */ return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); } } void crypto_destroyreq(struct cryptop *crp) { #ifdef DIAGNOSTIC { struct cryptop *crp2; struct crypto_ret_worker *ret_worker; CRYPTO_Q_LOCK(); TAILQ_FOREACH(crp2, &crp_q, crp_next) { KASSERT(crp2 != crp, ("Freeing cryptop from the crypto queue (%p).", crp)); } CRYPTO_Q_UNLOCK(); FOREACH_CRYPTO_RETW(ret_worker) { CRYPTO_RETW_LOCK(ret_worker); TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) { KASSERT(crp2 != crp, ("Freeing cryptop from the return queue (%p).", crp)); } CRYPTO_RETW_UNLOCK(ret_worker); } } #endif } void crypto_freereq(struct cryptop *crp) { if (crp == NULL) return; crypto_destroyreq(crp); uma_zfree(cryptop_zone, crp); } static void _crypto_initreq(struct cryptop *crp, crypto_session_t cses) { crp->crp_session = cses; } void crypto_initreq(struct cryptop *crp, crypto_session_t cses) { memset(crp, 0, sizeof(*crp)); _crypto_initreq(crp, cses); } struct cryptop * crypto_getreq(crypto_session_t cses, int how) { struct cryptop *crp; MPASS(how == M_WAITOK || how == M_NOWAIT); crp = uma_zalloc(cryptop_zone, how | M_ZERO); if (crp != NULL) _crypto_initreq(crp, cses); return (crp); } /* * Invoke the callback on behalf of the driver. */ void crypto_done(struct cryptop *crp) { KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); crp->crp_flags |= CRYPTO_F_DONE; if (crp->crp_etype != 0) CRYPTOSTAT_INC(cs_errs); /* * CBIMM means unconditionally do the callback immediately; * CBIFSYNC means do the callback immediately only if the * operation was done synchronously. Both are used to avoid * doing extraneous context switches; the latter is mostly * used with the software crypto driver. */ if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 && ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 || ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 && CRYPTO_SESS_SYNC(crp->crp_session)))) { /* * Do the callback directly. This is ok when the * callback routine does very little (e.g. the * /dev/crypto callback method just does a wakeup). */ crp->crp_callback(crp); } else { struct crypto_ret_worker *ret_worker; bool wake; ret_worker = CRYPTO_RETW(crp->crp_retw_id); /* * Normal case; queue the callback for the thread. */ CRYPTO_RETW_LOCK(ret_worker); if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) { struct cryptop *tmp; TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q, cryptop_q, crp_next) { if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) { TAILQ_INSERT_AFTER( &ret_worker->crp_ordered_ret_q, tmp, crp, crp_next); break; } } if (tmp == NULL) { TAILQ_INSERT_HEAD( &ret_worker->crp_ordered_ret_q, crp, crp_next); } wake = crp->crp_seq == ret_worker->reorder_cur_seq; } else { wake = TAILQ_EMPTY(&ret_worker->crp_ret_q); TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next); } if (wake) wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */ CRYPTO_RETW_UNLOCK(ret_worker); } } /* * Terminate a thread at module unload. The process that * initiated this is waiting for us to signal that we're gone; * wake it up and exit. We use the driver table lock to insure * we don't do the wakeup before they're waiting. There is no * race here because the waiter sleeps on the proc lock for the * thread so it gets notified at the right time because of an * extra wakeup that's done in exit1(). */ static void crypto_finis(void *chan) { CRYPTO_DRIVER_LOCK(); wakeup_one(chan); CRYPTO_DRIVER_UNLOCK(); kproc_exit(0); } /* * Crypto thread, dispatches crypto requests. */ static void crypto_proc(void) { struct cryptop *crp, *submit; struct cryptocap *cap; int result, hint; #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) fpu_kern_thread(FPU_KERN_NORMAL); #endif CRYPTO_Q_LOCK(); for (;;) { /* * Find the first element in the queue that can be * processed and look-ahead to see if multiple ops * are ready for the same driver. */ submit = NULL; hint = 0; TAILQ_FOREACH(crp, &crp_q, crp_next) { cap = crp->crp_session->cap; /* * Driver cannot disappeared when there is an active * session. */ KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { /* Op needs to be migrated, process it. */ if (submit == NULL) submit = crp; break; } if (!cap->cc_qblocked) { if (submit != NULL) { /* * We stop on finding another op, * regardless whether its for the same * driver or not. We could keep * searching the queue but it might be * better to just use a per-driver * queue instead. */ if (submit->crp_session->cap == cap) hint = CRYPTO_HINT_MORE; } else { submit = crp; } break; } } if (submit != NULL) { TAILQ_REMOVE(&crp_q, submit, crp_next); cap = submit->crp_session->cap; KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); CRYPTO_Q_UNLOCK(); result = crypto_invoke(cap, submit, hint); CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the * driver ``blocked'' for cryptop's and put * the request back in the queue. It would * best to put the request back where we got * it but that's hard so for now we put it * at the front. This should be ok; putting * it at the end does not work. */ cap->cc_qblocked = 1; TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); CRYPTOSTAT_INC(cs_blocks); } } else { /* * Nothing more to be processed. Sleep until we're * woken because there are more ops to process. * This happens either by submission or by a driver * becoming unblocked and notifying us through * crypto_unblock. Note that when we wakeup we * start processing each queue again from the * front. It's not clear that it's important to * preserve this ordering since ops may finish * out of order if dispatched to different devices * and some become blocked while others do not. */ crp_sleep = 1; msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); crp_sleep = 0; if (cryptoproc == NULL) break; CRYPTOSTAT_INC(cs_intrs); } } CRYPTO_Q_UNLOCK(); crypto_finis(&crp_q); } /* * Crypto returns thread, does callbacks for processed crypto requests. * Callbacks are done here, rather than in the crypto drivers, because * callbacks typically are expensive and would slow interrupt handling. */ static void crypto_ret_proc(struct crypto_ret_worker *ret_worker) { struct cryptop *crpt; CRYPTO_RETW_LOCK(ret_worker); for (;;) { /* Harvest return q's for completed ops */ crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q); if (crpt != NULL) { if (crpt->crp_seq == ret_worker->reorder_cur_seq) { TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next); ret_worker->reorder_cur_seq++; } else { crpt = NULL; } } if (crpt == NULL) { crpt = TAILQ_FIRST(&ret_worker->crp_ret_q); if (crpt != NULL) TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next); } if (crpt != NULL) { CRYPTO_RETW_UNLOCK(ret_worker); /* * Run callbacks unlocked. */ if (crpt != NULL) crpt->crp_callback(crpt); CRYPTO_RETW_LOCK(ret_worker); } else { /* * Nothing more to be processed. Sleep until we're * woken because there are more returns to process. */ msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT, "crypto_ret_wait", 0); if (ret_worker->cryptoretproc == NULL) break; CRYPTOSTAT_INC(cs_rets); } } CRYPTO_RETW_UNLOCK(ret_worker); crypto_finis(&ret_worker->crp_ret_q); } #ifdef DDB static void db_show_drivers(void) { int hid; db_printf("%12s %4s %8s %2s\n" , "Device" , "Ses" , "Flags" , "QB" ); for (hid = 0; hid < crypto_drivers_size; hid++) { const struct cryptocap *cap = crypto_drivers[hid]; if (cap == NULL) continue; db_printf("%-12s %4u %08x %2u\n" , device_get_nameunit(cap->cc_dev) , cap->cc_sessions , cap->cc_flags , cap->cc_qblocked ); } } DB_SHOW_COMMAND(crypto, db_show_crypto) { struct cryptop *crp; struct crypto_ret_worker *ret_worker; db_show_drivers(); db_printf("\n"); db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", "Device", "Callback"); TAILQ_FOREACH(crp, &crp_q, crp_next) { db_printf("%4u %08x %4u %4u %04x %8p %8p\n" , crp->crp_session->cap->cc_hid , (int) crypto_ses2caps(crp->crp_session) , crp->crp_olen , crp->crp_etype , crp->crp_flags , device_get_nameunit(crp->crp_session->cap->cc_dev) , crp->crp_callback ); } FOREACH_CRYPTO_RETW(ret_worker) { db_printf("\n%8s %4s %4s %4s %8s\n", "ret_worker", "HID", "Etype", "Flags", "Callback"); if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) { TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) { db_printf("%8td %4u %4u %04x %8p\n" , CRYPTO_RETW_ID(ret_worker) , crp->crp_session->cap->cc_hid , crp->crp_etype , crp->crp_flags , crp->crp_callback ); } } } } #endif int crypto_modevent(module_t mod, int type, void *unused); /* * Initialization code, both for static and dynamic loading. * Note this is not invoked with the usual MODULE_DECLARE * mechanism but instead is listed as a dependency by the * cryptosoft driver. This guarantees proper ordering of * calls on module load/unload. */ int crypto_modevent(module_t mod, int type, void *unused) { int error = EINVAL; switch (type) { case MOD_LOAD: error = crypto_init(); if (error == 0 && bootverbose) printf("crypto: \n"); break; case MOD_UNLOAD: /*XXX disallow if active sessions */ error = 0; crypto_destroy(); return 0; } return error; } MODULE_VERSION(crypto, 1); MODULE_DEPEND(crypto, zlib, 1, 1, 1); diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c index d179dd6348e5..6bbcf977ac72 100644 --- a/sys/opencrypto/cryptodev.c +++ b/sys/opencrypto/cryptodev.c @@ -1,1392 +1,1392 @@ /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ /*- * Copyright (c) 2001 Theo de Raadt * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Effort sponsored in part by the Defense Advanced Research Projects * Agency (DARPA) and Air Force Research Laboratory, Air Force * Materiel Command, USAF, under agreement number F30602-01-2-0537. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include SDT_PROVIDER_DECLARE(opencrypto); SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); #ifdef COMPAT_FREEBSD12 /* * Previously, most ioctls were performed against a cloned descriptor * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed * against /dev/crypto directly. */ #define CRIOGET _IOWR('c', 100, uint32_t) #endif /* the following are done against the cloned descriptor */ #ifdef COMPAT_FREEBSD32 #include #include struct session_op32 { uint32_t cipher; uint32_t mac; uint32_t keylen; uint32_t key; int mackeylen; uint32_t mackey; uint32_t ses; }; struct session2_op32 { uint32_t cipher; uint32_t mac; uint32_t keylen; uint32_t key; int mackeylen; uint32_t mackey; uint32_t ses; int crid; int pad[4]; }; struct crypt_op32 { uint32_t ses; uint16_t op; uint16_t flags; u_int len; uint32_t src, dst; uint32_t mac; uint32_t iv; }; struct crypt_aead32 { uint32_t ses; uint16_t op; uint16_t flags; u_int len; u_int aadlen; u_int ivlen; uint32_t src; uint32_t dst; uint32_t aad; uint32_t tag; uint32_t iv; }; #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) static void session_op_from_32(const struct session_op32 *from, struct session2_op *to) { memset(to, 0, sizeof(*to)); CP(*from, *to, cipher); CP(*from, *to, mac); CP(*from, *to, keylen); PTRIN_CP(*from, *to, key); CP(*from, *to, mackeylen); PTRIN_CP(*from, *to, mackey); CP(*from, *to, ses); to->crid = CRYPTOCAP_F_HARDWARE; } static void session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) { session_op_from_32((const struct session_op32 *)from, to); CP(*from, *to, crid); } static void session_op_to_32(const struct session2_op *from, struct session_op32 *to) { CP(*from, *to, cipher); CP(*from, *to, mac); CP(*from, *to, keylen); PTROUT_CP(*from, *to, key); CP(*from, *to, mackeylen); PTROUT_CP(*from, *to, mackey); CP(*from, *to, ses); } static void session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) { session_op_to_32(from, (struct session_op32 *)to); CP(*from, *to, crid); } static void crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) { CP(*from, *to, ses); CP(*from, *to, op); CP(*from, *to, flags); CP(*from, *to, len); PTRIN_CP(*from, *to, src); PTRIN_CP(*from, *to, dst); PTRIN_CP(*from, *to, mac); PTRIN_CP(*from, *to, iv); } static void crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) { CP(*from, *to, ses); CP(*from, *to, op); CP(*from, *to, flags); CP(*from, *to, len); PTROUT_CP(*from, *to, src); PTROUT_CP(*from, *to, dst); PTROUT_CP(*from, *to, mac); PTROUT_CP(*from, *to, iv); } static void crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) { CP(*from, *to, ses); CP(*from, *to, op); CP(*from, *to, flags); CP(*from, *to, len); CP(*from, *to, aadlen); CP(*from, *to, ivlen); PTRIN_CP(*from, *to, src); PTRIN_CP(*from, *to, dst); PTRIN_CP(*from, *to, aad); PTRIN_CP(*from, *to, tag); PTRIN_CP(*from, *to, iv); } static void crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) { CP(*from, *to, ses); CP(*from, *to, op); CP(*from, *to, flags); CP(*from, *to, len); CP(*from, *to, aadlen); CP(*from, *to, ivlen); PTROUT_CP(*from, *to, src); PTROUT_CP(*from, *to, dst); PTROUT_CP(*from, *to, aad); PTROUT_CP(*from, *to, tag); PTROUT_CP(*from, *to, iv); } #endif static void session2_op_from_op(const struct session_op *from, struct session2_op *to) { memset(to, 0, sizeof(*to)); memcpy(to, from, sizeof(*from)); to->crid = CRYPTOCAP_F_HARDWARE; } static void session2_op_to_op(const struct session2_op *from, struct session_op *to) { memcpy(to, from, sizeof(*to)); } struct csession { TAILQ_ENTRY(csession) next; crypto_session_t cses; volatile u_int refs; uint32_t ses; struct mtx lock; /* for op submission */ - struct enc_xform *txform; + const struct enc_xform *txform; int hashsize; int ivsize; int mode; void *key; void *mackey; }; struct cryptop_data { struct csession *cse; char *buf; char *obuf; char *aad; bool done; }; struct fcrypt { TAILQ_HEAD(csessionlist, csession) csessions; int sesn; struct mtx lock; }; static bool use_outputbuffers; SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, &use_outputbuffers, 0, "Use separate output buffers for /dev/crypto requests."); static bool use_separate_aad; SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, &use_separate_aad, 0, "Use separate AAD buffer for /dev/crypto requests."); /* * Check a crypto identifier to see if it requested * a software device/driver. This can be done either * by device name/class or through search constraints. */ static int checkforsoftware(int *cridp) { int crid; crid = *cridp; if (!crypto_devallowsoft) { if (crid & CRYPTOCAP_F_SOFTWARE) { if (crid & CRYPTOCAP_F_HARDWARE) { *cridp = CRYPTOCAP_F_HARDWARE; return 0; } return EINVAL; } if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) return EINVAL; } return 0; } static int cse_create(struct fcrypt *fcr, struct session2_op *sop) { struct crypto_session_params csp; struct csession *cse; - struct enc_xform *txform; - struct auth_hash *thash; + const struct enc_xform *txform; + const struct auth_hash *thash; void *key = NULL; void *mackey = NULL; crypto_session_t cses; int crid, error; switch (sop->cipher) { case 0: txform = NULL; break; case CRYPTO_AES_CBC: txform = &enc_xform_rijndael128; break; case CRYPTO_AES_XTS: txform = &enc_xform_aes_xts; break; case CRYPTO_NULL_CBC: txform = &enc_xform_null; break; case CRYPTO_CAMELLIA_CBC: txform = &enc_xform_camellia; break; case CRYPTO_AES_ICM: txform = &enc_xform_aes_icm; break; case CRYPTO_AES_NIST_GCM_16: txform = &enc_xform_aes_nist_gcm; break; case CRYPTO_CHACHA20: txform = &enc_xform_chacha20; break; case CRYPTO_AES_CCM_16: txform = &enc_xform_ccm; break; case CRYPTO_CHACHA20_POLY1305: txform = &enc_xform_chacha20_poly1305; break; default: CRYPTDEB("invalid cipher"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } switch (sop->mac) { case 0: thash = NULL; break; case CRYPTO_POLY1305: thash = &auth_hash_poly1305; break; case CRYPTO_SHA1_HMAC: thash = &auth_hash_hmac_sha1; break; case CRYPTO_SHA2_224_HMAC: thash = &auth_hash_hmac_sha2_224; break; case CRYPTO_SHA2_256_HMAC: thash = &auth_hash_hmac_sha2_256; break; case CRYPTO_SHA2_384_HMAC: thash = &auth_hash_hmac_sha2_384; break; case CRYPTO_SHA2_512_HMAC: thash = &auth_hash_hmac_sha2_512; break; case CRYPTO_RIPEMD160_HMAC: thash = &auth_hash_hmac_ripemd_160; break; #ifdef COMPAT_FREEBSD12 case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: /* Should always be paired with GCM. */ if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { CRYPTDEB("GMAC without GCM"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } break; #endif case CRYPTO_AES_NIST_GMAC: switch (sop->mackeylen * 8) { case 128: thash = &auth_hash_nist_gmac_aes_128; break; case 192: thash = &auth_hash_nist_gmac_aes_192; break; case 256: thash = &auth_hash_nist_gmac_aes_256; break; default: CRYPTDEB("invalid GMAC key length"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } break; case CRYPTO_AES_CCM_CBC_MAC: switch (sop->mackeylen) { case 16: thash = &auth_hash_ccm_cbc_mac_128; break; case 24: thash = &auth_hash_ccm_cbc_mac_192; break; case 32: thash = &auth_hash_ccm_cbc_mac_256; break; default: CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } break; case CRYPTO_SHA1: thash = &auth_hash_sha1; break; case CRYPTO_SHA2_224: thash = &auth_hash_sha2_224; break; case CRYPTO_SHA2_256: thash = &auth_hash_sha2_256; break; case CRYPTO_SHA2_384: thash = &auth_hash_sha2_384; break; case CRYPTO_SHA2_512: thash = &auth_hash_sha2_512; break; case CRYPTO_NULL_HMAC: thash = &auth_hash_null; break; case CRYPTO_BLAKE2B: thash = &auth_hash_blake2b; break; case CRYPTO_BLAKE2S: thash = &auth_hash_blake2s; break; default: CRYPTDEB("invalid mac"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } if (txform == NULL && thash == NULL) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } memset(&csp, 0, sizeof(csp)); if (use_outputbuffers) csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { switch (sop->mac) { #ifdef COMPAT_FREEBSD12 case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: if (sop->keylen != sop->mackeylen) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } break; #endif case 0: break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } csp.csp_mode = CSP_MODE_AEAD; } else if (sop->cipher == CRYPTO_AES_CCM_16) { switch (sop->mac) { #ifdef COMPAT_FREEBSD12 case CRYPTO_AES_CCM_CBC_MAC: if (sop->keylen != sop->mackeylen) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } thash = NULL; break; #endif case 0: break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } csp.csp_mode = CSP_MODE_AEAD; } else if (sop->cipher == CRYPTO_CHACHA20_POLY1305) { if (sop->mac != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } csp.csp_mode = CSP_MODE_AEAD; } else if (txform != NULL && thash != NULL) csp.csp_mode = CSP_MODE_ETA; else if (txform != NULL) csp.csp_mode = CSP_MODE_CIPHER; else csp.csp_mode = CSP_MODE_DIGEST; switch (csp.csp_mode) { case CSP_MODE_AEAD: case CSP_MODE_ETA: if (use_separate_aad) csp.csp_flags |= CSP_F_SEPARATE_AAD; break; } if (txform != NULL) { csp.csp_cipher_alg = txform->type; csp.csp_cipher_klen = sop->keylen; if (sop->keylen > txform->maxkey || sop->keylen < txform->minkey) { CRYPTDEB("invalid cipher parameters"); error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); error = copyin(sop->key, key, csp.csp_cipher_klen); if (error) { CRYPTDEB("invalid key"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } csp.csp_cipher_key = key; csp.csp_ivlen = txform->ivsize; } if (thash != NULL) { csp.csp_auth_alg = thash->type; csp.csp_auth_klen = sop->mackeylen; if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { CRYPTDEB("invalid mac key length"); error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } if (csp.csp_auth_klen != 0) { mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK); error = copyin(sop->mackey, mackey, csp.csp_auth_klen); if (error) { CRYPTDEB("invalid mac key"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } csp.csp_auth_key = mackey; } if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) csp.csp_ivlen = AES_GCM_IV_LEN; if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) csp.csp_ivlen = AES_CCM_IV_LEN; } crid = sop->crid; error = checkforsoftware(&crid); if (error) { CRYPTDEB("checkforsoftware"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } error = crypto_newsession(&cses, &csp, crid); if (error) { CRYPTDEB("crypto_newsession"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO); mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); refcount_init(&cse->refs, 1); cse->key = key; cse->mackey = mackey; cse->mode = csp.csp_mode; cse->cses = cses; cse->txform = txform; if (thash != NULL) cse->hashsize = thash->hashsize; else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) cse->hashsize = AES_GMAC_HASH_LEN; else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16) cse->hashsize = AES_CBC_MAC_HASH_LEN; else if (csp.csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) cse->hashsize = POLY1305_HASH_LEN; cse->ivsize = csp.csp_ivlen; mtx_lock(&fcr->lock); TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); cse->ses = fcr->sesn++; mtx_unlock(&fcr->lock); sop->ses = cse->ses; /* return hardware/driver id */ sop->crid = crypto_ses2hid(cse->cses); bail: if (error) { free(key, M_XDATA); free(mackey, M_XDATA); } return (error); } static struct csession * cse_find(struct fcrypt *fcr, u_int ses) { struct csession *cse; mtx_lock(&fcr->lock); TAILQ_FOREACH(cse, &fcr->csessions, next) { if (cse->ses == ses) { refcount_acquire(&cse->refs); mtx_unlock(&fcr->lock); return (cse); } } mtx_unlock(&fcr->lock); return (NULL); } static void cse_free(struct csession *cse) { if (!refcount_release(&cse->refs)) return; crypto_freesession(cse->cses); mtx_destroy(&cse->lock); if (cse->key) free(cse->key, M_XDATA); if (cse->mackey) free(cse->mackey, M_XDATA); free(cse, M_XDATA); } static bool cse_delete(struct fcrypt *fcr, u_int ses) { struct csession *cse; mtx_lock(&fcr->lock); TAILQ_FOREACH(cse, &fcr->csessions, next) { if (cse->ses == ses) { TAILQ_REMOVE(&fcr->csessions, cse, next); mtx_unlock(&fcr->lock); cse_free(cse); return (true); } } mtx_unlock(&fcr->lock); return (false); } static struct cryptop_data * cod_alloc(struct csession *cse, size_t aad_len, size_t len) { struct cryptop_data *cod; cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); cod->cse = cse; if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { if (aad_len != 0) cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); cod->buf = malloc(len, M_XDATA, M_WAITOK); } else cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) cod->obuf = malloc(len, M_XDATA, M_WAITOK); return (cod); } static void cod_free(struct cryptop_data *cod) { free(cod->aad, M_XDATA); free(cod->obuf, M_XDATA); free(cod->buf, M_XDATA); free(cod, M_XDATA); } static int cryptodev_cb(struct cryptop *crp) { struct cryptop_data *cod = crp->crp_opaque; /* * Lock to ensure the wakeup() is not missed by the loops * waiting on cod->done in cryptodev_op() and * cryptodev_aead(). */ mtx_lock(&cod->cse->lock); cod->done = true; mtx_unlock(&cod->cse->lock); wakeup(cod); return (0); } static int cryptodev_op(struct csession *cse, const struct crypt_op *cop) { struct cryptop_data *cod = NULL; struct cryptop *crp = NULL; char *dst; int error; if (cop->len > 256*1024-4) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (E2BIG); } if (cse->txform) { if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } } if (cop->mac && cse->hashsize == 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } /* * The COP_F_CIPHER_FIRST flag predates explicit session * modes, but the only way it was used was for EtA so allow it * as long as it is consistent with EtA. */ if (cop->flags & COP_F_CIPHER_FIRST) { if (cop->op != COP_ENCRYPT) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } } cod = cod_alloc(cse, 0, cop->len + cse->hashsize); dst = cop->dst; crp = crypto_getreq(cse->cses, M_WAITOK); error = copyin(cop->src, cod->buf, cop->len); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } crp->crp_payload_start = 0; crp->crp_payload_length = cop->len; if (cse->hashsize) crp->crp_digest_start = cop->len; switch (cse->mode) { case CSP_MODE_COMPRESS: switch (cop->op) { case COP_ENCRYPT: crp->crp_op = CRYPTO_OP_COMPRESS; break; case COP_DECRYPT: crp->crp_op = CRYPTO_OP_DECOMPRESS; break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } break; case CSP_MODE_CIPHER: switch (cop->op) { case COP_ENCRYPT: crp->crp_op = CRYPTO_OP_ENCRYPT; break; case COP_DECRYPT: crp->crp_op = CRYPTO_OP_DECRYPT; break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } break; case CSP_MODE_DIGEST: switch (cop->op) { case 0: case COP_ENCRYPT: case COP_DECRYPT: crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; if (cod->obuf != NULL) crp->crp_digest_start = 0; break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } break; case CSP_MODE_ETA: switch (cop->op) { case COP_ENCRYPT: crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; break; case COP_DECRYPT: crp->crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); if (cod->obuf) crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); crp->crp_callback = cryptodev_cb; crp->crp_opaque = cod; if (cop->iv) { if (cse->ivsize == 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } error = copyin(cop->iv, crp->crp_iv, cse->ivsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } crp->crp_flags |= CRYPTO_F_IV_SEPARATE; } else if (cse->ivsize != 0) { if (crp->crp_payload_length < cse->ivsize) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } crp->crp_iv_start = 0; crp->crp_payload_start += cse->ivsize; crp->crp_payload_length -= cse->ivsize; dst += cse->ivsize; } if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { error = copyin(cop->mac, cod->buf + crp->crp_digest_start, cse->hashsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } again: /* * Let the dispatch run unlocked, then, interlock against the * callback before checking if the operation completed and going * to sleep. This insures drivers don't inherit our lock which * results in a lock order reversal between crypto_dispatch forced * entry and the crypto_done callback into us. */ error = crypto_dispatch(crp); if (error != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } mtx_lock(&cse->lock); while (!cod->done) mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); mtx_unlock(&cse->lock); if (crp->crp_etype == EAGAIN) { crp->crp_etype = 0; crp->crp_flags &= ~CRYPTO_F_DONE; cod->done = false; goto again; } if (crp->crp_etype != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = crp->crp_etype; goto bail; } if (cop->dst != NULL) { error = copyout(cod->obuf != NULL ? cod->obuf : cod->buf + crp->crp_payload_start, dst, crp->crp_payload_length); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + crp->crp_digest_start, cop->mac, cse->hashsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } bail: crypto_freereq(crp); cod_free(cod); return (error); } static int cryptodev_aead(struct csession *cse, struct crypt_aead *caead) { struct cryptop_data *cod = NULL; struct cryptop *crp = NULL; char *dst; int error; if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (E2BIG); } if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || (caead->len % cse->txform->blocksize) != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } /* * The COP_F_CIPHER_FIRST flag predates explicit session * modes, but the only way it was used was for EtA so allow it * as long as it is consistent with EtA. */ if (caead->flags & COP_F_CIPHER_FIRST) { if (caead->op != COP_ENCRYPT) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } } cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); dst = caead->dst; crp = crypto_getreq(cse->cses, M_WAITOK); if (cod->aad != NULL) error = copyin(caead->aad, cod->aad, caead->aadlen); else error = copyin(caead->aad, cod->buf, caead->aadlen); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } crp->crp_aad = cod->aad; crp->crp_aad_start = 0; crp->crp_aad_length = caead->aadlen; if (cod->aad != NULL) crp->crp_payload_start = 0; else crp->crp_payload_start = caead->aadlen; error = copyin(caead->src, cod->buf + crp->crp_payload_start, caead->len); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } crp->crp_payload_length = caead->len; if (caead->op == COP_ENCRYPT && cod->obuf != NULL) crp->crp_digest_start = crp->crp_payload_output_start + caead->len; else crp->crp_digest_start = crp->crp_payload_start + caead->len; switch (cse->mode) { case CSP_MODE_AEAD: case CSP_MODE_ETA: switch (caead->op) { case COP_ENCRYPT: crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; break; case COP_DECRYPT: crp->crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } break; default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + cse->hashsize); if (cod->obuf != NULL) crypto_use_output_buf(crp, cod->obuf, caead->len + cse->hashsize); crp->crp_callback = cryptodev_cb; crp->crp_opaque = cod; if (caead->iv) { /* * Permit a 16-byte IV for AES-XTS, but only use the * first 8 bytes as a block number. */ if (cse->mode == CSP_MODE_ETA && caead->ivlen == AES_BLOCK_LEN && cse->ivsize == AES_XTS_IV_LEN) caead->ivlen = AES_XTS_IV_LEN; if (cse->ivsize == 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } if (caead->ivlen != cse->ivsize) { error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } error = copyin(caead->iv, crp->crp_iv, cse->ivsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } crp->crp_flags |= CRYPTO_F_IV_SEPARATE; } else { error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { error = copyin(caead->tag, cod->buf + crp->crp_digest_start, cse->hashsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } again: /* * Let the dispatch run unlocked, then, interlock against the * callback before checking if the operation completed and going * to sleep. This insures drivers don't inherit our lock which * results in a lock order reversal between crypto_dispatch forced * entry and the crypto_done callback into us. */ error = crypto_dispatch(crp); if (error != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } mtx_lock(&cse->lock); while (!cod->done) mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); mtx_unlock(&cse->lock); if (crp->crp_etype == EAGAIN) { crp->crp_etype = 0; crp->crp_flags &= ~CRYPTO_F_DONE; cod->done = false; goto again; } if (crp->crp_etype != 0) { error = crp->crp_etype; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } if (caead->dst != NULL) { error = copyout(cod->obuf != NULL ? cod->obuf : cod->buf + crp->crp_payload_start, dst, crp->crp_payload_length); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + crp->crp_digest_start, caead->tag, cse->hashsize); if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } } bail: crypto_freereq(crp); cod_free(cod); return (error); } static int cryptodev_find(struct crypt_find_op *find) { device_t dev; size_t fnlen = sizeof find->name; if (find->crid != -1) { dev = crypto_find_device_byhid(find->crid); if (dev == NULL) return (ENOENT); strncpy(find->name, device_get_nameunit(dev), fnlen); find->name[fnlen - 1] = '\x0'; } else { find->name[fnlen - 1] = '\x0'; find->crid = crypto_find_driver(find->name); if (find->crid == -1) return (ENOENT); } return (0); } static void fcrypt_dtor(void *data) { struct fcrypt *fcr = data; struct csession *cse; while ((cse = TAILQ_FIRST(&fcr->csessions))) { TAILQ_REMOVE(&fcr->csessions, cse, next); KASSERT(refcount_load(&cse->refs) == 1, ("%s: crypto session %p with %d refs", __func__, cse, refcount_load(&cse->refs))); cse_free(cse); } mtx_destroy(&fcr->lock); free(fcr, M_XDATA); } static int crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) { struct fcrypt *fcr; int error; fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); TAILQ_INIT(&fcr->csessions); mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); error = devfs_set_cdevpriv(fcr, fcrypt_dtor); if (error) fcrypt_dtor(fcr); return (error); } static int crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) { struct fcrypt *fcr; struct csession *cse; struct session2_op *sop; struct crypt_op *cop; struct crypt_aead *caead; uint32_t ses; int error = 0; union { struct session2_op sopc; #ifdef COMPAT_FREEBSD32 struct crypt_op copc; struct crypt_aead aeadc; #endif } thunk; #ifdef COMPAT_FREEBSD32 u_long cmd32; void *data32; cmd32 = 0; data32 = NULL; switch (cmd) { case CIOCGSESSION32: cmd32 = cmd; data32 = data; cmd = CIOCGSESSION; data = (void *)&thunk.sopc; session_op_from_32((struct session_op32 *)data32, &thunk.sopc); break; case CIOCGSESSION232: cmd32 = cmd; data32 = data; cmd = CIOCGSESSION2; data = (void *)&thunk.sopc; session2_op_from_32((struct session2_op32 *)data32, &thunk.sopc); break; case CIOCCRYPT32: cmd32 = cmd; data32 = data; cmd = CIOCCRYPT; data = (void *)&thunk.copc; crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); break; case CIOCCRYPTAEAD32: cmd32 = cmd; data32 = data; cmd = CIOCCRYPTAEAD; data = (void *)&thunk.aeadc; crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); break; } #endif devfs_get_cdevpriv((void **)&fcr); switch (cmd) { #ifdef COMPAT_FREEBSD12 case CRIOGET: /* * NB: This may fail in cases that the old * implementation did not if the current process has * restricted filesystem access (e.g. running in a * jail that does not expose /dev/crypto or in * capability mode). */ error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, O_RDWR, 0); if (error == 0) *(uint32_t *)data = td->td_retval[0]; break; #endif case CIOCGSESSION: case CIOCGSESSION2: if (cmd == CIOCGSESSION) { session2_op_from_op((void *)data, &thunk.sopc); sop = &thunk.sopc; } else sop = (struct session2_op *)data; error = cse_create(fcr, sop); if (cmd == CIOCGSESSION && error == 0) session2_op_to_op(sop, (void *)data); break; case CIOCFSESSION: ses = *(uint32_t *)data; if (!cse_delete(fcr, ses)) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } break; case CIOCCRYPT: cop = (struct crypt_op *)data; cse = cse_find(fcr, cop->ses); if (cse == NULL) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } error = cryptodev_op(cse, cop); cse_free(cse); break; case CIOCFINDDEV: error = cryptodev_find((struct crypt_find_op *)data); break; case CIOCCRYPTAEAD: caead = (struct crypt_aead *)data; cse = cse_find(fcr, caead->ses); if (cse == NULL) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } error = cryptodev_aead(cse, caead); cse_free(cse); break; default: error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); break; } #ifdef COMPAT_FREEBSD32 switch (cmd32) { case CIOCGSESSION32: if (error == 0) session_op_to_32((void *)data, data32); break; case CIOCGSESSION232: if (error == 0) session2_op_to_32((void *)data, data32); break; case CIOCCRYPT32: if (error == 0) crypt_op_to_32((void *)data, data32); break; case CIOCCRYPTAEAD32: if (error == 0) crypt_aead_to_32((void *)data, data32); break; } #endif return (error); } static struct cdevsw crypto_cdevsw = { .d_version = D_VERSION, .d_open = crypto_open, .d_ioctl = crypto_ioctl, .d_name = "crypto", }; static struct cdev *crypto_dev; /* * Initialization code, both for static and dynamic loading. */ static int cryptodev_modevent(module_t mod, int type, void *unused) { switch (type) { case MOD_LOAD: if (bootverbose) printf("crypto: \n"); crypto_dev = make_dev(&crypto_cdevsw, 0, UID_ROOT, GID_WHEEL, 0666, "crypto"); return 0; case MOD_UNLOAD: /*XXX disallow if active sessions */ destroy_dev(crypto_dev); return 0; } return EINVAL; } static moduledata_t cryptodev_mod = { "cryptodev", cryptodev_modevent, 0 }; MODULE_VERSION(cryptodev, 1); DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h index ce5bfefaed9d..79dec8c44f51 100644 --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -1,685 +1,685 @@ /* $FreeBSD$ */ /* $OpenBSD: cryptodev.h,v 1.31 2002/06/11 11:14:29 beck Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000 Angelos D. Keromytis * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. * * Copyright (c) 2001 Theo de Raadt * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Effort sponsored in part by the Defense Advanced Research Projects * Agency (DARPA) and Air Force Research Laboratory, Air Force * Materiel Command, USAF, under agreement number F30602-01-2-0537. * */ #ifndef _CRYPTO_CRYPTO_H_ #define _CRYPTO_CRYPTO_H_ #include #ifdef _KERNEL #include #include #endif /* Some initial values */ #define CRYPTO_DRIVERS_INITIAL 4 /* Hash values */ #define NULL_HASH_LEN 16 #define SHA1_HASH_LEN 20 #define RIPEMD160_HASH_LEN 20 #define SHA2_224_HASH_LEN 28 #define SHA2_256_HASH_LEN 32 #define SHA2_384_HASH_LEN 48 #define SHA2_512_HASH_LEN 64 #define AES_GMAC_HASH_LEN 16 #define POLY1305_HASH_LEN 16 #define AES_CBC_MAC_HASH_LEN 16 /* Maximum hash algorithm result length */ #define HASH_MAX_LEN SHA2_512_HASH_LEN /* Keep this updated */ #define SHA1_BLOCK_LEN 64 #define RIPEMD160_BLOCK_LEN 64 #define SHA2_224_BLOCK_LEN 64 #define SHA2_256_BLOCK_LEN 64 #define SHA2_384_BLOCK_LEN 128 #define SHA2_512_BLOCK_LEN 128 /* HMAC values */ #define NULL_HMAC_BLOCK_LEN 64 /* Maximum HMAC block length */ #define HMAC_MAX_BLOCK_LEN SHA2_512_BLOCK_LEN /* Keep this updated */ #define HMAC_IPAD_VAL 0x36 #define HMAC_OPAD_VAL 0x5C /* HMAC Key Length */ #define AES_128_GMAC_KEY_LEN 16 #define AES_192_GMAC_KEY_LEN 24 #define AES_256_GMAC_KEY_LEN 32 #define AES_128_CBC_MAC_KEY_LEN 16 #define AES_192_CBC_MAC_KEY_LEN 24 #define AES_256_CBC_MAC_KEY_LEN 32 #define POLY1305_KEY_LEN 32 /* Encryption algorithm block sizes */ #define NULL_BLOCK_LEN 4 /* IPsec to maintain alignment */ #define RIJNDAEL128_BLOCK_LEN 16 #define AES_BLOCK_LEN 16 #define AES_ICM_BLOCK_LEN 1 #define CAMELLIA_BLOCK_LEN 16 #define CHACHA20_NATIVE_BLOCK_LEN 64 #define EALG_MAX_BLOCK_LEN CHACHA20_NATIVE_BLOCK_LEN /* Keep this updated */ /* IV Lengths */ #define AES_GCM_IV_LEN 12 #define AES_CCM_IV_LEN 12 #define AES_XTS_IV_LEN 8 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ #define CHACHA20_POLY1305_IV_LEN 12 /* Min and Max Encryption Key Sizes */ #define NULL_MIN_KEY 0 #define NULL_MAX_KEY 256 /* 2048 bits, max key */ #define RIJNDAEL_MIN_KEY 16 #define RIJNDAEL_MAX_KEY 32 #define AES_MIN_KEY RIJNDAEL_MIN_KEY #define AES_MAX_KEY RIJNDAEL_MAX_KEY #define AES_XTS_MIN_KEY (2 * AES_MIN_KEY) #define AES_XTS_MAX_KEY (2 * AES_MAX_KEY) #define CAMELLIA_MIN_KEY 16 #define CAMELLIA_MAX_KEY 32 #define CHACHA20_POLY1305_KEY 32 /* Maximum hash algorithm result length */ #define AALG_MAX_RESULT_LEN 64 /* Keep this updated */ #define CRYPTO_ALGORITHM_MIN 1 #define CRYPTO_DES_CBC 1 #define CRYPTO_3DES_CBC 2 #define CRYPTO_BLF_CBC 3 #define CRYPTO_CAST_CBC 4 #define CRYPTO_SKIPJACK_CBC 5 #define CRYPTO_MD5_HMAC 6 #define CRYPTO_SHA1_HMAC 7 #define CRYPTO_RIPEMD160_HMAC 8 #define CRYPTO_MD5_KPDK 9 #define CRYPTO_SHA1_KPDK 10 #define CRYPTO_RIJNDAEL128_CBC 11 /* 128 bit blocksize */ #define CRYPTO_AES_CBC 11 /* 128 bit blocksize -- the same as above */ #define CRYPTO_ARC4 12 #define CRYPTO_MD5 13 #define CRYPTO_SHA1 14 #define CRYPTO_NULL_HMAC 15 #define CRYPTO_NULL_CBC 16 #define CRYPTO_DEFLATE_COMP 17 /* Deflate compression algorithm */ #define CRYPTO_SHA2_256_HMAC 18 #define CRYPTO_SHA2_384_HMAC 19 #define CRYPTO_SHA2_512_HMAC 20 #define CRYPTO_CAMELLIA_CBC 21 #define CRYPTO_AES_XTS 22 #define CRYPTO_AES_ICM 23 /* commonly known as CTR mode */ #define CRYPTO_AES_NIST_GMAC 24 /* GMAC only */ #define CRYPTO_AES_NIST_GCM_16 25 /* 16 byte ICV */ #ifdef _KERNEL #define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */ #define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */ #define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */ #endif #define CRYPTO_BLAKE2B 29 /* Blake2b hash */ #define CRYPTO_BLAKE2S 30 /* Blake2s hash */ #define CRYPTO_CHACHA20 31 /* Chacha20 stream cipher */ #define CRYPTO_SHA2_224_HMAC 32 #define CRYPTO_RIPEMD160 33 #define CRYPTO_SHA2_224 34 #define CRYPTO_SHA2_256 35 #define CRYPTO_SHA2_384 36 #define CRYPTO_SHA2_512 37 #define CRYPTO_POLY1305 38 #define CRYPTO_AES_CCM_CBC_MAC 39 /* auth side */ #define CRYPTO_AES_CCM_16 40 /* cipher side */ #define CRYPTO_CHACHA20_POLY1305 41 /* combined AEAD cipher per RFC 8439 */ #define CRYPTO_ALGORITHM_MAX 41 /* Keep updated - see below */ #define CRYPTO_ALGO_VALID(x) ((x) >= CRYPTO_ALGORITHM_MIN && \ (x) <= CRYPTO_ALGORITHM_MAX) /* * Crypto driver/device flags. They can set in the crid * parameter when creating a session or submitting a key * op to affect the device/driver assigned. If neither * of these are specified then the crid is assumed to hold * the driver id of an existing (and suitable) device that * must be used to satisfy the request. */ #define CRYPTO_FLAG_HARDWARE 0x01000000 /* hardware accelerated */ #define CRYPTO_FLAG_SOFTWARE 0x02000000 /* software implementation */ /* Does the kernel support vmpage buffers on this platform? */ #ifdef __powerpc__ #define CRYPTO_MAY_HAVE_VMPAGE 1 #else #define CRYPTO_MAY_HAVE_VMPAGE ( PMAP_HAS_DMAP ) #endif /* Does the currently running system support vmpage buffers on this platform? */ #define CRYPTO_HAS_VMPAGE ( PMAP_HAS_DMAP ) /* NB: deprecated */ struct session_op { uint32_t cipher; /* ie. CRYPTO_AES_CBC */ uint32_t mac; /* ie. CRYPTO_SHA2_256_HMAC */ uint32_t keylen; /* cipher key */ const void *key; int mackeylen; /* mac key */ const void *mackey; uint32_t ses; /* returns: session # */ }; /* * session and crypt _op structs are used by userspace programs to interact * with /dev/crypto. Confusingly, the internal kernel interface is named * "cryptop" (no underscore). */ struct session2_op { uint32_t cipher; /* ie. CRYPTO_AES_CBC */ uint32_t mac; /* ie. CRYPTO_SHA2_256_HMAC */ uint32_t keylen; /* cipher key */ const void *key; int mackeylen; /* mac key */ const void *mackey; uint32_t ses; /* returns: session # */ int crid; /* driver id + flags (rw) */ int pad[4]; /* for future expansion */ }; struct crypt_op { uint32_t ses; uint16_t op; /* i.e. COP_ENCRYPT */ #define COP_ENCRYPT 1 #define COP_DECRYPT 2 uint16_t flags; #define COP_F_CIPHER_FIRST 0x0001 /* Cipher before MAC. */ #define COP_F_BATCH 0x0008 /* Batch op if possible */ u_int len; const void *src; /* become iov[] inside kernel */ void *dst; void *mac; /* must be big enough for chosen MAC */ const void *iv; }; /* op and flags the same as crypt_op */ struct crypt_aead { uint32_t ses; uint16_t op; /* i.e. COP_ENCRYPT */ uint16_t flags; u_int len; u_int aadlen; u_int ivlen; const void *src; /* become iov[] inside kernel */ void *dst; const void *aad; /* additional authenticated data */ void *tag; /* must fit for chosen TAG length */ const void *iv; }; /* * Parameters for looking up a crypto driver/device by * device name or by id. The latter are returned for * created sessions (crid) and completed key operations. */ struct crypt_find_op { int crid; /* driver id + flags */ char name[32]; /* device/driver name */ }; #define CIOCGSESSION _IOWR('c', 101, struct session_op) #define CIOCFSESSION _IOW('c', 102, uint32_t) #define CIOCCRYPT _IOWR('c', 103, struct crypt_op) #define CIOCGSESSION2 _IOWR('c', 106, struct session2_op) #define CIOCFINDDEV _IOWR('c', 108, struct crypt_find_op) #define CIOCCRYPTAEAD _IOWR('c', 109, struct crypt_aead) struct cryptostats { uint64_t cs_ops; /* symmetric crypto ops submitted */ uint64_t cs_errs; /* symmetric crypto ops that failed */ uint64_t cs_kops; /* asymetric/key ops submitted */ uint64_t cs_kerrs; /* asymetric/key ops that failed */ uint64_t cs_intrs; /* crypto swi thread activations */ uint64_t cs_rets; /* crypto return thread activations */ uint64_t cs_blocks; /* symmetric op driver block */ uint64_t cs_kblocks; /* symmetric op driver block */ }; #ifdef _KERNEL /* * Return values for cryptodev_probesession methods. */ #define CRYPTODEV_PROBE_HARDWARE (-100) #define CRYPTODEV_PROBE_ACCEL_SOFTWARE (-200) #define CRYPTODEV_PROBE_SOFTWARE (-500) #if 0 #define CRYPTDEB(s, ...) do { \ printf("%s:%d: " s "\n", __FILE__, __LINE__, ## __VA_ARGS__); \ } while (0) #else #define CRYPTDEB(...) do { } while (0) #endif struct crypto_session_params { int csp_mode; /* Type of operations to perform. */ #define CSP_MODE_NONE 0 #define CSP_MODE_COMPRESS 1 /* Compression/decompression. */ #define CSP_MODE_CIPHER 2 /* Encrypt/decrypt. */ #define CSP_MODE_DIGEST 3 /* Compute/verify digest. */ #define CSP_MODE_AEAD 4 /* Combined auth/encryption. */ #define CSP_MODE_ETA 5 /* IPsec style encrypt-then-auth */ int csp_flags; #define CSP_F_SEPARATE_OUTPUT 0x0001 /* Requests can use separate output */ #define CSP_F_SEPARATE_AAD 0x0002 /* Requests can use separate AAD */ #define CSP_F_ESN 0x0004 /* Requests can use seperate ESN field */ int csp_ivlen; /* IV length in bytes. */ int csp_cipher_alg; int csp_cipher_klen; /* Key length in bytes. */ const void *csp_cipher_key; int csp_auth_alg; int csp_auth_klen; /* Key length in bytes. */ const void *csp_auth_key; int csp_auth_mlen; /* Number of digest bytes to use. 0 means all. */ }; enum crypto_buffer_type { CRYPTO_BUF_NONE = 0, CRYPTO_BUF_CONTIG, CRYPTO_BUF_UIO, CRYPTO_BUF_MBUF, CRYPTO_BUF_VMPAGE, CRYPTO_BUF_SINGLE_MBUF, CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF }; /* * Description of a data buffer for a request. Requests can either * have a single buffer that is modified in place or separate input * and output buffers. */ struct crypto_buffer { union { struct { char *cb_buf; int cb_buf_len; }; struct mbuf *cb_mbuf; struct { vm_page_t *cb_vm_page; int cb_vm_page_len; int cb_vm_page_offset; }; struct uio *cb_uio; }; enum crypto_buffer_type cb_type; }; /* * A cursor is used to iterate through a crypto request data buffer. */ struct crypto_buffer_cursor { union { char *cc_buf; struct mbuf *cc_mbuf; struct iovec *cc_iov; vm_page_t *cc_vmpage; }; /* Optional bytes of valid data remaining */ int cc_buf_len; /* * Optional offset within the current buffer segment where * valid data begins */ size_t cc_offset; enum crypto_buffer_type cc_type; }; /* Structure describing complete operation */ struct cryptop { TAILQ_ENTRY(cryptop) crp_next; struct task crp_task; crypto_session_t crp_session; /* Session */ int crp_olen; /* Result total length */ int crp_etype; /* * Error type (zero means no error). * All error codes except EAGAIN * indicate possible data corruption (as in, * the data have been touched). On all * errors, the crp_session may have changed * (reset to a new one), so the caller * should always check and use the new * value on future requests. */ int crp_flags; #define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */ #define CRYPTO_F_DONE 0x0020 /* Operation completed */ #define CRYPTO_F_CBIFSYNC 0x0040 /* Do CBIMM if op is synchronous */ #define CRYPTO_F_ASYNC_ORDERED 0x0100 /* Completions must happen in order */ #define CRYPTO_F_IV_SEPARATE 0x0200 /* Use crp_iv[] as IV. */ int crp_op; struct crypto_buffer crp_buf; struct crypto_buffer crp_obuf; void *crp_aad; /* AAD buffer. */ int crp_aad_start; /* Location of AAD. */ int crp_aad_length; /* 0 => no AAD. */ uint8_t crp_esn[4]; /* high-order ESN */ int crp_iv_start; /* Location of IV. IV length is from * the session. */ int crp_payload_start; /* Location of ciphertext. */ int crp_payload_output_start; int crp_payload_length; int crp_digest_start; /* Location of MAC/tag. Length is * from the session. */ uint8_t crp_iv[EALG_MAX_BLOCK_LEN]; /* IV if IV_SEPARATE. */ const void *crp_cipher_key; /* New cipher key if non-NULL. */ const void *crp_auth_key; /* New auth key if non-NULL. */ void *crp_opaque; /* Opaque pointer, passed along */ int (*crp_callback)(struct cryptop *); /* Callback function */ struct bintime crp_tstamp; /* performance time stamp */ uint32_t crp_seq; /* used for ordered dispatch */ uint32_t crp_retw_id; /* * the return worker to be used, * used for ordered dispatch */ }; TAILQ_HEAD(cryptopq, cryptop); static __inline void _crypto_use_buf(struct crypto_buffer *cb, void *buf, int len) { cb->cb_buf = buf; cb->cb_buf_len = len; cb->cb_type = CRYPTO_BUF_CONTIG; } static __inline void _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m) { cb->cb_mbuf = m; cb->cb_type = CRYPTO_BUF_MBUF; } static __inline void _crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m) { cb->cb_mbuf = m; cb->cb_type = CRYPTO_BUF_SINGLE_MBUF; } static __inline void _crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len, int offset) { cb->cb_vm_page = pages; cb->cb_vm_page_len = len; cb->cb_vm_page_offset = offset; cb->cb_type = CRYPTO_BUF_VMPAGE; } static __inline void _crypto_use_uio(struct crypto_buffer *cb, struct uio *uio) { cb->cb_uio = uio; cb->cb_type = CRYPTO_BUF_UIO; } static __inline void crypto_use_buf(struct cryptop *crp, void *buf, int len) { _crypto_use_buf(&crp->crp_buf, buf, len); } static __inline void crypto_use_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_mbuf(&crp->crp_buf, m); } static __inline void crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_single_mbuf(&crp->crp_buf, m); } static __inline void crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) { _crypto_use_vmpage(&crp->crp_buf, pages, len, offset); } static __inline void crypto_use_uio(struct cryptop *crp, struct uio *uio) { _crypto_use_uio(&crp->crp_buf, uio); } static __inline void crypto_use_output_buf(struct cryptop *crp, void *buf, int len) { _crypto_use_buf(&crp->crp_obuf, buf, len); } static __inline void crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_mbuf(&crp->crp_obuf, m); } static __inline void crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_single_mbuf(&crp->crp_obuf, m); } static __inline void crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) { _crypto_use_vmpage(&crp->crp_obuf, pages, len, offset); } static __inline void crypto_use_output_uio(struct cryptop *crp, struct uio *uio) { _crypto_use_uio(&crp->crp_obuf, uio); } #define CRYPTO_HAS_OUTPUT_BUFFER(crp) \ ((crp)->crp_obuf.cb_type != CRYPTO_BUF_NONE) /* Flags in crp_op. */ #define CRYPTO_OP_DECRYPT 0x0 #define CRYPTO_OP_ENCRYPT 0x1 #define CRYPTO_OP_IS_ENCRYPT(op) ((op) & CRYPTO_OP_ENCRYPT) #define CRYPTO_OP_COMPUTE_DIGEST 0x0 #define CRYPTO_OP_VERIFY_DIGEST 0x2 #define CRYPTO_OP_DECOMPRESS CRYPTO_OP_DECRYPT #define CRYPTO_OP_COMPRESS CRYPTO_OP_ENCRYPT #define CRYPTO_OP_IS_COMPRESS(op) ((op) & CRYPTO_OP_COMPRESS) /* * Hints passed to process methods. */ #define CRYPTO_HINT_MORE 0x1 /* more ops coming shortly */ uint32_t crypto_ses2hid(crypto_session_t crypto_session); uint32_t crypto_ses2caps(crypto_session_t crypto_session); void *crypto_get_driver_session(crypto_session_t crypto_session); const struct crypto_session_params *crypto_get_params( crypto_session_t crypto_session); -struct auth_hash *crypto_auth_hash(const struct crypto_session_params *csp); -struct enc_xform *crypto_cipher(const struct crypto_session_params *csp); +const struct auth_hash *crypto_auth_hash(const struct crypto_session_params *csp); +const struct enc_xform *crypto_cipher(const struct crypto_session_params *csp); MALLOC_DECLARE(M_CRYPTO_DATA); extern int crypto_newsession(crypto_session_t *cses, const struct crypto_session_params *params, int hard); extern void crypto_freesession(crypto_session_t cses); #define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE #define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE #define CRYPTOCAP_F_SYNC 0x04000000 /* operates synchronously */ #define CRYPTOCAP_F_ACCEL_SOFTWARE 0x08000000 #define CRYPTO_SESS_SYNC(sess) \ ((crypto_ses2caps(sess) & CRYPTOCAP_F_SYNC) != 0) extern int32_t crypto_get_driverid(device_t dev, size_t session_size, int flags); extern int crypto_find_driver(const char *); extern device_t crypto_find_device_byhid(int hid); extern int crypto_getcaps(int hid); extern int crypto_unregister_all(uint32_t driverid); extern int crypto_dispatch(struct cryptop *crp); #define CRYPTO_ASYNC_ORDERED 0x1 /* complete in order dispatched */ extern int crypto_dispatch_async(struct cryptop *crp, int flags); extern void crypto_dispatch_batch(struct cryptopq *crpq, int flags); #define CRYPTO_SYMQ 0x1 extern int crypto_unblock(uint32_t, int); extern void crypto_done(struct cryptop *crp); extern void crypto_destroyreq(struct cryptop *crp); extern void crypto_initreq(struct cryptop *crp, crypto_session_t cses); extern void crypto_freereq(struct cryptop *crp); extern struct cryptop *crypto_getreq(crypto_session_t cses, int how); extern int crypto_usercrypto; /* userland may do crypto requests */ extern int crypto_devallowsoft; /* only use hardware crypto */ #ifdef SYSCTL_DECL SYSCTL_DECL(_kern_crypto); #endif /* Helper routines for drivers to initialize auth contexts for HMAC. */ struct auth_hash; void hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx); void hmac_init_opad(const struct auth_hash *axf, const char *key, int klen, void *auth_ctx); /* * Crypto-related utility routines used mainly by drivers. * * Similar to m_copyback/data, *_copyback copy data from the 'src' * buffer into the crypto request's data buffer while *_copydata copy * data from the crypto request's data buffer into the the 'dst' * buffer. */ void crypto_copyback(struct cryptop *crp, int off, int size, const void *src); void crypto_copydata(struct cryptop *crp, int off, int size, void *dst); int crypto_apply(struct cryptop *crp, int off, int len, int (*f)(void *, const void *, u_int), void *arg); void *crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len); int crypto_apply_buf(struct crypto_buffer *cb, int off, int len, int (*f)(void *, const void *, u_int), void *arg); void *crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip, size_t len); size_t crypto_buffer_len(struct crypto_buffer *cb); void crypto_cursor_init(struct crypto_buffer_cursor *cc, const struct crypto_buffer *cb); void crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount); void *crypto_cursor_segment(struct crypto_buffer_cursor *cc, size_t *len); void crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size, const void *vsrc); void crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst); void crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size, void *vdst); static __inline void crypto_read_iv(struct cryptop *crp, void *iv) { const struct crypto_session_params *csp; csp = crypto_get_params(crp->crp_session); if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) memcpy(iv, crp->crp_iv, csp->csp_ivlen); else crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv); } #endif /* _KERNEL */ #endif /* _CRYPTO_CRYPTO_H_ */ diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c index 9e551ba9652b..ef927f117bcc 100644 --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -1,1704 +1,1704 @@ /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000, 2001 Angelos D. Keromytis * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" struct swcr_auth { void *sw_ictx; void *sw_octx; - struct auth_hash *sw_axf; + const struct auth_hash *sw_axf; uint16_t sw_mlen; }; struct swcr_encdec { void *sw_kschedule; - struct enc_xform *sw_exf; + const struct enc_xform *sw_exf; }; struct swcr_compdec { - struct comp_algo *sw_cxf; + const struct comp_algo *sw_cxf; }; struct swcr_session { struct mtx swcr_lock; int (*swcr_process)(struct swcr_session *, struct cryptop *); struct swcr_auth swcr_auth; struct swcr_encdec swcr_encdec; struct swcr_compdec swcr_compdec; }; static int32_t swcr_id; static void swcr_freesession(device_t dev, crypto_session_t cses); /* Used for CRYPTO_NULL_CBC. */ static int swcr_null(struct swcr_session *ses, struct cryptop *crp) { return (0); } /* * Apply a symmetric encryption/decryption algorithm. */ static int swcr_encdec(struct swcr_session *ses, struct cryptop *crp) { unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; const struct crypto_session_params *csp; + const struct enc_xform *exf; struct swcr_encdec *sw; - struct enc_xform *exf; size_t inlen, outlen; int i, blks, ivlen, resid; struct crypto_buffer_cursor cc_in, cc_out; const unsigned char *inblk; unsigned char *outblk; int error; bool encrypting; error = 0; sw = &ses->swcr_encdec; exf = sw->sw_exf; ivlen = exf->ivsize; if (exf->native_blocksize == 0) { /* Check for non-padded data */ if ((crp->crp_payload_length % exf->blocksize) != 0) return (EINVAL); blks = exf->blocksize; } else blks = exf->native_blocksize; if (exf == &enc_xform_aes_icm && (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); if (crp->crp_cipher_key != NULL) { csp = crypto_get_params(crp->crp_session); error = exf->setkey(sw->sw_kschedule, crp->crp_cipher_key, csp->csp_cipher_klen); if (error) return (error); } crypto_read_iv(crp, iv); if (exf->reinit) { /* * xforms that provide a reinit method perform all IV * handling themselves. */ exf->reinit(sw->sw_kschedule, iv); } ivp = iv; crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); inblk = crypto_cursor_segment(&cc_in, &inlen); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; outblk = crypto_cursor_segment(&cc_out, &outlen); resid = crp->crp_payload_length; encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); /* * Loop through encrypting blocks. 'inlen' is the remaining * length of the current segment in the input buffer. * 'outlen' is the remaining length of current segment in the * output buffer. */ while (resid >= blks) { /* * If the current block is not contained within the * current input/output segment, use 'blk' as a local * buffer. */ if (inlen < blks) { crypto_cursor_copydata(&cc_in, blks, blk); inblk = blk; } if (outlen < blks) outblk = blk; /* * Ciphers without a 'reinit' hook are assumed to be * used in CBC mode where the chaining is done here. */ if (exf->reinit != NULL) { if (encrypting) exf->encrypt(sw->sw_kschedule, inblk, outblk); else exf->decrypt(sw->sw_kschedule, inblk, outblk); } else if (encrypting) { /* XOR with previous block */ for (i = 0; i < blks; i++) outblk[i] = inblk[i] ^ ivp[i]; exf->encrypt(sw->sw_kschedule, outblk, outblk); /* * Keep encrypted block for XOR'ing * with next block */ memcpy(iv, outblk, blks); ivp = iv; } else { /* decrypt */ /* * Keep encrypted block for XOR'ing * with next block */ nivp = (ivp == iv) ? iv2 : iv; memcpy(nivp, inblk, blks); exf->decrypt(sw->sw_kschedule, inblk, outblk); /* XOR with previous block */ for (i = 0; i < blks; i++) outblk[i] ^= ivp[i]; ivp = nivp; } if (inlen < blks) { inblk = crypto_cursor_segment(&cc_in, &inlen); } else { crypto_cursor_advance(&cc_in, blks); inlen -= blks; inblk += blks; } if (outlen < blks) { crypto_cursor_copyback(&cc_out, blks, blk); outblk = crypto_cursor_segment(&cc_out, &outlen); } else { crypto_cursor_advance(&cc_out, blks); outlen -= blks; outblk += blks; } resid -= blks; } /* Handle trailing partial block for stream ciphers. */ if (resid > 0) { KASSERT(exf->native_blocksize != 0, ("%s: partial block of %d bytes for cipher %s", __func__, i, exf->name)); KASSERT(exf->reinit != NULL, ("%s: partial block cipher %s without reinit hook", __func__, exf->name)); KASSERT(resid < blks, ("%s: partial block too big", __func__)); inblk = crypto_cursor_segment(&cc_in, &inlen); outblk = crypto_cursor_segment(&cc_out, &outlen); if (inlen < resid) { crypto_cursor_copydata(&cc_in, resid, blk); inblk = blk; } if (outlen < resid) outblk = blk; if (encrypting) exf->encrypt_last(sw->sw_kschedule, inblk, outblk, resid); else exf->decrypt_last(sw->sw_kschedule, inblk, outblk, resid); if (outlen < resid) crypto_cursor_copyback(&cc_out, resid, blk); } explicit_bzero(blk, sizeof(blk)); explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv2, sizeof(iv2)); return (0); } static void -swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw, +swcr_authprepare(const struct auth_hash *axf, struct swcr_auth *sw, const uint8_t *key, int klen) { switch (axf->type) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: hmac_init_ipad(axf, key, klen, sw->sw_ictx); hmac_init_opad(axf, key, klen, sw->sw_octx); break; case CRYPTO_POLY1305: case CRYPTO_BLAKE2B: case CRYPTO_BLAKE2S: axf->Setkey(sw->sw_ictx, key, klen); axf->Init(sw->sw_ictx); break; default: panic("%s: algorithm %d doesn't use keys", __func__, axf->type); } } /* * Compute or verify hash. */ static int swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) { u_char aalg[HASH_MAX_LEN]; const struct crypto_session_params *csp; struct swcr_auth *sw; - struct auth_hash *axf; + const struct auth_hash *axf; union authctx ctx; int err; sw = &ses->swcr_auth; axf = sw->sw_axf; csp = crypto_get_params(crp->crp_session); if (crp->crp_auth_key != NULL) { swcr_authprepare(axf, sw, crp->crp_auth_key, csp->csp_auth_klen); } bcopy(sw->sw_ictx, &ctx, axf->ctxsize); if (crp->crp_aad != NULL) err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (err) goto out; if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) err = crypto_apply_buf(&crp->crp_obuf, crp->crp_payload_output_start, crp->crp_payload_length, axf->Update, &ctx); else err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); if (err) goto out; if (csp->csp_flags & CSP_F_ESN) axf->Update(&ctx, crp->crp_esn, 4); axf->Final(aalg, &ctx); if (sw->sw_octx != NULL) { bcopy(sw->sw_octx, &ctx, axf->ctxsize); axf->Update(&ctx, aalg, axf->hashsize); axf->Final(aalg, &ctx); } if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char uaalg[HASH_MAX_LEN]; crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) err = EBADMSG; explicit_bzero(uaalg, sizeof(uaalg)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); } explicit_bzero(aalg, sizeof(aalg)); out: explicit_bzero(&ctx, sizeof(ctx)); return (err); } CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ static int swcr_gmac(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[GMAC_DIGEST_LEN]; u_char iv[AES_BLOCK_LEN]; struct crypto_buffer_cursor cc; const u_char *inblk; union authctx ctx; struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; uint32_t *blkp; size_t len; int blksz, error, ivlen, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = GMAC_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); /* Initialize the IV */ ivlen = AES_GCM_IV_LEN; crypto_read_iv(crp, iv); axf->Reinit(&ctx, iv, ivlen); crypto_cursor_init(&cc, &crp->crp_buf); crypto_cursor_advance(&cc, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { inblk = crypto_cursor_segment(&cc, &len); if (len >= blksz) { len = rounddown(MIN(len, resid), blksz); crypto_cursor_advance(&cc, len); } else { len = blksz; crypto_cursor_copydata(&cc, len, blk); inblk = blk; } axf->Update(&ctx, inblk, len); } if (resid > 0) { memset(blk, 0, blksz); crypto_cursor_copydata(&cc, resid, blk); axf->Update(&ctx, blk, blksz); } /* length block */ memset(blk, 0, blksz); blkp = (uint32_t *)blk + 1; *blkp = htobe32(crp->crp_payload_length * 8); axf->Update(&ctx, blk, blksz); /* Finalize MAC */ axf->Final(tag, &ctx); error = 0; if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char tag2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) error = EBADMSG; explicit_bzero(tag2, sizeof(tag2)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_gcm(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[GMAC_DIGEST_LEN]; u_char iv[AES_BLOCK_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; - struct auth_hash *axf; - struct enc_xform *exf; + const struct auth_hash *axf; + const struct enc_xform *exf; uint32_t *blkp; size_t len; int blksz, error, ivlen, r, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = GMAC_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); swe = &ses->swcr_encdec; exf = swe->sw_exf; KASSERT(axf->blocksize == exf->native_blocksize, ("%s: blocksize mismatch", __func__)); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* Initialize the IV */ ivlen = AES_GCM_IV_LEN; bcopy(crp->crp_iv, iv, ivlen); /* Supply MAC with IV */ axf->Reinit(&ctx, iv, ivlen); /* Supply MAC with AAD */ if (crp->crp_aad != NULL) { len = rounddown(crp->crp_aad_length, blksz); if (len != 0) axf->Update(&ctx, crp->crp_aad, len); if (crp->crp_aad_length != len) { memset(blk, 0, blksz); memcpy(blk, (char *)crp->crp_aad + len, crp->crp_aad_length - len); axf->Update(&ctx, blk, blksz); } } else { crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_aad_start); for (resid = crp->crp_aad_length; resid >= blksz; resid -= len) { inblk = crypto_cursor_segment(&cc_in, &len); if (len >= blksz) { len = rounddown(MIN(len, resid), blksz); crypto_cursor_advance(&cc_in, len); } else { len = blksz; crypto_cursor_copydata(&cc_in, len, blk); inblk = blk; } axf->Update(&ctx, inblk, len); } if (resid > 0) { memset(blk, 0, blksz); crypto_cursor_copydata(&cc_in, resid, blk); axf->Update(&ctx, blk, blksz); } } if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, crypto_get_params(crp->crp_session)->csp_cipher_klen); exf->reinit(swe->sw_kschedule, iv); /* Do encryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else { crypto_cursor_advance(&cc_in, blksz); } if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->encrypt(swe->sw_kschedule, inblk, outblk); axf->Update(&ctx, outblk, blksz); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { axf->Update(&ctx, inblk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } axf->Update(&ctx, blk, resid); } /* length block */ memset(blk, 0, blksz); blkp = (uint32_t *)blk + 1; *blkp = htobe32(crp->crp_aad_length * 8); blkp = (uint32_t *)blk + 3; *blkp = htobe32(crp->crp_payload_length * 8); axf->Update(&ctx, blk, blksz); /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[GMAC_DIGEST_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) { u_char tag[AES_CBC_MAC_HASH_LEN]; u_char iv[AES_BLOCK_LEN]; union authctx ctx; struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; int error, ivlen; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); /* Initialize the IV */ ivlen = AES_CCM_IV_LEN; crypto_read_iv(crp, iv); /* * AES CCM-CBC-MAC needs to know the length of both the auth * data and payload data before doing the auth computation. */ ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length; ctx.aes_cbc_mac_ctx.cryptDataLength = 0; axf->Reinit(&ctx, iv, ivlen); if (crp->crp_aad != NULL) error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else error = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, axf->Update, &ctx); if (error) return (error); /* Finalize MAC */ axf->Final(tag, &ctx); if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { u_char tag2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) error = EBADMSG; explicit_bzero(tag2, sizeof(tag)); } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } explicit_bzero(tag, sizeof(tag)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_ccm(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[AES_CBC_MAC_HASH_LEN]; u_char iv[AES_BLOCK_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; - struct auth_hash *axf; - struct enc_xform *exf; + const struct auth_hash *axf; + const struct enc_xform *exf; size_t len; int blksz, error, ivlen, r, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; bcopy(swa->sw_ictx, &ctx, axf->ctxsize); blksz = AES_BLOCK_LEN; KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", __func__)); swe = &ses->swcr_encdec; exf = swe->sw_exf; KASSERT(axf->blocksize == exf->native_blocksize, ("%s: blocksize mismatch", __func__)); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); /* Initialize the IV */ ivlen = AES_CCM_IV_LEN; bcopy(crp->crp_iv, iv, ivlen); /* * AES CCM-CBC-MAC needs to know the length of both the auth * data and payload data before doing the auth computation. */ ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; /* Supply MAC with IV */ axf->Reinit(&ctx, iv, ivlen); /* Supply MAC with AAD */ if (crp->crp_aad != NULL) error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (error) return (error); if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, crypto_get_params(crp->crp_session)->csp_cipher_klen); exf->reinit(swe->sw_kschedule, iv); /* Do encryption/decryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; axf->Update(&ctx, inblk, blksz); exf->encrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { /* * One of the problems with CCM+CBC is that * the authentication is done on the * unencrypted data. As a result, we have to * decrypt the data twice: once to generate * the tag and a second time after the tag is * verified. */ exf->decrypt(swe->sw_kschedule, inblk, blk); axf->Update(&ctx, blk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { axf->Update(&ctx, blk, resid); exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } else { exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); axf->Update(&ctx, blk, resid); } } /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[AES_CBC_MAC_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ exf->reinit(swe->sw_kschedule, iv); crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(iv, sizeof(iv)); return (error); } static int swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) { const struct crypto_session_params *csp; uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; u_char *blk = (u_char *)blkbuf; u_char tag[POLY1305_HASH_LEN]; struct crypto_buffer_cursor cc_in, cc_out; const u_char *inblk; u_char *outblk; uint64_t *blkp; union authctx ctx; struct swcr_auth *swa; struct swcr_encdec *swe; - struct auth_hash *axf; - struct enc_xform *exf; + const struct auth_hash *axf; + const struct enc_xform *exf; size_t len; int blksz, error, r, resid; swa = &ses->swcr_auth; axf = swa->sw_axf; swe = &ses->swcr_encdec; exf = swe->sw_exf; blksz = exf->native_blocksize; KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); csp = crypto_get_params(crp->crp_session); /* Generate Poly1305 key. */ if (crp->crp_cipher_key != NULL) axf->Setkey(&ctx, crp->crp_cipher_key, csp->csp_cipher_klen); else axf->Setkey(&ctx, csp->csp_cipher_key, csp->csp_cipher_klen); axf->Reinit(&ctx, crp->crp_iv, csp->csp_ivlen); /* Supply MAC with AAD */ if (crp->crp_aad != NULL) axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); else crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, axf->Update, &ctx); if (crp->crp_aad_length % 16 != 0) { /* padding1 */ memset(blk, 0, 16); axf->Update(&ctx, blk, 16 - crp->crp_aad_length % 16); } if (crp->crp_cipher_key != NULL) exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, csp->csp_cipher_klen); exf->reinit(swe->sw_kschedule, crp->crp_iv); /* Do encryption with MAC */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { crypto_cursor_init(&cc_out, &crp->crp_obuf); crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); } else cc_out = cc_in; for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->encrypt(swe->sw_kschedule, inblk, outblk); axf->Update(&ctx, outblk, blksz); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } else { axf->Update(&ctx, inblk, blksz); } } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } axf->Update(&ctx, blk, resid); if (resid % 16 != 0) { /* padding2 */ memset(blk, 0, 16); axf->Update(&ctx, blk, 16 - resid % 16); } } /* lengths */ blkp = (uint64_t *)blk; blkp[0] = htole64(crp->crp_aad_length); blkp[1] = htole64(crp->crp_payload_length); axf->Update(&ctx, blk, sizeof(uint64_t) * 2); /* Finalize MAC */ axf->Final(tag, &ctx); /* Validate tag */ error = 0; if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { u_char tag2[POLY1305_HASH_LEN]; crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); explicit_bzero(tag2, sizeof(tag2)); if (r != 0) { error = EBADMSG; goto out; } /* tag matches, decrypt data */ crypto_cursor_init(&cc_in, &crp->crp_buf); crypto_cursor_advance(&cc_in, crp->crp_payload_start); for (resid = crp->crp_payload_length; resid > blksz; resid -= blksz) { inblk = crypto_cursor_segment(&cc_in, &len); if (len < blksz) { crypto_cursor_copydata(&cc_in, blksz, blk); inblk = blk; } else crypto_cursor_advance(&cc_in, blksz); outblk = crypto_cursor_segment(&cc_out, &len); if (len < blksz) outblk = blk; exf->decrypt(swe->sw_kschedule, inblk, outblk); if (outblk == blk) crypto_cursor_copyback(&cc_out, blksz, blk); else crypto_cursor_advance(&cc_out, blksz); } if (resid > 0) { crypto_cursor_copydata(&cc_in, resid, blk); exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); crypto_cursor_copyback(&cc_out, resid, blk); } } else { /* Inject the authentication data */ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); } out: explicit_bzero(blkbuf, sizeof(blkbuf)); explicit_bzero(tag, sizeof(tag)); explicit_bzero(&ctx, sizeof(ctx)); return (error); } /* * Apply a cipher and a digest to perform EtA. */ static int swcr_eta(struct swcr_session *ses, struct cryptop *crp) { int error; if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { error = swcr_encdec(ses, crp); if (error == 0) error = swcr_authcompute(ses, crp); } else { error = swcr_authcompute(ses, crp); if (error == 0) error = swcr_encdec(ses, crp); } return (error); } /* * Apply a compression/decompression algorithm */ static int swcr_compdec(struct swcr_session *ses, struct cryptop *crp) { + const struct comp_algo *cxf; uint8_t *data, *out; - struct comp_algo *cxf; int adj; uint32_t result; cxf = ses->swcr_compdec.sw_cxf; /* We must handle the whole buffer of data in one time * then if there is not all the data in the mbuf, we must * copy in a buffer. */ data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); if (data == NULL) return (EINVAL); crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, data); if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) result = cxf->compress(data, crp->crp_payload_length, &out); else result = cxf->decompress(data, crp->crp_payload_length, &out); free(data, M_CRYPTO_DATA); if (result == 0) return (EINVAL); crp->crp_olen = result; /* Check the compressed size when doing compression */ if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { if (result >= crp->crp_payload_length) { /* Compression was useless, we lost time */ free(out, M_CRYPTO_DATA); return (0); } } /* Copy back the (de)compressed data. m_copyback is * extending the mbuf as necessary. */ crypto_copyback(crp, crp->crp_payload_start, result, out); if (result < crp->crp_payload_length) { switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: case CRYPTO_BUF_SINGLE_MBUF: adj = result - crp->crp_payload_length; m_adj(crp->crp_buf.cb_mbuf, adj); break; case CRYPTO_BUF_UIO: { struct uio *uio = crp->crp_buf.cb_uio; int ind; adj = crp->crp_payload_length - result; ind = uio->uio_iovcnt - 1; while (adj > 0 && ind >= 0) { if (adj < uio->uio_iov[ind].iov_len) { uio->uio_iov[ind].iov_len -= adj; break; } adj -= uio->uio_iov[ind].iov_len; uio->uio_iov[ind].iov_len = 0; ind--; uio->uio_iovcnt--; } } break; case CRYPTO_BUF_VMPAGE: adj = crp->crp_payload_length - result; crp->crp_buf.cb_vm_page_len -= adj; break; default: break; } } free(out, M_CRYPTO_DATA); return 0; } static int swcr_setup_cipher(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_encdec *swe; - struct enc_xform *txf; + const struct enc_xform *txf; int error; swe = &ses->swcr_encdec; txf = crypto_cipher(csp); MPASS(txf->ivsize == csp->csp_ivlen); if (txf->ctxsize != 0) { swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swe->sw_kschedule == NULL) return (ENOMEM); } if (csp->csp_cipher_key != NULL) { error = txf->setkey(swe->sw_kschedule, csp->csp_cipher_key, csp->csp_cipher_klen); if (error) return (error); } swe->sw_exf = txf; return (0); } static int swcr_setup_auth(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; swa = &ses->swcr_auth; axf = crypto_auth_hash(csp); swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_octx == NULL) return (ENOBUFS); if (csp->csp_auth_key != NULL) { swcr_authprepare(axf, swa, csp->csp_auth_key, csp->csp_auth_klen); } if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: axf->Init(swa->sw_ictx); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_AES_NIST_GMAC: axf->Init(swa->sw_ictx); axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_gmac; break; case CRYPTO_POLY1305: case CRYPTO_BLAKE2B: case CRYPTO_BLAKE2S: /* * Blake2b and Blake2s support an optional key but do * not require one. */ if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); axf->Init(swa->sw_ictx); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_authcompute; break; case CRYPTO_AES_CCM_CBC_MAC: axf->Init(swa->sw_ictx); axf->Setkey(swa->sw_ictx, csp->csp_auth_key, csp->csp_auth_klen); if (csp->csp_mode == CSP_MODE_DIGEST) ses->swcr_process = swcr_ccm_cbc_mac; break; } return (0); } static int swcr_setup_gcm(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; if (csp->csp_ivlen != AES_GCM_IV_LEN) return (EINVAL); /* First, setup the auth side. */ swa = &ses->swcr_auth; switch (csp->csp_cipher_klen * 8) { case 128: axf = &auth_hash_nist_gmac_aes_128; break; case 192: axf = &auth_hash_nist_gmac_aes_192; break; case 256: axf = &auth_hash_nist_gmac_aes_256; break; default: return (EINVAL); } swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); axf->Init(swa->sw_ictx); if (csp->csp_cipher_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, csp->csp_cipher_klen); /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static int swcr_setup_ccm(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; if (csp->csp_ivlen != AES_CCM_IV_LEN) return (EINVAL); /* First, setup the auth side. */ swa = &ses->swcr_auth; switch (csp->csp_cipher_klen * 8) { case 128: axf = &auth_hash_ccm_cbc_mac_128; break; case 192: axf = &auth_hash_ccm_cbc_mac_192; break; case 256: axf = &auth_hash_ccm_cbc_mac_256; break; default: return (EINVAL); } swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if (swa->sw_ictx == NULL) return (ENOBUFS); axf->Init(swa->sw_ictx); if (csp->csp_cipher_key != NULL) axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, csp->csp_cipher_klen); /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static int swcr_setup_chacha20_poly1305(struct swcr_session *ses, const struct crypto_session_params *csp) { struct swcr_auth *swa; - struct auth_hash *axf; + const struct auth_hash *axf; if (csp->csp_ivlen != CHACHA20_POLY1305_IV_LEN) return (EINVAL); /* First, setup the auth side. */ swa = &ses->swcr_auth; axf = &auth_hash_chacha20_poly1305; swa->sw_axf = axf; if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) return (EINVAL); if (csp->csp_auth_mlen == 0) swa->sw_mlen = axf->hashsize; else swa->sw_mlen = csp->csp_auth_mlen; /* The auth state is regenerated for each nonce. */ /* Second, setup the cipher side. */ return (swcr_setup_cipher(ses, csp)); } static bool swcr_auth_supported(const struct crypto_session_params *csp) { - struct auth_hash *axf; + const struct auth_hash *axf; axf = crypto_auth_hash(csp); if (axf == NULL) return (false); switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: break; case CRYPTO_AES_NIST_GMAC: switch (csp->csp_auth_klen * 8) { case 128: case 192: case 256: break; default: return (false); } if (csp->csp_auth_key == NULL) return (false); if (csp->csp_ivlen != AES_GCM_IV_LEN) return (false); break; case CRYPTO_POLY1305: if (csp->csp_auth_klen != POLY1305_KEY_LEN) return (false); break; case CRYPTO_AES_CCM_CBC_MAC: switch (csp->csp_auth_klen * 8) { case 128: case 192: case 256: break; default: return (false); } if (csp->csp_auth_key == NULL) return (false); if (csp->csp_ivlen != AES_CCM_IV_LEN) return (false); break; } return (true); } static bool swcr_cipher_supported(const struct crypto_session_params *csp) { - struct enc_xform *txf; + const struct enc_xform *txf; txf = crypto_cipher(csp); if (txf == NULL) return (false); if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && txf->ivsize != csp->csp_ivlen) return (false); return (true); } #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) static int swcr_probesession(device_t dev, const struct crypto_session_params *csp) { if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (EINVAL); switch (csp->csp_mode) { case CSP_MODE_COMPRESS: switch (csp->csp_cipher_alg) { case CRYPTO_DEFLATE_COMP: break; default: return (EINVAL); } break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: return (EINVAL); default: if (!swcr_cipher_supported(csp)) return (EINVAL); break; } break; case CSP_MODE_DIGEST: if (!swcr_auth_supported(csp)) return (EINVAL); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: break; default: return (EINVAL); } break; case CSP_MODE_ETA: /* AEAD algorithms cannot be used for EtA. */ switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: return (EINVAL); } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: case CRYPTO_AES_CCM_CBC_MAC: return (EINVAL); } if (!swcr_cipher_supported(csp) || !swcr_auth_supported(csp)) return (EINVAL); break; default: return (EINVAL); } return (CRYPTODEV_PROBE_SOFTWARE); } /* * Generate a new software session. */ static int swcr_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct swcr_session *ses; struct swcr_encdec *swe; struct swcr_auth *swa; - struct comp_algo *cxf; + const struct comp_algo *cxf; int error; ses = crypto_get_driver_session(cses); mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); error = 0; swe = &ses->swcr_encdec; swa = &ses->swcr_auth; switch (csp->csp_mode) { case CSP_MODE_COMPRESS: switch (csp->csp_cipher_alg) { case CRYPTO_DEFLATE_COMP: cxf = &comp_algo_deflate; break; #ifdef INVARIANTS default: panic("bad compression algo"); #endif } ses->swcr_compdec.sw_cxf = cxf; ses->swcr_process = swcr_compdec; break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_NULL_CBC: ses->swcr_process = swcr_null; break; #ifdef INVARIANTS case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: panic("bad cipher algo"); #endif default: error = swcr_setup_cipher(ses, csp); if (error == 0) ses->swcr_process = swcr_encdec; } break; case CSP_MODE_DIGEST: error = swcr_setup_auth(ses, csp); break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: error = swcr_setup_gcm(ses, csp); if (error == 0) ses->swcr_process = swcr_gcm; break; case CRYPTO_AES_CCM_16: error = swcr_setup_ccm(ses, csp); if (error == 0) ses->swcr_process = swcr_ccm; break; case CRYPTO_CHACHA20_POLY1305: error = swcr_setup_chacha20_poly1305(ses, csp); if (error == 0) ses->swcr_process = swcr_chacha20_poly1305; break; #ifdef INVARIANTS default: panic("bad aead algo"); #endif } break; case CSP_MODE_ETA: #ifdef INVARIANTS switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: case CRYPTO_CHACHA20_POLY1305: panic("bad eta cipher algo"); } switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: case CRYPTO_AES_CCM_CBC_MAC: panic("bad eta auth algo"); } #endif error = swcr_setup_auth(ses, csp); if (error) break; if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { /* Effectively degrade to digest mode. */ ses->swcr_process = swcr_authcompute; break; } error = swcr_setup_cipher(ses, csp); if (error == 0) ses->swcr_process = swcr_eta; break; default: error = EINVAL; } if (error) swcr_freesession(dev, cses); return (error); } static void swcr_freesession(device_t dev, crypto_session_t cses) { struct swcr_session *ses; ses = crypto_get_driver_session(cses); mtx_destroy(&ses->swcr_lock); zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); } /* * Process a software request. */ static int swcr_process(device_t dev, struct cryptop *crp, int hint) { struct swcr_session *ses; ses = crypto_get_driver_session(crp->crp_session); mtx_lock(&ses->swcr_lock); crp->crp_etype = ses->swcr_process(ses, crp); mtx_unlock(&ses->swcr_lock); crypto_done(crp); return (0); } static void swcr_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "cryptosoft", -1) == NULL && BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) panic("cryptosoft: could not attach"); } static int swcr_probe(device_t dev) { device_set_desc(dev, "software crypto"); device_quiet(dev); return (BUS_PROBE_NOWILDCARD); } static int swcr_attach(device_t dev) { swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); if (swcr_id < 0) { device_printf(dev, "cannot initialize!"); return (ENXIO); } return (0); } static int swcr_detach(device_t dev) { crypto_unregister_all(swcr_id); return 0; } static device_method_t swcr_methods[] = { DEVMETHOD(device_identify, swcr_identify), DEVMETHOD(device_probe, swcr_probe), DEVMETHOD(device_attach, swcr_attach), DEVMETHOD(device_detach, swcr_detach), DEVMETHOD(cryptodev_probesession, swcr_probesession), DEVMETHOD(cryptodev_newsession, swcr_newsession), DEVMETHOD(cryptodev_freesession,swcr_freesession), DEVMETHOD(cryptodev_process, swcr_process), {0, 0}, }; static driver_t swcr_driver = { "cryptosoft", swcr_methods, 0, /* NB: no softc */ }; static devclass_t swcr_devclass; /* * NB: We explicitly reference the crypto module so we * get the necessary ordering when built as a loadable * module. This is required because we bundle the crypto * module code together with the cryptosoft driver (otherwise * normal module dependencies would handle things). */ extern int crypto_modevent(struct module *, int, void *); /* XXX where to attach */ DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); MODULE_VERSION(cryptosoft, 1); MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); diff --git a/sys/opencrypto/xform_aes_icm.c b/sys/opencrypto/xform_aes_icm.c index 5f81f8df8a87..618b812ceebf 100644 --- a/sys/opencrypto/xform_aes_icm.c +++ b/sys/opencrypto/xform_aes_icm.c @@ -1,194 +1,194 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include static int aes_icm_setkey(void *, const uint8_t *, int); static void aes_icm_crypt(void *, const uint8_t *, uint8_t *); static void aes_icm_crypt_last(void *, const uint8_t *, uint8_t *, size_t); static void aes_icm_reinit(void *, const uint8_t *); static void aes_gcm_reinit(void *, const uint8_t *); static void aes_ccm_reinit(void *, const uint8_t *); /* Encryption instances */ -struct enc_xform enc_xform_aes_icm = { +const struct enc_xform enc_xform_aes_icm = { .type = CRYPTO_AES_ICM, .name = "AES-ICM", .ctxsize = sizeof(struct aes_icm_ctx), .blocksize = 1, .native_blocksize = AES_BLOCK_LEN, .ivsize = AES_BLOCK_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, .encrypt = aes_icm_crypt, .decrypt = aes_icm_crypt, .setkey = aes_icm_setkey, .reinit = aes_icm_reinit, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, }; -struct enc_xform enc_xform_aes_nist_gcm = { +const struct enc_xform enc_xform_aes_nist_gcm = { .type = CRYPTO_AES_NIST_GCM_16, .name = "AES-GCM", .ctxsize = sizeof(struct aes_icm_ctx), .blocksize = 1, .native_blocksize = AES_BLOCK_LEN, .ivsize = AES_GCM_IV_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, .encrypt = aes_icm_crypt, .decrypt = aes_icm_crypt, .setkey = aes_icm_setkey, .reinit = aes_gcm_reinit, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, }; -struct enc_xform enc_xform_ccm = { +const struct enc_xform enc_xform_ccm = { .type = CRYPTO_AES_CCM_16, .name = "AES-CCM", .ctxsize = sizeof(struct aes_icm_ctx), .blocksize = 1, .native_blocksize = AES_BLOCK_LEN, .ivsize = AES_CCM_IV_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, .encrypt = aes_icm_crypt, .decrypt = aes_icm_crypt, .setkey = aes_icm_setkey, .reinit = aes_ccm_reinit, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, }; /* * Encryption wrapper routines. */ static void aes_icm_reinit(void *key, const uint8_t *iv) { struct aes_icm_ctx *ctx; ctx = key; bcopy(iv, ctx->ac_block, AESICM_BLOCKSIZE); } static void aes_gcm_reinit(void *key, const uint8_t *iv) { struct aes_icm_ctx *ctx; aes_icm_reinit(key, iv); ctx = key; /* GCM starts with 2 as counter 1 is used for final xor of tag. */ bzero(&ctx->ac_block[AESICM_BLOCKSIZE - 4], 4); ctx->ac_block[AESICM_BLOCKSIZE - 1] = 2; } static void aes_ccm_reinit(void *key, const uint8_t *iv) { struct aes_icm_ctx *ctx; ctx = key; /* CCM has flags, then the IV, then the counter, which starts at 1 */ bzero(ctx->ac_block, sizeof(ctx->ac_block)); /* 3 bytes for length field; this gives a nonce of 12 bytes */ ctx->ac_block[0] = (15 - AES_CCM_IV_LEN) - 1; bcopy(iv, ctx->ac_block+1, AES_CCM_IV_LEN); ctx->ac_block[AESICM_BLOCKSIZE - 1] = 1; } static void aes_icm_crypt(void *key, const uint8_t *in, uint8_t *out) { struct aes_icm_ctx *ctx; int i; ctx = key; aes_icm_crypt_last(key, in, out, AESICM_BLOCKSIZE); /* increment counter */ for (i = AESICM_BLOCKSIZE - 1; i >= 0; i--) if (++ctx->ac_block[i]) /* continue on overflow */ break; } static void aes_icm_crypt_last(void *key, const uint8_t *in, uint8_t *out, size_t len) { struct aes_icm_ctx *ctx; uint8_t keystream[AESICM_BLOCKSIZE]; int i; ctx = key; rijndaelEncrypt(ctx->ac_ek, ctx->ac_nr, ctx->ac_block, keystream); for (i = 0; i < len; i++) out[i] = in[i] ^ keystream[i]; explicit_bzero(keystream, sizeof(keystream)); } static int aes_icm_setkey(void *sched, const uint8_t *key, int len) { struct aes_icm_ctx *ctx; if (len != 16 && len != 24 && len != 32) return (EINVAL); ctx = sched; ctx->ac_nr = rijndaelKeySetupEnc(ctx->ac_ek, key, len * 8); return (0); } diff --git a/sys/opencrypto/xform_aes_xts.c b/sys/opencrypto/xform_aes_xts.c index 0b415f1d6346..457535621511 100644 --- a/sys/opencrypto/xform_aes_xts.c +++ b/sys/opencrypto/xform_aes_xts.c @@ -1,157 +1,157 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static int aes_xts_setkey(void *, const uint8_t *, int); static void aes_xts_encrypt(void *, const uint8_t *, uint8_t *); static void aes_xts_decrypt(void *, const uint8_t *, uint8_t *); static void aes_xts_reinit(void *, const uint8_t *); /* Encryption instances */ -struct enc_xform enc_xform_aes_xts = { +const struct enc_xform enc_xform_aes_xts = { .type = CRYPTO_AES_XTS, .name = "AES-XTS", .ctxsize = sizeof(struct aes_xts_ctx), .blocksize = AES_BLOCK_LEN, .ivsize = AES_XTS_IV_LEN, .minkey = AES_XTS_MIN_KEY, .maxkey = AES_XTS_MAX_KEY, .encrypt = aes_xts_encrypt, .decrypt = aes_xts_decrypt, .setkey = aes_xts_setkey, .reinit = aes_xts_reinit }; /* * Encryption wrapper routines. */ static void aes_xts_reinit(void *key, const uint8_t *iv) { struct aes_xts_ctx *ctx = key; uint64_t blocknum; u_int i; /* * Prepare tweak as E_k2(IV). IV is specified as LE representation * of a 64-bit block number which we allow to be passed in directly. */ bcopy(iv, &blocknum, AES_XTS_IVSIZE); for (i = 0; i < AES_XTS_IVSIZE; i++) { ctx->tweak[i] = blocknum & 0xff; blocknum >>= 8; } /* Last 64 bits of IV are always zero */ bzero(ctx->tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); rijndael_encrypt(&ctx->key2, ctx->tweak, ctx->tweak); } static void aes_xts_crypt(struct aes_xts_ctx *ctx, const uint8_t *in, uint8_t *out, u_int do_encrypt) { uint8_t block[AES_XTS_BLOCKSIZE]; u_int i, carry_in, carry_out; for (i = 0; i < AES_XTS_BLOCKSIZE; i++) block[i] = in[i] ^ ctx->tweak[i]; if (do_encrypt) rijndael_encrypt(&ctx->key1, block, out); else rijndael_decrypt(&ctx->key1, block, out); for (i = 0; i < AES_XTS_BLOCKSIZE; i++) out[i] ^= ctx->tweak[i]; /* Exponentiate tweak */ carry_in = 0; for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { carry_out = ctx->tweak[i] & 0x80; ctx->tweak[i] = (ctx->tweak[i] << 1) | (carry_in ? 1 : 0); carry_in = carry_out; } if (carry_in) ctx->tweak[0] ^= AES_XTS_ALPHA; explicit_bzero(block, sizeof(block)); } static void aes_xts_encrypt(void *key, const uint8_t *in, uint8_t *out) { aes_xts_crypt(key, in, out, 1); } static void aes_xts_decrypt(void *key, const uint8_t *in, uint8_t *out) { aes_xts_crypt(key, in, out, 0); } static int aes_xts_setkey(void *sched, const uint8_t *key, int len) { struct aes_xts_ctx *ctx; if (len != 32 && len != 64) return (EINVAL); ctx = sched; rijndael_set_key(&ctx->key1, key, len * 4); rijndael_set_key(&ctx->key2, key + (len / 2), len * 4); return (0); } diff --git a/sys/opencrypto/xform_auth.h b/sys/opencrypto/xform_auth.h index 6427965671d3..aa2f55564c5f 100644 --- a/sys/opencrypto/xform_auth.h +++ b/sys/opencrypto/xform_auth.h @@ -1,100 +1,100 @@ /* $FreeBSD$ */ /* $OpenBSD: xform.h,v 1.8 2001/08/28 12:20:43 ben Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000 Angelos D. Keromytis * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #ifndef _CRYPTO_XFORM_AUTH_H_ #define _CRYPTO_XFORM_AUTH_H_ #include #include #include #include #include #include #include #include #include #include #include /* XXX use a define common with other hash stuff ! */ #define AH_ALEN_MAX 64 /* max authenticator hash length */ /* Declarations */ struct auth_hash { int type; - char *name; + const char *name; uint16_t keysize; uint16_t hashsize; uint16_t ctxsize; uint16_t blocksize; void (*Init) (void *); void (*Setkey) (void *, const uint8_t *, u_int); void (*Reinit) (void *, const uint8_t *, u_int); int (*Update) (void *, const void *, u_int); void (*Final) (uint8_t *, void *); }; -extern struct auth_hash auth_hash_null; -extern struct auth_hash auth_hash_hmac_sha1; -extern struct auth_hash auth_hash_hmac_ripemd_160; -extern struct auth_hash auth_hash_hmac_sha2_224; -extern struct auth_hash auth_hash_hmac_sha2_256; -extern struct auth_hash auth_hash_hmac_sha2_384; -extern struct auth_hash auth_hash_hmac_sha2_512; -extern struct auth_hash auth_hash_sha1; -extern struct auth_hash auth_hash_sha2_224; -extern struct auth_hash auth_hash_sha2_256; -extern struct auth_hash auth_hash_sha2_384; -extern struct auth_hash auth_hash_sha2_512; -extern struct auth_hash auth_hash_nist_gmac_aes_128; -extern struct auth_hash auth_hash_nist_gmac_aes_192; -extern struct auth_hash auth_hash_nist_gmac_aes_256; -extern struct auth_hash auth_hash_blake2b; -extern struct auth_hash auth_hash_blake2s; -extern struct auth_hash auth_hash_poly1305; -extern struct auth_hash auth_hash_ccm_cbc_mac_128; -extern struct auth_hash auth_hash_ccm_cbc_mac_192; -extern struct auth_hash auth_hash_ccm_cbc_mac_256; -extern struct auth_hash auth_hash_chacha20_poly1305; +extern const struct auth_hash auth_hash_null; +extern const struct auth_hash auth_hash_hmac_sha1; +extern const struct auth_hash auth_hash_hmac_ripemd_160; +extern const struct auth_hash auth_hash_hmac_sha2_224; +extern const struct auth_hash auth_hash_hmac_sha2_256; +extern const struct auth_hash auth_hash_hmac_sha2_384; +extern const struct auth_hash auth_hash_hmac_sha2_512; +extern const struct auth_hash auth_hash_sha1; +extern const struct auth_hash auth_hash_sha2_224; +extern const struct auth_hash auth_hash_sha2_256; +extern const struct auth_hash auth_hash_sha2_384; +extern const struct auth_hash auth_hash_sha2_512; +extern const struct auth_hash auth_hash_nist_gmac_aes_128; +extern const struct auth_hash auth_hash_nist_gmac_aes_192; +extern const struct auth_hash auth_hash_nist_gmac_aes_256; +extern const struct auth_hash auth_hash_blake2b; +extern const struct auth_hash auth_hash_blake2s; +extern const struct auth_hash auth_hash_poly1305; +extern const struct auth_hash auth_hash_ccm_cbc_mac_128; +extern const struct auth_hash auth_hash_ccm_cbc_mac_192; +extern const struct auth_hash auth_hash_ccm_cbc_mac_256; +extern const struct auth_hash auth_hash_chacha20_poly1305; union authctx { SHA1_CTX sha1ctx; RMD160_CTX rmd160ctx; SHA224_CTX sha224ctx; SHA256_CTX sha256ctx; SHA384_CTX sha384ctx; SHA512_CTX sha512ctx; struct aes_gmac_ctx aes_gmac_ctx; struct aes_cbc_mac_ctx aes_cbc_mac_ctx; }; #endif /* _CRYPTO_XFORM_AUTH_H_ */ diff --git a/sys/opencrypto/xform_cbc_mac.c b/sys/opencrypto/xform_cbc_mac.c index 755dd51d9517..d55e66e45255 100644 --- a/sys/opencrypto/xform_cbc_mac.c +++ b/sys/opencrypto/xform_cbc_mac.c @@ -1,46 +1,46 @@ #include __FBSDID("$FreeBSD$"); #include #include /* Authentication instances */ -struct auth_hash auth_hash_ccm_cbc_mac_128 = { +const struct auth_hash auth_hash_ccm_cbc_mac_128 = { .type = CRYPTO_AES_CCM_CBC_MAC, .name = "CBC-CCM-AES-128", .keysize = AES_128_CBC_MAC_KEY_LEN, .hashsize = AES_CBC_MAC_HASH_LEN, .ctxsize = sizeof(struct aes_cbc_mac_ctx), .blocksize = CCM_CBC_BLOCK_LEN, .Init = AES_CBC_MAC_Init, .Setkey = AES_CBC_MAC_Setkey, .Reinit = AES_CBC_MAC_Reinit, .Update = AES_CBC_MAC_Update, .Final = AES_CBC_MAC_Final, }; -struct auth_hash auth_hash_ccm_cbc_mac_192 = { +const struct auth_hash auth_hash_ccm_cbc_mac_192 = { .type = CRYPTO_AES_CCM_CBC_MAC, .name = "CBC-CCM-AES-192", .keysize = AES_192_CBC_MAC_KEY_LEN, .hashsize = AES_CBC_MAC_HASH_LEN, .ctxsize = sizeof(struct aes_cbc_mac_ctx), .blocksize = CCM_CBC_BLOCK_LEN, .Init = AES_CBC_MAC_Init, .Setkey = AES_CBC_MAC_Setkey, .Reinit = AES_CBC_MAC_Reinit, .Update = AES_CBC_MAC_Update, .Final = AES_CBC_MAC_Final, }; -struct auth_hash auth_hash_ccm_cbc_mac_256 = { +const struct auth_hash auth_hash_ccm_cbc_mac_256 = { .type = CRYPTO_AES_CCM_CBC_MAC, .name = "CBC-CCM-AES-256", .keysize = AES_256_CBC_MAC_KEY_LEN, .hashsize = AES_CBC_MAC_HASH_LEN, .ctxsize = sizeof(struct aes_cbc_mac_ctx), .blocksize = CCM_CBC_BLOCK_LEN, .Init = AES_CBC_MAC_Init, .Setkey = AES_CBC_MAC_Setkey, .Reinit = AES_CBC_MAC_Reinit, .Update = AES_CBC_MAC_Update, .Final = AES_CBC_MAC_Final, }; diff --git a/sys/opencrypto/xform_chacha20_poly1305.c b/sys/opencrypto/xform_chacha20_poly1305.c index 3a72c06f931b..543c16bcc4e0 100644 --- a/sys/opencrypto/xform_chacha20_poly1305.c +++ b/sys/opencrypto/xform_chacha20_poly1305.c @@ -1,163 +1,163 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Netflix Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include struct chacha20_poly1305_cipher_ctx { const void *key; uint32_t ic; char nonce[CHACHA20_POLY1305_IV_LEN]; }; static int chacha20_poly1305_setkey(void *vctx, const uint8_t *key, int len) { struct chacha20_poly1305_cipher_ctx *ctx = vctx; if (len != CHACHA20_POLY1305_KEY) return (EINVAL); ctx->key = key; return (0); } static void chacha20_poly1305_reinit(void *vctx, const uint8_t *iv) { struct chacha20_poly1305_cipher_ctx *ctx = vctx; /* Block 0 is used for the poly1305 key. */ memcpy(ctx->nonce, iv, sizeof(ctx->nonce)); ctx->ic = 1; } static void chacha20_poly1305_crypt(void *vctx, const uint8_t *in, uint8_t *out) { struct chacha20_poly1305_cipher_ctx *ctx = vctx; int error; error = crypto_stream_chacha20_ietf_xor_ic(out, in, CHACHA20_NATIVE_BLOCK_LEN, ctx->nonce, ctx->ic, ctx->key); KASSERT(error == 0, ("%s failed: %d", __func__, error)); ctx->ic++; } static void chacha20_poly1305_crypt_last(void *vctx, const uint8_t *in, uint8_t *out, size_t len) { struct chacha20_poly1305_cipher_ctx *ctx = vctx; int error; error = crypto_stream_chacha20_ietf_xor_ic(out, in, len, ctx->nonce, ctx->ic, ctx->key); KASSERT(error == 0, ("%s failed: %d", __func__, error)); } -struct enc_xform enc_xform_chacha20_poly1305 = { +const struct enc_xform enc_xform_chacha20_poly1305 = { .type = CRYPTO_CHACHA20_POLY1305, .name = "ChaCha20-Poly1305", .ctxsize = sizeof(struct chacha20_poly1305_cipher_ctx), .blocksize = 1, .native_blocksize = CHACHA20_NATIVE_BLOCK_LEN, .ivsize = CHACHA20_POLY1305_IV_LEN, .minkey = CHACHA20_POLY1305_KEY, .maxkey = CHACHA20_POLY1305_KEY, .encrypt = chacha20_poly1305_crypt, .decrypt = chacha20_poly1305_crypt, .setkey = chacha20_poly1305_setkey, .reinit = chacha20_poly1305_reinit, .encrypt_last = chacha20_poly1305_crypt_last, .decrypt_last = chacha20_poly1305_crypt_last, }; struct chacha20_poly1305_auth_ctx { struct crypto_onetimeauth_poly1305_state state; const void *key; }; CTASSERT(sizeof(union authctx) >= sizeof(struct chacha20_poly1305_auth_ctx)); static void chacha20_poly1305_Init(void *vctx) { } static void chacha20_poly1305_Setkey(void *vctx, const uint8_t *key, u_int klen) { struct chacha20_poly1305_auth_ctx *ctx = vctx; ctx->key = key; } static void chacha20_poly1305_Reinit(void *vctx, const uint8_t *nonce, u_int noncelen) { struct chacha20_poly1305_auth_ctx *ctx = vctx; char block[CHACHA20_NATIVE_BLOCK_LEN]; crypto_stream_chacha20_ietf(block, sizeof(block), nonce, ctx->key); crypto_onetimeauth_poly1305_init(&ctx->state, block); explicit_bzero(block, sizeof(block)); } static int chacha20_poly1305_Update(void *vctx, const void *data, u_int len) { struct chacha20_poly1305_auth_ctx *ctx = vctx; crypto_onetimeauth_poly1305_update(&ctx->state, data, len); return (0); } static void chacha20_poly1305_Final(uint8_t *digest, void *vctx) { struct chacha20_poly1305_auth_ctx *ctx = vctx; crypto_onetimeauth_poly1305_final(&ctx->state, digest); } -struct auth_hash auth_hash_chacha20_poly1305 = { +const struct auth_hash auth_hash_chacha20_poly1305 = { .type = CRYPTO_POLY1305, .name = "ChaCha20-Poly1305", .keysize = POLY1305_KEY_LEN, .hashsize = POLY1305_HASH_LEN, .ctxsize = sizeof(struct chacha20_poly1305_auth_ctx), .blocksize = crypto_onetimeauth_poly1305_BYTES, .Init = chacha20_poly1305_Init, .Setkey = chacha20_poly1305_Setkey, .Reinit = chacha20_poly1305_Reinit, .Update = chacha20_poly1305_Update, .Final = chacha20_poly1305_Final, }; diff --git a/sys/opencrypto/xform_cml.c b/sys/opencrypto/xform_cml.c index 3f25f2cb6aaf..ac1dabd90d30 100644 --- a/sys/opencrypto/xform_cml.c +++ b/sys/opencrypto/xform_cml.c @@ -1,98 +1,98 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static int cml_setkey(void *, const uint8_t *, int); static void cml_encrypt(void *, const uint8_t *, uint8_t *); static void cml_decrypt(void *, const uint8_t *, uint8_t *); /* Encryption instances */ -struct enc_xform enc_xform_camellia = { +const struct enc_xform enc_xform_camellia = { .type = CRYPTO_CAMELLIA_CBC, .name = "Camellia-CBC", .ctxsize = sizeof(camellia_ctx), .blocksize = CAMELLIA_BLOCK_LEN, .ivsize = CAMELLIA_BLOCK_LEN, .minkey = CAMELLIA_MIN_KEY, .maxkey = CAMELLIA_MAX_KEY, .encrypt = cml_encrypt, .decrypt = cml_decrypt, .setkey = cml_setkey, }; /* * Encryption wrapper routines. */ static void cml_encrypt(void *ctx, const uint8_t *in, uint8_t *out) { camellia_encrypt(ctx, in, out); } static void cml_decrypt(void *ctx, const uint8_t *in, uint8_t *out) { camellia_decrypt(ctx, in, out); } static int cml_setkey(void *ctx, const uint8_t *key, int len) { if (len != 16 && len != 24 && len != 32) return (EINVAL); camellia_set_key(ctx, key, len * 8); return (0); } diff --git a/sys/opencrypto/xform_comp.h b/sys/opencrypto/xform_comp.h index 1d0ba7a505bf..11bf59a94b39 100644 --- a/sys/opencrypto/xform_comp.h +++ b/sys/opencrypto/xform_comp.h @@ -1,51 +1,51 @@ /* $FreeBSD$ */ /* $OpenBSD: xform.h,v 1.8 2001/08/28 12:20:43 ben Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000 Angelos D. Keromytis * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #ifndef _CRYPTO_XFORM_COMP_H_ #define _CRYPTO_XFORM_COMP_H_ #include #include #include #include /* Declarations */ struct comp_algo { int type; char *name; size_t minlen; uint32_t (*compress) (uint8_t *, uint32_t, uint8_t **); uint32_t (*decompress) (uint8_t *, uint32_t, uint8_t **); }; -extern struct comp_algo comp_algo_deflate; +extern const struct comp_algo comp_algo_deflate; #endif /* _CRYPTO_XFORM_COMP_H_ */ diff --git a/sys/opencrypto/xform_deflate.c b/sys/opencrypto/xform_deflate.c index 8d93f4843244..b295ceb2ea9b 100644 --- a/sys/opencrypto/xform_deflate.c +++ b/sys/opencrypto/xform_deflate.c @@ -1,80 +1,80 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static uint32_t deflate_compress(uint8_t *, uint32_t, uint8_t **); static uint32_t deflate_decompress(uint8_t *, uint32_t, uint8_t **); /* Compression instance */ -struct comp_algo comp_algo_deflate = { +const struct comp_algo comp_algo_deflate = { CRYPTO_DEFLATE_COMP, "Deflate", 90, deflate_compress, deflate_decompress }; /* * And compression */ static uint32_t deflate_compress(uint8_t *data, uint32_t size, uint8_t **out) { return deflate_global(data, size, 0, out); } static uint32_t deflate_decompress(uint8_t *data, uint32_t size, uint8_t **out) { return deflate_global(data, size, 1, out); } diff --git a/sys/opencrypto/xform_enc.h b/sys/opencrypto/xform_enc.h index e8325f20917b..6f95b49986c5 100644 --- a/sys/opencrypto/xform_enc.h +++ b/sys/opencrypto/xform_enc.h @@ -1,100 +1,100 @@ /* $FreeBSD$ */ /* $OpenBSD: xform.h,v 1.8 2001/08/28 12:20:43 ben Exp $ */ /*- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) * * This code was written by Angelos D. Keromytis in Athens, Greece, in * February 2000. Network Security Technologies Inc. (NSTI) kindly * supported the development of this code. * * Copyright (c) 2000 Angelos D. Keromytis * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software without fee * is hereby granted, provided that this entire notice is included in * all source code copies of any software which is or includes a copy or * modification of this software. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #ifndef _CRYPTO_XFORM_ENC_H_ #define _CRYPTO_XFORM_ENC_H_ #include #include #include #include #include #ifdef _STANDALONE #include #endif #define AESICM_BLOCKSIZE AES_BLOCK_LEN #define AES_XTS_BLOCKSIZE 16 #define AES_XTS_IVSIZE 8 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ /* Declarations */ struct enc_xform { int type; - char *name; + const char *name; size_t ctxsize; uint16_t blocksize; /* Required input block size -- 1 for stream ciphers. */ uint16_t native_blocksize; /* Used for stream ciphers. */ uint16_t ivsize; uint16_t minkey, maxkey; /* * Encrypt/decrypt a single block. For stream ciphers this * encrypts/decrypts a single "native" block. */ void (*encrypt) (void *, const uint8_t *, uint8_t *); void (*decrypt) (void *, const uint8_t *, uint8_t *); int (*setkey) (void *, const uint8_t *, int len); void (*reinit) (void *, const uint8_t *); /* * For stream ciphers, encrypt/decrypt the final partial block * of 'len' bytes. */ void (*encrypt_last) (void *, const uint8_t *, uint8_t *, size_t len); void (*decrypt_last) (void *, const uint8_t *, uint8_t *, size_t len); }; -extern struct enc_xform enc_xform_null; -extern struct enc_xform enc_xform_rijndael128; -extern struct enc_xform enc_xform_aes_icm; -extern struct enc_xform enc_xform_aes_nist_gcm; -extern struct enc_xform enc_xform_aes_nist_gmac; -extern struct enc_xform enc_xform_aes_xts; -extern struct enc_xform enc_xform_camellia; -extern struct enc_xform enc_xform_chacha20; -extern struct enc_xform enc_xform_chacha20_poly1305; -extern struct enc_xform enc_xform_ccm; +extern const struct enc_xform enc_xform_null; +extern const struct enc_xform enc_xform_rijndael128; +extern const struct enc_xform enc_xform_aes_icm; +extern const struct enc_xform enc_xform_aes_nist_gcm; +extern const struct enc_xform enc_xform_aes_nist_gmac; +extern const struct enc_xform enc_xform_aes_xts; +extern const struct enc_xform enc_xform_camellia; +extern const struct enc_xform enc_xform_chacha20; +extern const struct enc_xform enc_xform_chacha20_poly1305; +extern const struct enc_xform enc_xform_ccm; struct aes_icm_ctx { uint32_t ac_ek[4*(RIJNDAEL_MAXNR + 1)]; /* ac_block is initialized to IV */ uint8_t ac_block[AESICM_BLOCKSIZE]; int ac_nr; }; struct aes_xts_ctx { rijndael_ctx key1; rijndael_ctx key2; uint8_t tweak[AES_XTS_BLOCKSIZE]; }; #endif /* _CRYPTO_XFORM_ENC_H_ */ diff --git a/sys/opencrypto/xform_gmac.c b/sys/opencrypto/xform_gmac.c index 0b981f2c95c3..0cd5ef9b60be 100644 --- a/sys/opencrypto/xform_gmac.c +++ b/sys/opencrypto/xform_gmac.c @@ -1,107 +1,107 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include /* Encryption instances */ -struct enc_xform enc_xform_aes_nist_gmac = { +const struct enc_xform enc_xform_aes_nist_gmac = { .type = CRYPTO_AES_NIST_GMAC, .name = "AES-GMAC", .blocksize = AES_ICM_BLOCK_LEN, .ivsize = AES_GCM_IV_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, }; /* Authentication instances */ -struct auth_hash auth_hash_nist_gmac_aes_128 = { +const struct auth_hash auth_hash_nist_gmac_aes_128 = { .type = CRYPTO_AES_NIST_GMAC, .name = "GMAC-AES-128", .keysize = AES_128_GMAC_KEY_LEN, .hashsize = AES_GMAC_HASH_LEN, .ctxsize = sizeof(struct aes_gmac_ctx), .blocksize = GMAC_BLOCK_LEN, .Init = AES_GMAC_Init, .Setkey = AES_GMAC_Setkey, .Reinit = AES_GMAC_Reinit, .Update = AES_GMAC_Update, .Final = AES_GMAC_Final, }; -struct auth_hash auth_hash_nist_gmac_aes_192 = { +const struct auth_hash auth_hash_nist_gmac_aes_192 = { .type = CRYPTO_AES_NIST_GMAC, .name = "GMAC-AES-192", .keysize = AES_192_GMAC_KEY_LEN, .hashsize = AES_GMAC_HASH_LEN, .ctxsize = sizeof(struct aes_gmac_ctx), .blocksize = GMAC_BLOCK_LEN, .Init = AES_GMAC_Init, .Setkey = AES_GMAC_Setkey, .Reinit = AES_GMAC_Reinit, .Update = AES_GMAC_Update, .Final = AES_GMAC_Final, }; -struct auth_hash auth_hash_nist_gmac_aes_256 = { +const struct auth_hash auth_hash_nist_gmac_aes_256 = { .type = CRYPTO_AES_NIST_GMAC, .name = "GMAC-AES-256", .keysize = AES_256_GMAC_KEY_LEN, .hashsize = AES_GMAC_HASH_LEN, .ctxsize = sizeof(struct aes_gmac_ctx), .blocksize = GMAC_BLOCK_LEN, .Init = AES_GMAC_Init, .Setkey = AES_GMAC_Setkey, .Reinit = AES_GMAC_Reinit, .Update = AES_GMAC_Update, .Final = AES_GMAC_Final, }; diff --git a/sys/opencrypto/xform_null.c b/sys/opencrypto/xform_null.c index d1b79e1385b2..6cd49baab0ac 100644 --- a/sys/opencrypto/xform_null.c +++ b/sys/opencrypto/xform_null.c @@ -1,132 +1,132 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static int null_setkey(void *, const uint8_t *, int); static void null_crypt(void *, const uint8_t *, uint8_t *); static void null_init(void *); static void null_reinit(void *ctx, const uint8_t *buf, u_int len); static int null_update(void *, const void *, u_int); static void null_final(uint8_t *, void *); /* Encryption instances */ -struct enc_xform enc_xform_null = { +const struct enc_xform enc_xform_null = { .type = CRYPTO_NULL_CBC, .name = "NULL", /* NB: blocksize of 4 is to generate a properly aligned ESP header */ .blocksize = NULL_BLOCK_LEN, .ivsize = 0, .minkey = NULL_MIN_KEY, .maxkey = NULL_MAX_KEY, .encrypt = null_crypt, .decrypt = null_crypt, .setkey = null_setkey, }; /* Authentication instances */ -struct auth_hash auth_hash_null = { +const struct auth_hash auth_hash_null = { .type = CRYPTO_NULL_HMAC, .name = "NULL-HMAC", .keysize = 0, .hashsize = NULL_HASH_LEN, .ctxsize = sizeof(int), /* NB: context isn't used */ .blocksize = NULL_HMAC_BLOCK_LEN, .Init = null_init, .Setkey = null_reinit, .Reinit = null_reinit, .Update = null_update, .Final = null_final, }; /* * Encryption wrapper routines. */ static void null_crypt(void *key, const uint8_t *in, uint8_t *out) { } static int null_setkey(void *sched, const uint8_t *key, int len) { return (0); } /* * And now for auth. */ static void null_init(void *ctx) { } static void null_reinit(void *ctx, const uint8_t *buf, u_int len) { } static int null_update(void *ctx, const void *buf, u_int len) { return 0; } static void null_final(uint8_t *buf, void *ctx) { if (buf != (uint8_t *) 0) bzero(buf, 12); } diff --git a/sys/opencrypto/xform_poly1305.c b/sys/opencrypto/xform_poly1305.c index d8ceab47deca..c885192f9df6 100644 --- a/sys/opencrypto/xform_poly1305.c +++ b/sys/opencrypto/xform_poly1305.c @@ -1,72 +1,72 @@ /* This file is in the public domain. */ #include __FBSDID("$FreeBSD$"); #include #include struct poly1305_xform_ctx { struct crypto_onetimeauth_poly1305_state state; }; CTASSERT(sizeof(union authctx) >= sizeof(struct poly1305_xform_ctx)); CTASSERT(POLY1305_KEY_LEN == crypto_onetimeauth_poly1305_KEYBYTES); CTASSERT(POLY1305_HASH_LEN == crypto_onetimeauth_poly1305_BYTES); static void xform_Poly1305_Init(void *polyctx) { /* Nop */ } static void xform_Poly1305_Setkey(void *ctx, const uint8_t *key, u_int klen) { struct poly1305_xform_ctx *polyctx = ctx; int rc; if (klen != POLY1305_KEY_LEN) panic("%s: Bogus keylen: %u bytes", __func__, (unsigned)klen); rc = crypto_onetimeauth_poly1305_init(&polyctx->state, key); if (rc != 0) panic("%s: Invariant violated: %d", __func__, rc); } static int xform_Poly1305_Update(void *ctx, const void *data, u_int len) { struct poly1305_xform_ctx *polyctx = ctx; int rc; rc = crypto_onetimeauth_poly1305_update(&polyctx->state, data, len); if (rc != 0) panic("%s: Invariant violated: %d", __func__, rc); return (0); } static void xform_Poly1305_Final(uint8_t *digest, void *ctx) { struct poly1305_xform_ctx *polyctx = ctx; int rc; rc = crypto_onetimeauth_poly1305_final(&polyctx->state, digest); if (rc != 0) panic("%s: Invariant violated: %d", __func__, rc); } -struct auth_hash auth_hash_poly1305 = { +const struct auth_hash auth_hash_poly1305 = { .type = CRYPTO_POLY1305, .name = "Poly-1305", .keysize = POLY1305_KEY_LEN, .hashsize = POLY1305_HASH_LEN, .ctxsize = sizeof(struct poly1305_xform_ctx), .blocksize = crypto_onetimeauth_poly1305_BYTES, .Init = xform_Poly1305_Init, .Setkey = xform_Poly1305_Setkey, .Update = xform_Poly1305_Update, .Final = xform_Poly1305_Final, }; diff --git a/sys/opencrypto/xform_rijndael.c b/sys/opencrypto/xform_rijndael.c index d9e2497e7e53..685e53640c48 100644 --- a/sys/opencrypto/xform_rijndael.c +++ b/sys/opencrypto/xform_rijndael.c @@ -1,98 +1,98 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static int rijndael128_setkey(void *, const uint8_t *, int); static void rijndael128_encrypt(void *, const uint8_t *, uint8_t *); static void rijndael128_decrypt(void *, const uint8_t *, uint8_t *); /* Encryption instances */ -struct enc_xform enc_xform_rijndael128 = { +const struct enc_xform enc_xform_rijndael128 = { .type = CRYPTO_RIJNDAEL128_CBC, .name = "Rijndael-128/AES", .ctxsize = sizeof(rijndael_ctx), .blocksize = RIJNDAEL128_BLOCK_LEN, .ivsize = RIJNDAEL128_BLOCK_LEN, .minkey = RIJNDAEL_MIN_KEY, .maxkey = RIJNDAEL_MAX_KEY, .encrypt = rijndael128_encrypt, .decrypt = rijndael128_decrypt, .setkey = rijndael128_setkey, }; /* * Encryption wrapper routines. */ static void rijndael128_encrypt(void *key, const uint8_t *in, uint8_t *out) { rijndael_encrypt(key, in, out); } static void rijndael128_decrypt(void *key, const uint8_t *in, uint8_t *out) { rijndael_decrypt(key, in, out); } static int rijndael128_setkey(void *sched, const uint8_t *key, int len) { if (len != 16 && len != 24 && len != 32) return (EINVAL); rijndael_set_key(sched, key, len * 8); return (0); } diff --git a/sys/opencrypto/xform_rmd160.c b/sys/opencrypto/xform_rmd160.c index b4a845a7b323..8480a63d12dc 100644 --- a/sys/opencrypto/xform_rmd160.c +++ b/sys/opencrypto/xform_rmd160.c @@ -1,90 +1,90 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static void RMD160Init_int(void *); static int RMD160Update_int(void *, const void *, u_int); static void RMD160Final_int(uint8_t *, void *); /* Authentication instances */ -struct auth_hash auth_hash_hmac_ripemd_160 = { +const struct auth_hash auth_hash_hmac_ripemd_160 = { .type = CRYPTO_RIPEMD160_HMAC, .name = "HMAC-RIPEMD-160", .keysize = RIPEMD160_BLOCK_LEN, .hashsize = RIPEMD160_HASH_LEN, .ctxsize = sizeof(RMD160_CTX), .blocksize = RIPEMD160_BLOCK_LEN, .Init = RMD160Init_int, .Update = RMD160Update_int, .Final = RMD160Final_int, }; static void RMD160Init_int(void *ctx) { RMD160Init(ctx); } static int RMD160Update_int(void *ctx, const void *buf, u_int len) { RMD160Update(ctx, buf, len); return 0; } static void RMD160Final_int(uint8_t *digest, void *ctx) { RMD160Final(digest, ctx); } diff --git a/sys/opencrypto/xform_sha1.c b/sys/opencrypto/xform_sha1.c index a96f5024f57c..25bed2dc20b5 100644 --- a/sys/opencrypto/xform_sha1.c +++ b/sys/opencrypto/xform_sha1.c @@ -1,105 +1,105 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include static void SHA1Init_int(void *); static int SHA1Update_int(void *, const void *, u_int); static void SHA1Final_int(uint8_t *, void *); /* Plain hash */ -struct auth_hash auth_hash_sha1 = { +const struct auth_hash auth_hash_sha1 = { .type = CRYPTO_SHA1, .name = "SHA1", .hashsize = SHA1_HASH_LEN, .ctxsize = sizeof(SHA1_CTX), .blocksize = SHA1_BLOCK_LEN, .Init = SHA1Init_int, .Update = SHA1Update_int, .Final = SHA1Final_int, }; /* Authentication instances */ -struct auth_hash auth_hash_hmac_sha1 = { +const struct auth_hash auth_hash_hmac_sha1 = { .type = CRYPTO_SHA1_HMAC, .name = "HMAC-SHA1", .keysize = SHA1_BLOCK_LEN, .hashsize = SHA1_HASH_LEN, .ctxsize = sizeof(SHA1_CTX), .blocksize = SHA1_BLOCK_LEN, .Init = SHA1Init_int, .Update = SHA1Update_int, .Final = SHA1Final_int, }; /* * And now for auth. */ static void SHA1Init_int(void *ctx) { SHA1Init(ctx); } static int SHA1Update_int(void *ctx, const void *buf, u_int len) { SHA1Update(ctx, buf, len); return 0; } static void SHA1Final_int(uint8_t *blk, void *ctx) { SHA1Final(blk, ctx); } diff --git a/sys/opencrypto/xform_sha2.c b/sys/opencrypto/xform_sha2.c index b4a5533baa19..39795bbda639 100644 --- a/sys/opencrypto/xform_sha2.c +++ b/sys/opencrypto/xform_sha2.c @@ -1,190 +1,190 @@ /* $OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $ */ /*- * The authors of this code are John Ioannidis (ji@tla.org), * Angelos D. Keromytis (kermit@csd.uch.gr), * Niels Provos (provos@physnet.uni-hamburg.de) and * Damien Miller (djm@mindrot.org). * * This code was written by John Ioannidis for BSD/OS in Athens, Greece, * in November 1995. * * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, * by Angelos D. Keromytis. * * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis * and Niels Provos. * * Additional features in 1999 by Angelos D. Keromytis. * * AES XTS implementation in 2008 by Damien Miller * * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, * Angelos D. Keromytis and Niels Provos. * * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in * all copies of any software which is or includes a copy or * modification of this software. * You may use this code under the GNU public license if you so wish. Please * contribute changes back to the authors under this freer than GPL license * so that we may further the use of strong encryption without limitations to * all. * * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR * PURPOSE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include static int SHA224Update_int(void *, const void *, u_int); static int SHA256Update_int(void *, const void *, u_int); static int SHA384Update_int(void *, const void *, u_int); static int SHA512Update_int(void *, const void *, u_int); /* Plain hashes */ -struct auth_hash auth_hash_sha2_224 = { +const struct auth_hash auth_hash_sha2_224 = { .type = CRYPTO_SHA2_224, .name = "SHA2-224", .hashsize = SHA2_224_HASH_LEN, .ctxsize = sizeof(SHA224_CTX), .blocksize = SHA2_224_BLOCK_LEN, .Init = (void (*)(void *)) SHA224_Init, .Update = SHA224Update_int, .Final = (void (*)(uint8_t *, void *)) SHA224_Final, }; -struct auth_hash auth_hash_sha2_256 = { +const struct auth_hash auth_hash_sha2_256 = { .type = CRYPTO_SHA2_256, .name = "SHA2-256", .keysize = SHA2_256_BLOCK_LEN, .hashsize = SHA2_256_HASH_LEN, .ctxsize = sizeof(SHA256_CTX), .blocksize = SHA2_256_BLOCK_LEN, .Init = (void (*)(void *)) SHA256_Init, .Update = SHA256Update_int, .Final = (void (*)(uint8_t *, void *)) SHA256_Final, }; -struct auth_hash auth_hash_sha2_384 = { +const struct auth_hash auth_hash_sha2_384 = { .type = CRYPTO_SHA2_384, .name = "SHA2-384", .keysize = SHA2_384_BLOCK_LEN, .hashsize = SHA2_384_HASH_LEN, .ctxsize = sizeof(SHA384_CTX), .blocksize = SHA2_384_BLOCK_LEN, .Init = (void (*)(void *)) SHA384_Init, .Update = SHA384Update_int, .Final = (void (*)(uint8_t *, void *)) SHA384_Final, }; -struct auth_hash auth_hash_sha2_512 = { +const struct auth_hash auth_hash_sha2_512 = { .type = CRYPTO_SHA2_512, .name = "SHA2-512", .keysize = SHA2_512_BLOCK_LEN, .hashsize = SHA2_512_HASH_LEN, .ctxsize = sizeof(SHA512_CTX), .blocksize = SHA2_512_BLOCK_LEN, .Init = (void (*)(void *)) SHA512_Init, .Update = SHA512Update_int, .Final = (void (*)(uint8_t *, void *)) SHA512_Final, }; /* Authentication instances */ -struct auth_hash auth_hash_hmac_sha2_224 = { +const struct auth_hash auth_hash_hmac_sha2_224 = { .type = CRYPTO_SHA2_224_HMAC, .name = "HMAC-SHA2-224", .keysize = SHA2_224_BLOCK_LEN, .hashsize = SHA2_224_HASH_LEN, .ctxsize = sizeof(SHA224_CTX), .blocksize = SHA2_224_BLOCK_LEN, .Init = (void (*)(void *)) SHA224_Init, .Update = SHA224Update_int, .Final = (void (*)(uint8_t *, void *)) SHA224_Final, }; -struct auth_hash auth_hash_hmac_sha2_256 = { +const struct auth_hash auth_hash_hmac_sha2_256 = { .type = CRYPTO_SHA2_256_HMAC, .name = "HMAC-SHA2-256", .keysize = SHA2_256_BLOCK_LEN, .hashsize = SHA2_256_HASH_LEN, .ctxsize = sizeof(SHA256_CTX), .blocksize = SHA2_256_BLOCK_LEN, .Init = (void (*)(void *)) SHA256_Init, .Update = SHA256Update_int, .Final = (void (*)(uint8_t *, void *)) SHA256_Final, }; -struct auth_hash auth_hash_hmac_sha2_384 = { +const struct auth_hash auth_hash_hmac_sha2_384 = { .type = CRYPTO_SHA2_384_HMAC, .name = "HMAC-SHA2-384", .keysize = SHA2_384_BLOCK_LEN, .hashsize = SHA2_384_HASH_LEN, .ctxsize = sizeof(SHA384_CTX), .blocksize = SHA2_384_BLOCK_LEN, .Init = (void (*)(void *)) SHA384_Init, .Update = SHA384Update_int, .Final = (void (*)(uint8_t *, void *)) SHA384_Final, }; -struct auth_hash auth_hash_hmac_sha2_512 = { +const struct auth_hash auth_hash_hmac_sha2_512 = { .type = CRYPTO_SHA2_512_HMAC, .name = "HMAC-SHA2-512", .keysize = SHA2_512_BLOCK_LEN, .hashsize = SHA2_512_HASH_LEN, .ctxsize = sizeof(SHA512_CTX), .blocksize = SHA2_512_BLOCK_LEN, .Init = (void (*)(void *)) SHA512_Init, .Update = SHA512Update_int, .Final = (void (*)(uint8_t *, void *)) SHA512_Final, }; /* * And now for auth. */ static int SHA224Update_int(void *ctx, const void *buf, u_int len) { SHA224_Update(ctx, buf, len); return 0; } static int SHA256Update_int(void *ctx, const void *buf, u_int len) { SHA256_Update(ctx, buf, len); return 0; } static int SHA384Update_int(void *ctx, const void *buf, u_int len) { SHA384_Update(ctx, buf, len); return 0; } static int SHA512Update_int(void *ctx, const void *buf, u_int len) { SHA512_Update(ctx, buf, len); return 0; }