hyperv/vmbus: Nuke unnecessary function indirection
ClosedPublic
Actions

Authored by sepherosa_gmail.com on May 17 2016, 6:54 AM.

Details

Reviewers

howard0su_gmail.com
honzhan_microsoft.com
decui_microsoft.com
delphij
royger
adrian

Commits

rS300121: hyperv/vmbus: Nuke unnecessary function indirection

Diff Detail

Lint

Lint Skipped

Unit

Tests Skipped

Event Timeline

sepherosa_gmail.com updated this revision to Diff 16454.May 17 2016, 6:54 AM

sepherosa_gmail.com retitled this revision from to hyperv/vmbus: Nuke unnecessary function indirection.

sepherosa_gmail.com updated this object.

sepherosa_gmail.com edited the test plan for this revision. (Show Details)

sepherosa_gmail.com added reviewers: adrian, delphij, royger, decui_microsoft.com, honzhan_microsoft.com, howard0su_gmail.com.

Closed by commit rS300121: hyperv/vmbus: Nuke unnecessary function indirection (authored by sephe). · Explain WhyMay 18 2016, 6:29 AM

This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Path

Size

sys/

dev/

hyperv/

vmbus/

hv_vmbus_drv_freebsd.c

17 lines

Diff 16454

View Options

sys/dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c

This file was added.

				/*-
				* Copyright (c) 2017 Chelsio Communications, Inc.
				* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
				* All rights reserved.
				* Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
				*
				* Redistribution and use in source and binary forms, with or without
				* modification, are permitted provided that the following conditions
				* are met:
				* 1. Redistributions of source code must retain the above copyright
				* notice, this list of conditions and the following disclaimer.
				* 2. Redistributions in binary form must reproduce the above copyright
				* notice, this list of conditions and the following disclaimer in the
				* documentation and/or other materials provided with the distribution.
				*
				* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
				* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
				* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
				* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
				* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
				* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
				* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
				* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
				* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
				* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
				* SUCH DAMAGE.
				*/

				#include <sys/cdefs.h>
				__FBSDID("$FreeBSD$");

				#include "opt_ddb.h"

				#include <sys/types.h>
				#include <sys/bus.h>
				#include <sys/lock.h>
				#include <sys/kernel.h>
				#include <sys/malloc.h>
				#include <sys/mutex.h>
				#include <sys/module.h>
				#include <sys/random.h>
				#include <sys/sglist.h>
				#include <sys/sysctl.h>

				#ifdef DDB
				#include <ddb/ddb.h>
				#endif

				#include <dev/pci/pcivar.h>

				#include <dev/random/randomdev.h>

				#include <opencrypto/cryptodev.h>
				#include <opencrypto/xform.h>

				#include "cryptodev_if.h"

				#include "ccp.h"
				#include "ccp_hardware.h"

				MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");

				/*
				* Need a global softc available for garbage random_source API, which lacks any
				* context pointer. It's also handy for debugging.
				*/
				struct ccp_softc *g_ccp_softc;

				static struct pciid {
				uint32_t devid;
				const char *desc;
				} ccp_ids[] = {
				{ 0x14561022, "AMD CCP-5a" },
				{ 0x14681022, "AMD CCP-5b" },
				};
				MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, sizeof(ccp_ids[0]),
				nitems(ccp_ids));

				static struct random_source random_ccp = {
				.rs_ident = "AMD CCP TRNG",
				.rs_source = RANDOM_PURE_CCP,
				.rs_read = random_ccp_read,
				};

				/*
				* ccp_populate_sglist() generates a scatter/gather list that covers the entire
				* crypto operation buffer.
				*/
				static int
				ccp_populate_sglist(struct sglist sg, struct cryptop crp)
				{
				int error;

				sglist_reset(sg);
				if (crp->crp_flags & CRYPTO_F_IMBUF)
				error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
				else if (crp->crp_flags & CRYPTO_F_IOV)
				error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
				else
				error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
				return (error);
				}

				/*
				* Handle a GCM request with an empty payload by performing the
				* operation in software. Derived from swcr_authenc().
				*/
				static void
				ccp_gcm_soft(struct ccp_session s, struct cryptop crp,
				struct cryptodesc crda, struct cryptodesc crde)
				{
				struct aes_gmac_ctx gmac_ctx;
				char block[GMAC_BLOCK_LEN];
				char digest[GMAC_DIGEST_LEN];
				char iv[AES_BLOCK_LEN];
				int i, len;

				/*
				* This assumes a 12-byte IV from the crp. See longer comment
				* above in ccp_gcm() for more details.
				*/
				if (crde->crd_flags & CRD_F_ENCRYPT) {
				if (crde->crd_flags & CRD_F_IV_EXPLICIT)
				memcpy(iv, crde->crd_iv, 12);
				else
				arc4rand(iv, 12, 0);
				} else {
				if (crde->crd_flags & CRD_F_IV_EXPLICIT)
				memcpy(iv, crde->crd_iv, 12);
				else
				crypto_copydata(crp->crp_flags, crp->crp_buf,
				crde->crd_inject, 12, iv);
				}
				(uint32_t )&iv[12] = htobe32(1);

				/* Initialize the MAC. */
				AES_GMAC_Init(&gmac_ctx);
				AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
				AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));

				/* MAC the AAD. */
				for (i = 0; i < crda->crd_len; i += sizeof(block)) {
				len = imin(crda->crd_len - i, sizeof(block));
				crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
				i, len, block);
				bzero(block + len, sizeof(block) - len);
				AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
				}

				/* Length block. */
				bzero(block, sizeof(block));
				((uint32_t )block)[1] = htobe32(crda->crd_len 8);
				AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
				AES_GMAC_Final(digest, &gmac_ctx);

				if (crde->crd_flags & CRD_F_ENCRYPT) {
				crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
				sizeof(digest), digest);
				crp->crp_etype = 0;
				} else {
				char digest2[GMAC_DIGEST_LEN];

				crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
				sizeof(digest2), digest2);
				if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
				crp->crp_etype = 0;
				else
				crp->crp_etype = EBADMSG;
				}
				crypto_done(crp);
				}

				static int
				ccp_probe(device_t dev)
				{
				struct pciid *ip;
				uint32_t id;

				id = pci_get_devid(dev);
				for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
				if (id == ip->devid) {
				device_set_desc(dev, ip->desc);
				return (0);
				}
				}
				return (ENXIO);
				}

				static void
				ccp_initialize_queues(struct ccp_softc *sc)
				{
				struct ccp_queue *qp;
				size_t i;

				for (i = 0; i < nitems(sc->queues); i++) {
				qp = &sc->queues[i];
				markjUnsubmitted Done Inline Actions s/SGLs/SGEs/ markj: s/SGLs/SGEs/

				qp->cq_softc = sc;
				qp->cq_qindex = i;
				mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
				/* XXX - arbitrarily chosen sizes */
				qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
				/* Two more SGEs than sg_crp to accommodate ipad. */
				qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
				qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
				}
				}

				static void
				ccp_free_queues(struct ccp_softc *sc)
				{
				struct ccp_queue *qp;
				size_t i;

				for (i = 0; i < nitems(sc->queues); i++) {
				qp = &sc->queues[i];

				mtx_destroy(&qp->cq_lock);
				sglist_free(qp->cq_sg_crp);
				sglist_free(qp->cq_sg_ulptx);
				sglist_free(qp->cq_sg_dst);
				}
				}

				static int
				ccp_attach(device_t dev)
				{
				struct ccp_softc *sc;
				int error;

				sc = device_get_softc(dev);
				sc->dev = dev;

				device_printf(dev, "XXX%s: sc=%p\n", __func__, sc);
				markjUnsubmitted Done Inline Actions There's a lot of printf("XXX...") in the driver, which ought to be cleaned up at some point. markj: There's a lot of printf("XXX...") in the driver, which ought to be cleaned up at some point.
				cemAuthorUnsubmitted Done Inline Actions Definitely. At one point I was keeping this stuff separate in a git stash but I guess it got merged in sometime. Some of it even leaks stuff like key information. cem: Definitely. At one point I was keeping this stuff separate in a git stash but I guess it got…

				sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
				markjUnsubmitted Not Done Inline Actions Not something to be addressed here, but crypto_get_driverid() should probably just use unr(9). :/ markj: Not something to be addressed here, but crypto_get_driverid() should probably just use unr(9).
				if (sc->cid < 0) {
				device_printf(dev, "could not get crypto driver id\n");
				return (ENXIO);
				markjUnsubmitted Done Inline Actions Since ccp_initialize_queue() is only used once, it be a bit neater to have a ccp_initialize_queues() instead and move this loop there. markj: Since ccp_initialize_queue() is only used once, it be a bit neater to have a…
				cemAuthorUnsubmitted Not Done Inline Actions Ok cem: Ok
				}

				error = ccp_hw_attach(dev);
				if (error != 0)
				return (error);

				mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);

				ccp_initialize_queues(sc);

				if (g_ccp_softc == NULL) {
				g_ccp_softc = sc;
				if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
				random_source_register(&random_ccp);
				}

				if ((sc->hw_features & VERSION_CAP_AES) != 0) {
				crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
				#if 0
				crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
				#endif
				}
				if ((sc->hw_features & VERSION_CAP_SHA) != 0) {
				crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
				}

				return (0);
				}

				static int
				ccp_detach(device_t dev)
				{
				struct ccp_softc *sc;
				int i;

				sc = device_get_softc(dev);

				mtx_lock(&sc->lock);
				for (i = 0; i < sc->nsessions; i++) {
				if (sc->sessions[i].active \|\| sc->sessions[i].pending != 0) {
				mtx_unlock(&sc->lock);
				return (EBUSY);
				}
				}
				sc->detaching = true;
				mtx_unlock(&sc->lock);

				crypto_unregister_all(sc->cid);
				if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
				random_source_deregister(&random_ccp);

				ccp_hw_detach(dev);
				ccp_free_queues(sc);

				if (g_ccp_softc == sc)
				g_ccp_softc = NULL;

				free(sc->sessions, M_CCP);
				mtx_destroy(&sc->lock);
				return (0);
				}

				static void
				ccp_init_hmac_digest(struct ccp_session s, int cri_alg, char key,
				int klen)
				{
				union authctx auth_ctx;
				struct auth_hash *axf;
				u_int i;

				/*
				* If the key is larger than the block size, use the digest of
				* the key as the key instead.
				*/
				axf = s->hmac.auth_hash;
				klen /= 8;
				if (klen > axf->blocksize) {
				axf->Init(&auth_ctx);
				axf->Update(&auth_ctx, key, klen);
				axf->Final(s->hmac.ipad, &auth_ctx);
				explicit_bzero(&auth_ctx, sizeof(auth_ctx));
				klen = axf->hashsize;
				} else
				memcpy(s->hmac.ipad, key, klen);

				memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
				memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);

				for (i = 0; i < axf->blocksize; i++) {
				s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
				s->hmac.opad[i] ^= HMAC_OPAD_VAL;
				}
				}

				static int
				ccp_aes_check_keylen(int alg, int klen)
				{

				switch (klen) {
				case 128:
				case 192:
				if (alg == CRYPTO_AES_XTS)
				return (EINVAL);
				break;
				case 256:
				break;
				case 512:
				if (alg != CRYPTO_AES_XTS)
				return (EINVAL);
				break;
				default:
				return (EINVAL);
				}
				return (0);
				}

				/*
				* Borrowed from cesa_prep_aes_key(). We should perhaps have a public
				* function to generate this instead.
				*
				* NB: The crypto engine wants the words in the decryption key in reverse
				* order.
				*/
				static void
				ccp_aes_getdeckey(void dec_key, const void enc_key, unsigned int kbits)
				{
				uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
				uint32_t *dkey;
				int i;

				rijndaelKeySetupEnc(ek, enc_key, kbits);
				dkey = dec_key;
				dkey += (kbits / 8) / 4;

				switch (kbits) {
				case 128:
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 10 + i]);
				break;
				case 192:
				for (i = 0; i < 2; i++)
				--dkey = htobe32(ek[4 11 + 2 + i]);
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 12 + i]);
				break;
				case 256:
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 13 + i]);
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 14 + i]);
				break;
				}
				MPASS(dkey == dec_key);
				}

				static void
				ccp_aes_setkey(struct ccp_session s, int alg, const void key, int klen)
				{
				unsigned kbits;

				if (alg == CRYPTO_AES_XTS)
				kbits = klen / 2;
				else
				kbits = klen;

				switch (kbits) {
				case 128:
				s->blkcipher.cipher_type = CCP_AES_TYPE_128;
				break;
				case 192:
				s->blkcipher.cipher_type = CCP_AES_TYPE_192;
				break;
				case 256:
				s->blkcipher.cipher_type = CCP_AES_TYPE_256;
				break;
				default:
				panic("should not get here");
				}

				s->blkcipher.key_len = klen / 8;
				memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
				switch (alg) {
				case CRYPTO_AES_XTS:
				ccp_aes_getdeckey(s->blkcipher.deckey, key, kbits);
				break;
				}
				}

				static int
				ccp_newsession(device_t dev, uint32_t sidp, struct cryptoini cri)
				{
				struct ccp_softc *sc;
				struct ccp_session *s;
				struct auth_hash *auth_hash;
				struct cryptoini c, hash, *cipher;
				enum ccp_aes_mode cipher_mode;
				unsigned auth_mode, iv_len;
				unsigned partial_digest_len;
				unsigned q;
				int error, i, sess;
				bool gcm_hash;

				if (sidp == NULL \|\| cri == NULL)
				return (EINVAL);

				gcm_hash = false;
				cipher = NULL;
				hash = NULL;
				auth_hash = NULL;
				/* XXX reconcile auth_mode with use by ccp_sha */
				auth_mode = 0;
				cipher_mode = CCP_AES_MODE_ECB;
				markjUnsubmitted Not Done Inline Actions Seems like an odd initialization default, especially since we don't set cipher_mode in the AES-XTS case. We don't register support for AES-XTS, so this isn't a major problem at the moment, but it looks weird. markj: Seems like an odd initialization default, especially since we don't set cipher_mode in the AES…
				cemAuthorUnsubmitted Not Done Inline Actions I suppose. Mostly it just needs to be initialized to something other than CCP_AES_MODE_GCTR. Ideally AES-XTS will be added at some point, and that will make it more clear. We could `#if 0` the the XTS case of the outer switch, making it return EINVAL? (Though as you point out, it should never be invoked as we don't register support.) cem: I suppose. Mostly it just needs to be initialized to something other than CCP_AES_MODE_GCTR.
				cemAuthorUnsubmitted Not Done Inline Actions The other reason to use ECB is that it is so clearly a bogus value. The hardware will do it, but OCF and this driver don't support it (for good reason). cem: The other reason to use ECB is that it is so clearly a bogus value. The hardware will do it…
				iv_len = 0;
				partial_digest_len = 0;
				for (c = cri; c != NULL; c = c->cri_next) {
				switch (c->cri_alg) {
				case CRYPTO_SHA1_HMAC:
				case CRYPTO_SHA2_256_HMAC:
				case CRYPTO_SHA2_384_HMAC:
				case CRYPTO_SHA2_512_HMAC:
				case CRYPTO_AES_128_NIST_GMAC:
				case CRYPTO_AES_192_NIST_GMAC:
				case CRYPTO_AES_256_NIST_GMAC:
				if (hash)
				return (EINVAL);
				hash = c;
				switch (c->cri_alg) {
				case CRYPTO_SHA1_HMAC:
				auth_hash = &auth_hash_hmac_sha1;
				auth_mode = SHA1;
				partial_digest_len = SHA1_HASH_LEN;
				break;
				case CRYPTO_SHA2_256_HMAC:
				auth_hash = &auth_hash_hmac_sha2_256;
				auth_mode = SHA2_256;
				partial_digest_len = SHA2_256_HASH_LEN;
				break;
				case CRYPTO_SHA2_384_HMAC:
				auth_hash = &auth_hash_hmac_sha2_384;
				auth_mode = SHA2_384;
				partial_digest_len = SHA2_512_HASH_LEN;
				break;
				case CRYPTO_SHA2_512_HMAC:
				auth_hash = &auth_hash_hmac_sha2_512;
				auth_mode = SHA2_512;
				partial_digest_len = SHA2_512_HASH_LEN;
				break;
				case CRYPTO_AES_128_NIST_GMAC:
				case CRYPTO_AES_192_NIST_GMAC:
				case CRYPTO_AES_256_NIST_GMAC:
				gcm_hash = true;
				#if 0
				auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
				#endif
				break;
				}
				break;
				case CRYPTO_AES_CBC:
				case CRYPTO_AES_ICM:
				case CRYPTO_AES_NIST_GCM_16:
				case CRYPTO_AES_XTS:
				if (cipher)
				return (EINVAL);
				cipher = c;
				switch (c->cri_alg) {
				case CRYPTO_AES_CBC:
				cipher_mode = CCP_AES_MODE_CBC;
				iv_len = AES_BLOCK_LEN;
				break;
				case CRYPTO_AES_ICM:
				cipher_mode = CCP_AES_MODE_CTR;
				iv_len = AES_BLOCK_LEN;
				break;
				case CRYPTO_AES_NIST_GCM_16:
				cipher_mode = CCP_AES_MODE_GCTR;
				iv_len = AES_GCM_IV_LEN;
				break;
				case CRYPTO_AES_XTS:
				#if 0
				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
				#endif
				iv_len = AES_BLOCK_LEN;
				break;
				}
				if (c->cri_key != NULL) {
				error = ccp_aes_check_keylen(c->cri_alg,
				c->cri_klen);
				if (error != 0)
				return (error);
				}
				break;
				default:
				return (EINVAL);
				}
				}
				if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR))
				return (EINVAL);
				if (hash == NULL && cipher == NULL)
				return (EINVAL);
				if (hash != NULL && hash->cri_key == NULL)
				return (EINVAL);

				sc = device_get_softc(dev);
				mtx_lock(&sc->lock);
				if (sc->detaching) {
				mtx_unlock(&sc->lock);
				return (ENXIO);
				}
				sess = -1;
				for (i = 0; i < sc->nsessions; i++) {
				if (!sc->sessions[i].active && sc->sessions[i].pending == 0) {
				sess = i;
				break;
				}
				}
				if (sess == -1) {
				s = malloc(sizeof(s) (sc->nsessions + 1), M_CCP,
				M_NOWAIT \| M_ZERO);
				if (s == NULL) {
				mtx_unlock(&sc->lock);
				return (ENOMEM);
				}
				if (sc->sessions != NULL)
				memcpy(s, sc->sessions, sizeof(s) sc->nsessions);
				sess = sc->nsessions;
				free(sc->sessions, M_CCP);
				sc->sessions = s;
				sc->nsessions++;
				}

				s = &sc->sessions[sess];

				/* Just grab the first usable queue for now. */
				for (q = 0; q < nitems(sc->queues); q++)
				if ((sc->valid_queues & (1 << q)) != 0)
				break;
				if (q == nitems(sc->queues)) {
				mtx_unlock(&sc->lock);
				return (ENXIO);
				}
				s->queue = q;

				if (gcm_hash)
				s->mode = GCM;
				else if (hash != NULL && cipher != NULL)
				s->mode = AUTHENC;
				else if (hash != NULL)
				s->mode = HMAC;
				else {
				MPASS(cipher != NULL);
				s->mode = BLKCIPHER;
				}
				if (gcm_hash) {
				if (hash->cri_mlen == 0)
				s->gmac.hash_len = AES_GMAC_HASH_LEN;
				else
				s->gmac.hash_len = hash->cri_mlen;
				} else if (hash != NULL) {
				s->hmac.auth_hash = auth_hash;
				s->hmac.auth_mode = auth_mode;
				s->hmac.partial_digest_len = partial_digest_len;
				if (hash->cri_mlen == 0)
				s->hmac.hash_len = auth_hash->hashsize;
				else
				s->hmac.hash_len = hash->cri_mlen;
				ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
				hash->cri_klen);
				}
				if (cipher != NULL) {
				s->blkcipher.cipher_mode = cipher_mode;
				s->blkcipher.iv_len = iv_len;
				if (cipher->cri_key != NULL)
				ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
				cipher->cri_klen);
				}

				s->active = true;
				mtx_unlock(&sc->lock);

				*sidp = sess;
				return (0);
				}

				static int
				ccp_freesession(device_t dev, uint64_t tid)
				{
				struct ccp_softc *sc;
				uint32_t sid;
				int error;

				sc = device_get_softc(dev);
				sid = CRYPTO_SESID2LID(tid);
				mtx_lock(&sc->lock);
				if (sid >= sc->nsessions \|\| !sc->sessions[sid].active)
				error = EINVAL;
				else {
				if (sc->sessions[sid].pending != 0)
				device_printf(dev,
				"session %d freed with %d pending requests\n", sid,
				sc->sessions[sid].pending);
				sc->sessions[sid].active = false;
				error = 0;
				}
				mtx_unlock(&sc->lock);
				return (error);
				}

				static int
				ccp_process(device_t dev, struct cryptop *crp, int hint)
				{
				struct ccp_softc *sc;
				struct ccp_queue *qp;
				struct ccp_session *s;
				struct cryptodesc crd, crda, *crde;
				uint32_t sid;
				int error;
				bool qpheld;

				qpheld = false;
				qp = NULL;
				if (crp == NULL)
				return (EINVAL);

				crd = crp->crp_desc;
				sid = CRYPTO_SESID2LID(crp->crp_sid);
				sc = device_get_softc(dev);
				mtx_lock(&sc->lock);
				if (sid >= sc->nsessions \|\| !sc->sessions[sid].active) {
				mtx_unlock(&sc->lock);
				error = EINVAL;
				goto out;
				}

				s = &sc->sessions[sid];
				qp = &sc->queues[s->queue];
				mtx_unlock(&sc->lock);
				error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
				if (error != 0)
				goto out;
				qpheld = true;

				error = ccp_populate_sglist(qp->cq_sg_crp, crp);
				if (error != 0)
				goto out;

				switch (s->mode) {
				case HMAC:
				if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
				ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
				crd->crd_klen);
				error = ccp_hmac(qp, s, crp);
				break;
				case BLKCIPHER:
				if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crd->crd_alg,
				crd->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crd->crd_alg, crd->crd_key,
				crd->crd_klen);
				}
				error = ccp_blkcipher(qp, s, crp);
				break;
				case AUTHENC:
				error = 0;
				switch (crd->crd_alg) {
				case CRYPTO_AES_CBC:
				case CRYPTO_AES_ICM:
				case CRYPTO_AES_XTS:
				/* Only encrypt-then-authenticate supported. */
				crde = crd;
				crda = crd->crd_next;
				if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
				error = EINVAL;
				break;
				}
				s->cipher_first = true;
				break;
				default:
				crda = crd;
				crde = crd->crd_next;
				if (crde->crd_flags & CRD_F_ENCRYPT) {
				error = EINVAL;
				break;
				}
				s->cipher_first = false;
				break;
				}
				if (error != 0)
				break;
				if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
				ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
				crda->crd_klen);
				if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crde->crd_alg,
				crde->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
				crde->crd_klen);
				}
				error = ccp_authenc(qp, s, crp, crda, crde);
				break;
				case GCM:
				error = 0;
				if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
				crde = crd;
				crda = crd->crd_next;
				s->cipher_first = true;
				} else {
				crda = crd;
				crde = crd->crd_next;
				s->cipher_first = false;
				}
				if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crde->crd_alg,
				crde->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
				crde->crd_klen);
				}
				if (crde->crd_len == 0) {
				mtx_unlock(&qp->cq_lock);
				ccp_gcm_soft(s, crp, crda, crde);
				return (0);
				}
				error = ccp_gcm(qp, s, crp, crda, crde);
				break;
				}

				if (error == 0)
				s->pending++;

				out:
				if (qpheld) {
				if (error != 0)
				ccp_queue_abort(qp);
				else
				ccp_queue_release(qp);
				}

				if (error != 0) {
				device_printf(dev, "XXX %s: early error:%d\n", __func__,
				error);
				crp->crp_etype = error;
				crypto_done(crp);
				}
				return (0);
				}

				static device_method_t ccp_methods[] = {
				DEVMETHOD(device_probe, ccp_probe),
				DEVMETHOD(device_attach, ccp_attach),
				DEVMETHOD(device_detach, ccp_detach),

				DEVMETHOD(cryptodev_newsession, ccp_newsession),
				DEVMETHOD(cryptodev_freesession, ccp_freesession),
				DEVMETHOD(cryptodev_process, ccp_process),

				DEVMETHOD_END
				};

				static driver_t ccp_driver = {
				"ccp",
				ccp_methods,
				sizeof(struct ccp_softc)
				markjUnsubmitted Done Inline Actions "error" is write-only. markj: "error" is write-only.
				};

				static devclass_t ccp_devclass;
				DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
				MODULE_VERSION(ccp, 1);
				MODULE_DEPEND(ccp, crypto, 1, 1, 1);
				MODULE_DEPEND(ccp, random_device, 1, 1, 1);

				static int
				ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
				{
				struct ccp_softc *sc;

				mtx_assert(&qp->cq_lock, MA_OWNED);
				sc = qp->cq_softc;

				if (n < 1 \|\| n >= (1 << sc->ring_size_order))
				return (EINVAL);

				markjUnsubmitted Not Done Inline Actions This function and the one below appear to be unused. markj: This function and the one below appear to be unused.
				cemAuthorUnsubmitted Not Done Inline Actions Yeah, I was thinking it would be nice to have a kernel API like ioat that doesn't go through opencrypto, but don't have that very concrete yet. cem: Yeah, I was thinking it would be nice to have a kernel API like ioat that doesn't go through…
				while (true) {
				if (ccp_queue_get_ring_space(qp) >= n)
				return (0);
				if ((mflags & M_WAITOK) == 0)
				return (EAGAIN);
				qp->cq_waiting = true;
				msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
				}
				}

				int
				ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
				{
				int error;

				mtx_lock(&qp->cq_lock);
				qp->cq_acq_tail = qp->cq_tail;
				error = ccp_queue_reserve_space(qp, n, mflags);
				if (error != 0)
				mtx_unlock(&qp->cq_lock);
				return (error);
				}

				void
				ccp_queue_release(struct ccp_queue *qp)
				{

				mtx_assert(&qp->cq_lock, MA_OWNED);
				if (qp->cq_tail != qp->cq_acq_tail) {
				wmb();
				ccp_queue_write_tail(qp);
				}
				mtx_unlock(&qp->cq_lock);
				}

				void
				ccp_queue_abort(struct ccp_queue *qp)
				{
				unsigned i;

				mtx_assert(&qp->cq_lock, MA_OWNED);

				/* Wipe out back any descriptors associated with this aborted txn. */
				for (i = qp->cq_acq_tail; i != qp->cq_tail;
				i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
				memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
				}
				qp->cq_tail = qp->cq_acq_tail;

				mtx_unlock(&qp->cq_lock);
				}

				#ifdef DDB
				#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)
				#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)
				static void
				db_show_ccp_sc(struct ccp_softc *sc)
				{

				db_printf("ccp softc at %p\n", sc);
				db_printf(" cid: %d\n", (int)sc->cid);
				db_printf(" nsessions: %d\n", sc->nsessions);

				db_printf(" lock: ");
				db_show_lock(&sc->lock);

				db_printf(" detaching: %d\n", (int)sc->detaching);
				db_printf(" ring_size_order: %u\n", sc->ring_size_order);

				db_printf(" hw_version: %d\n", (int)sc->hw_version);
				db_printf(" hw_features: %b\n", (int)sc->hw_features,
				"\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
				"\11SHA\0103DES\07AES");

				db_printf(" hw status:\n");
				db_ccp_show_hw(sc);
				}

				static void
				db_show_ccp_qp(struct ccp_queue *qp)
				{

				db_printf(" lock: ");
				db_show_lock(&qp->cq_lock);

				#if 0
				db_printf(" head: %u\n", sc->head);
				db_printf(" tail: %u\n", sc->tail);
				db_printf(" hw_head: %u\n", sc->hw_head);
				db_printf(" last_seen: 0x%lx\n", sc->last_seen);
				db_printf(" ring: %p\n", sc->ring);
				db_printf(" descriptors: %p\n", sc->hw_desc_ring);
				db_printf(" descriptors (phys): 0x%jx\n",
				(uintmax_t)sc->hw_desc_bus_addr);

				db_printf(" ring[%u] (tail):\n", sc->tail %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->tail)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->tail));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->tail)->generic.next);

				db_printf(" ring[%u] (head - 1):\n", (sc->head - 1) %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head - 1)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->head - 1));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->head - 1)->generic.next);

				db_printf(" ring[%u] (head):\n", (sc->head) %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->head));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->head)->generic.next);

				for (idx = 0; idx < (1 << sc->ring_size_order); idx++)
				if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK)
				== RING_PHYS_ADDR(sc, idx))
				db_printf(" ring[%u] == hardware tail\n", idx);
				#endif

				db_printf(" hw status:\n");
				db_ccp_show_queue_hw(qp);
				}

				DB_SHOW_COMMAND(ccp, db_show_ccp)
				{
				struct ccp_softc *sc;
				unsigned unit, qindex;

				if (!have_addr)
				goto usage;

				unit = (unsigned)addr;

				sc = devclass_get_softc(ccp_devclass, unit);
				if (sc == NULL) {
				db_printf("No such device ccp%u\n", unit);
				goto usage;
				}

				if (count == -1) {
				db_show_ccp_sc(sc);
				return;
				}

				qindex = (unsigned)count;
				if (qindex >= nitems(sc->queues)) {
				db_printf("No such queue %u\n", qindex);
				goto usage;
				}
				db_show_ccp_qp(&sc->queues[qindex]);
				return;

				usage:
				db_printf("usage: show ccp <unit>[,<qindex>]\n");
				return;
				}
				#endif /* DDB */

hyperv/vmbus: Nuke unnecessary function indirectionClosedPublicActions