Paths

Table of Contentst

-
share/man/man7/
-
man/
-
man7/
-
mitigations.7

mitigations.7: explain RELRO
ClosedPublic
Actions

Authored by emaste on Oct 6 2023, 2:39 PM.

Details

Reviewers

olce
kib
bcr

Commits

rGa2b289608d6b: mitigations.7: briefly explain RELRO

Diff Detail

Lint

Lint Skipped

Unit

Tests Skipped

Event Timeline

emaste requested review of this revision.Oct 6 2023, 2:39 PM

emaste created this revision.

emaste added a subscriber: secteam.

bcr added a subscriber: bcr.Oct 6 2023, 3:03 PM

bcr added inline comments.

share/man/man7/mitigations.7
208	superfluous 'is"

emaste marked an inline comment as done.Oct 6 2023, 3:06 PM

emaste added inline comments.

share/man/man7/mitigations.7
208	fixed locally

OK from manpages.

This revision is now accepted and ready to land.Oct 6 2023, 3:42 PM

kib added inline comments.Oct 6 2023, 5:00 PM

share/man/man7/mitigations.7
197
203
205	I think it is better to remove this line

some feedback from @kib

This revision now requires review to proceed.Oct 6 2023, 5:40 PM

kib accepted this revision.Oct 6 2023, 8:04 PM

This revision is now accepted and ready to land.Oct 6 2023, 8:04 PM

I'm not too competent on the matter.

Is describing the ELF metadata "protected" by RELRO and what is non-standard with WITH_BIND_NOW too much work or out-of-scope? Personally, I would learn something if they were.

share/man/man7/mitigations.7
205	Typo.

Is describing the ELF metadata "protected" by RELRO and what is non-standard with WITH_BIND_NOW too much work or out-of-scope? Personally, I would learn something if they were.

I think it's probably too much detail to include here, we probably have to describe too much of ELF, rtld, and dynamic linking. It's at least more than I'm up for writing right now :)

@fbsd-phab_maskray.me has some good blog posts, maybe we could just add a reference to those:

share/man/man7/mitigations.7
205	Indeed, fixed locally.

emaste marked an inline comment as done.Oct 6 2023, 8:41 PM

Closed by commit rGa2b289608d6b: mitigations.7: briefly explain RELRO (authored by emaste). · Explain WhyOct 6 2023, 8:45 PM

This revision was automatically updated to reflect the committed changes.

emaste added a commit: rGa2b289608d6b: mitigations.7: briefly explain RELRO.

Herald added a subscriber: imp. · View Herald TranscriptOct 6 2023, 8:45 PM

Revision Contents
Changeset List

Path

Size

share/

man/

man7/

mitigations.7

36 lines

Diff 128378

View Options

share/man/man7/mitigations.7

This file was added.

				/*-
				* Copyright (c) 2017 Chelsio Communications, Inc.
				* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
				* All rights reserved.
				* Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
				*
				* Redistribution and use in source and binary forms, with or without
				* modification, are permitted provided that the following conditions
				* are met:
				* 1. Redistributions of source code must retain the above copyright
				* notice, this list of conditions and the following disclaimer.
				* 2. Redistributions in binary form must reproduce the above copyright
				* notice, this list of conditions and the following disclaimer in the
				* documentation and/or other materials provided with the distribution.
				*
				* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
				* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
				* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
				* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
				* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
				* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
				* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
				* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
				* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
				* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
				* SUCH DAMAGE.
				*/

				#include <sys/cdefs.h>
				__FBSDID("$FreeBSD$");

				#include "opt_ddb.h"

				#include <sys/types.h>
				#include <sys/bus.h>
				#include <sys/lock.h>
				#include <sys/kernel.h>
				#include <sys/malloc.h>
				#include <sys/mutex.h>
				#include <sys/module.h>
				#include <sys/random.h>
				#include <sys/sglist.h>
				#include <sys/sysctl.h>

				#ifdef DDB
				#include <ddb/ddb.h>
				#endif

				#include <dev/pci/pcivar.h>

				#include <dev/random/randomdev.h>

				#include <opencrypto/cryptodev.h>
				#include <opencrypto/xform.h>

				#include "cryptodev_if.h"

				#include "ccp.h"
				#include "ccp_hardware.h"

				MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");

				/*
				* Need a global softc available for garbage random_source API, which lacks any
				* context pointer. It's also handy for debugging.
				*/
				struct ccp_softc *g_ccp_softc;

				static struct pciid {
				uint32_t devid;
				const char *desc;
				} ccp_ids[] = {
				{ 0x14561022, "AMD CCP-5a" },
				{ 0x14681022, "AMD CCP-5b" },
				};
				MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, sizeof(ccp_ids[0]),
				nitems(ccp_ids));

				static struct random_source random_ccp = {
				.rs_ident = "AMD CCP TRNG",
				.rs_source = RANDOM_PURE_CCP,
				.rs_read = random_ccp_read,
				};

				/*
				* ccp_populate_sglist() generates a scatter/gather list that covers the entire
				* crypto operation buffer.
				*/
				static int
				ccp_populate_sglist(struct sglist sg, struct cryptop crp)
				{
				int error;

				sglist_reset(sg);
				if (crp->crp_flags & CRYPTO_F_IMBUF)
				error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
				else if (crp->crp_flags & CRYPTO_F_IOV)
				error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
				else
				error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
				return (error);
				}

				/*
				* Handle a GCM request with an empty payload by performing the
				* operation in software. Derived from swcr_authenc().
				*/
				static void
				ccp_gcm_soft(struct ccp_session s, struct cryptop crp,
				struct cryptodesc crda, struct cryptodesc crde)
				{
				struct aes_gmac_ctx gmac_ctx;
				char block[GMAC_BLOCK_LEN];
				char digest[GMAC_DIGEST_LEN];
				char iv[AES_BLOCK_LEN];
				int i, len;

				/*
				* This assumes a 12-byte IV from the crp. See longer comment
				* above in ccp_gcm() for more details.
				*/
				if (crde->crd_flags & CRD_F_ENCRYPT) {
				if (crde->crd_flags & CRD_F_IV_EXPLICIT)
				memcpy(iv, crde->crd_iv, 12);
				else
				arc4rand(iv, 12, 0);
				} else {
				if (crde->crd_flags & CRD_F_IV_EXPLICIT)
				memcpy(iv, crde->crd_iv, 12);
				else
				crypto_copydata(crp->crp_flags, crp->crp_buf,
				crde->crd_inject, 12, iv);
				}
				(uint32_t )&iv[12] = htobe32(1);

				/* Initialize the MAC. */
				AES_GMAC_Init(&gmac_ctx);
				AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
				AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));

				/* MAC the AAD. */
				for (i = 0; i < crda->crd_len; i += sizeof(block)) {
				len = imin(crda->crd_len - i, sizeof(block));
				crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
				i, len, block);
				bzero(block + len, sizeof(block) - len);
				AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
				}

				/* Length block. */
				bzero(block, sizeof(block));
				((uint32_t )block)[1] = htobe32(crda->crd_len 8);
				AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
				AES_GMAC_Final(digest, &gmac_ctx);

				if (crde->crd_flags & CRD_F_ENCRYPT) {
				crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
				sizeof(digest), digest);
				crp->crp_etype = 0;
				} else {
				char digest2[GMAC_DIGEST_LEN];

				crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
				sizeof(digest2), digest2);
				if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
				crp->crp_etype = 0;
				else
				crp->crp_etype = EBADMSG;
				}
				crypto_done(crp);
				}

				static int
				ccp_probe(device_t dev)
				{
				struct pciid *ip;
				uint32_t id;

				id = pci_get_devid(dev);
				for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
				if (id == ip->devid) {
				device_set_desc(dev, ip->desc);
				return (0);
				}
				}
				return (ENXIO);
				}

				static void
				ccp_initialize_queues(struct ccp_softc *sc)
				{
				struct ccp_queue *qp;
				size_t i;

				for (i = 0; i < nitems(sc->queues); i++) {
				qp = &sc->queues[i];
				markjUnsubmitted Done Inline Actions s/SGLs/SGEs/ markj: s/SGLs/SGEs/

				qp->cq_softc = sc;
				qp->cq_qindex = i;
				mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
				/* XXX - arbitrarily chosen sizes */
				qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
				/* Two more SGEs than sg_crp to accommodate ipad. */
				qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
				qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
				}
				}

				static void
				ccp_free_queues(struct ccp_softc *sc)
				{
				struct ccp_queue *qp;
				size_t i;

				for (i = 0; i < nitems(sc->queues); i++) {
				qp = &sc->queues[i];

				mtx_destroy(&qp->cq_lock);
				sglist_free(qp->cq_sg_crp);
				sglist_free(qp->cq_sg_ulptx);
				sglist_free(qp->cq_sg_dst);
				}
				}

				static int
				ccp_attach(device_t dev)
				{
				struct ccp_softc *sc;
				int error;

				sc = device_get_softc(dev);
				sc->dev = dev;

				device_printf(dev, "XXX%s: sc=%p\n", __func__, sc);
				markjUnsubmitted Done Inline Actions There's a lot of printf("XXX...") in the driver, which ought to be cleaned up at some point. markj: There's a lot of printf("XXX...") in the driver, which ought to be cleaned up at some point.
				cemAuthorUnsubmitted Done Inline Actions Definitely. At one point I was keeping this stuff separate in a git stash but I guess it got merged in sometime. Some of it even leaks stuff like key information. cem: Definitely. At one point I was keeping this stuff separate in a git stash but I guess it got…

				sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
				markjUnsubmitted Not Done Inline Actions Not something to be addressed here, but crypto_get_driverid() should probably just use unr(9). :/ markj: Not something to be addressed here, but crypto_get_driverid() should probably just use unr(9).
				if (sc->cid < 0) {
				device_printf(dev, "could not get crypto driver id\n");
				return (ENXIO);
				markjUnsubmitted Done Inline Actions Since ccp_initialize_queue() is only used once, it be a bit neater to have a ccp_initialize_queues() instead and move this loop there. markj: Since ccp_initialize_queue() is only used once, it be a bit neater to have a…
				cemAuthorUnsubmitted Not Done Inline Actions Ok cem: Ok
				}

				error = ccp_hw_attach(dev);
				if (error != 0)
				return (error);

				mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);

				ccp_initialize_queues(sc);

				if (g_ccp_softc == NULL) {
				g_ccp_softc = sc;
				if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
				random_source_register(&random_ccp);
				}

				if ((sc->hw_features & VERSION_CAP_AES) != 0) {
				crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
				#if 0
				crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
				#endif
				}
				if ((sc->hw_features & VERSION_CAP_SHA) != 0) {
				crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
				crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
				}

				return (0);
				}

				static int
				ccp_detach(device_t dev)
				{
				struct ccp_softc *sc;
				int i;

				sc = device_get_softc(dev);

				mtx_lock(&sc->lock);
				for (i = 0; i < sc->nsessions; i++) {
				if (sc->sessions[i].active \|\| sc->sessions[i].pending != 0) {
				mtx_unlock(&sc->lock);
				return (EBUSY);
				}
				}
				sc->detaching = true;
				mtx_unlock(&sc->lock);

				crypto_unregister_all(sc->cid);
				if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
				random_source_deregister(&random_ccp);

				ccp_hw_detach(dev);
				ccp_free_queues(sc);

				if (g_ccp_softc == sc)
				g_ccp_softc = NULL;

				free(sc->sessions, M_CCP);
				mtx_destroy(&sc->lock);
				return (0);
				}

				static void
				ccp_init_hmac_digest(struct ccp_session s, int cri_alg, char key,
				int klen)
				{
				union authctx auth_ctx;
				struct auth_hash *axf;
				u_int i;

				/*
				* If the key is larger than the block size, use the digest of
				* the key as the key instead.
				*/
				axf = s->hmac.auth_hash;
				klen /= 8;
				if (klen > axf->blocksize) {
				axf->Init(&auth_ctx);
				axf->Update(&auth_ctx, key, klen);
				axf->Final(s->hmac.ipad, &auth_ctx);
				explicit_bzero(&auth_ctx, sizeof(auth_ctx));
				klen = axf->hashsize;
				} else
				memcpy(s->hmac.ipad, key, klen);

				memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
				memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);

				for (i = 0; i < axf->blocksize; i++) {
				s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
				s->hmac.opad[i] ^= HMAC_OPAD_VAL;
				}
				}

				static int
				ccp_aes_check_keylen(int alg, int klen)
				{

				switch (klen) {
				case 128:
				case 192:
				if (alg == CRYPTO_AES_XTS)
				return (EINVAL);
				break;
				case 256:
				break;
				case 512:
				if (alg != CRYPTO_AES_XTS)
				return (EINVAL);
				break;
				default:
				return (EINVAL);
				}
				return (0);
				}

				/*
				* Borrowed from cesa_prep_aes_key(). We should perhaps have a public
				* function to generate this instead.
				*
				* NB: The crypto engine wants the words in the decryption key in reverse
				* order.
				*/
				static void
				ccp_aes_getdeckey(void dec_key, const void enc_key, unsigned int kbits)
				{
				uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
				uint32_t *dkey;
				int i;

				rijndaelKeySetupEnc(ek, enc_key, kbits);
				dkey = dec_key;
				dkey += (kbits / 8) / 4;

				switch (kbits) {
				case 128:
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 10 + i]);
				break;
				case 192:
				for (i = 0; i < 2; i++)
				--dkey = htobe32(ek[4 11 + 2 + i]);
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 12 + i]);
				break;
				case 256:
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 13 + i]);
				for (i = 0; i < 4; i++)
				--dkey = htobe32(ek[4 14 + i]);
				break;
				}
				MPASS(dkey == dec_key);
				}

				static void
				ccp_aes_setkey(struct ccp_session s, int alg, const void key, int klen)
				{
				unsigned kbits;

				if (alg == CRYPTO_AES_XTS)
				kbits = klen / 2;
				else
				kbits = klen;

				switch (kbits) {
				case 128:
				s->blkcipher.cipher_type = CCP_AES_TYPE_128;
				break;
				case 192:
				s->blkcipher.cipher_type = CCP_AES_TYPE_192;
				break;
				case 256:
				s->blkcipher.cipher_type = CCP_AES_TYPE_256;
				break;
				default:
				panic("should not get here");
				}

				s->blkcipher.key_len = klen / 8;
				memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
				switch (alg) {
				case CRYPTO_AES_XTS:
				ccp_aes_getdeckey(s->blkcipher.deckey, key, kbits);
				break;
				}
				}

				static int
				ccp_newsession(device_t dev, uint32_t sidp, struct cryptoini cri)
				{
				struct ccp_softc *sc;
				struct ccp_session *s;
				struct auth_hash *auth_hash;
				struct cryptoini c, hash, *cipher;
				enum ccp_aes_mode cipher_mode;
				unsigned auth_mode, iv_len;
				unsigned partial_digest_len;
				unsigned q;
				int error, i, sess;
				bool gcm_hash;

				if (sidp == NULL \|\| cri == NULL)
				return (EINVAL);

				gcm_hash = false;
				cipher = NULL;
				hash = NULL;
				auth_hash = NULL;
				/* XXX reconcile auth_mode with use by ccp_sha */
				auth_mode = 0;
				cipher_mode = CCP_AES_MODE_ECB;
				markjUnsubmitted Not Done Inline Actions Seems like an odd initialization default, especially since we don't set cipher_mode in the AES-XTS case. We don't register support for AES-XTS, so this isn't a major problem at the moment, but it looks weird. markj: Seems like an odd initialization default, especially since we don't set cipher_mode in the AES…
				cemAuthorUnsubmitted Not Done Inline Actions I suppose. Mostly it just needs to be initialized to something other than CCP_AES_MODE_GCTR. Ideally AES-XTS will be added at some point, and that will make it more clear. We could `#if 0` the the XTS case of the outer switch, making it return EINVAL? (Though as you point out, it should never be invoked as we don't register support.) cem: I suppose. Mostly it just needs to be initialized to something other than CCP_AES_MODE_GCTR.
				cemAuthorUnsubmitted Not Done Inline Actions The other reason to use ECB is that it is so clearly a bogus value. The hardware will do it, but OCF and this driver don't support it (for good reason). cem: The other reason to use ECB is that it is so clearly a bogus value. The hardware will do it…
				iv_len = 0;
				partial_digest_len = 0;
				for (c = cri; c != NULL; c = c->cri_next) {
				switch (c->cri_alg) {
				case CRYPTO_SHA1_HMAC:
				case CRYPTO_SHA2_256_HMAC:
				case CRYPTO_SHA2_384_HMAC:
				case CRYPTO_SHA2_512_HMAC:
				case CRYPTO_AES_128_NIST_GMAC:
				case CRYPTO_AES_192_NIST_GMAC:
				case CRYPTO_AES_256_NIST_GMAC:
				if (hash)
				return (EINVAL);
				hash = c;
				switch (c->cri_alg) {
				case CRYPTO_SHA1_HMAC:
				auth_hash = &auth_hash_hmac_sha1;
				auth_mode = SHA1;
				partial_digest_len = SHA1_HASH_LEN;
				break;
				case CRYPTO_SHA2_256_HMAC:
				auth_hash = &auth_hash_hmac_sha2_256;
				auth_mode = SHA2_256;
				partial_digest_len = SHA2_256_HASH_LEN;
				break;
				case CRYPTO_SHA2_384_HMAC:
				auth_hash = &auth_hash_hmac_sha2_384;
				auth_mode = SHA2_384;
				partial_digest_len = SHA2_512_HASH_LEN;
				break;
				case CRYPTO_SHA2_512_HMAC:
				auth_hash = &auth_hash_hmac_sha2_512;
				auth_mode = SHA2_512;
				partial_digest_len = SHA2_512_HASH_LEN;
				break;
				case CRYPTO_AES_128_NIST_GMAC:
				case CRYPTO_AES_192_NIST_GMAC:
				case CRYPTO_AES_256_NIST_GMAC:
				gcm_hash = true;
				#if 0
				auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
				#endif
				break;
				}
				break;
				case CRYPTO_AES_CBC:
				case CRYPTO_AES_ICM:
				case CRYPTO_AES_NIST_GCM_16:
				case CRYPTO_AES_XTS:
				if (cipher)
				return (EINVAL);
				cipher = c;
				switch (c->cri_alg) {
				case CRYPTO_AES_CBC:
				cipher_mode = CCP_AES_MODE_CBC;
				iv_len = AES_BLOCK_LEN;
				break;
				case CRYPTO_AES_ICM:
				cipher_mode = CCP_AES_MODE_CTR;
				iv_len = AES_BLOCK_LEN;
				break;
				case CRYPTO_AES_NIST_GCM_16:
				cipher_mode = CCP_AES_MODE_GCTR;
				iv_len = AES_GCM_IV_LEN;
				break;
				case CRYPTO_AES_XTS:
				#if 0
				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
				#endif
				iv_len = AES_BLOCK_LEN;
				break;
				}
				if (c->cri_key != NULL) {
				error = ccp_aes_check_keylen(c->cri_alg,
				c->cri_klen);
				if (error != 0)
				return (error);
				}
				break;
				default:
				return (EINVAL);
				}
				}
				if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR))
				return (EINVAL);
				if (hash == NULL && cipher == NULL)
				return (EINVAL);
				if (hash != NULL && hash->cri_key == NULL)
				return (EINVAL);

				sc = device_get_softc(dev);
				mtx_lock(&sc->lock);
				if (sc->detaching) {
				mtx_unlock(&sc->lock);
				return (ENXIO);
				}
				sess = -1;
				for (i = 0; i < sc->nsessions; i++) {
				if (!sc->sessions[i].active && sc->sessions[i].pending == 0) {
				sess = i;
				break;
				}
				}
				if (sess == -1) {
				s = malloc(sizeof(s) (sc->nsessions + 1), M_CCP,
				M_NOWAIT \| M_ZERO);
				if (s == NULL) {
				mtx_unlock(&sc->lock);
				return (ENOMEM);
				}
				if (sc->sessions != NULL)
				memcpy(s, sc->sessions, sizeof(s) sc->nsessions);
				sess = sc->nsessions;
				free(sc->sessions, M_CCP);
				sc->sessions = s;
				sc->nsessions++;
				}

				s = &sc->sessions[sess];

				/* Just grab the first usable queue for now. */
				for (q = 0; q < nitems(sc->queues); q++)
				if ((sc->valid_queues & (1 << q)) != 0)
				break;
				if (q == nitems(sc->queues)) {
				mtx_unlock(&sc->lock);
				return (ENXIO);
				}
				s->queue = q;

				if (gcm_hash)
				s->mode = GCM;
				else if (hash != NULL && cipher != NULL)
				s->mode = AUTHENC;
				else if (hash != NULL)
				s->mode = HMAC;
				else {
				MPASS(cipher != NULL);
				s->mode = BLKCIPHER;
				}
				if (gcm_hash) {
				if (hash->cri_mlen == 0)
				s->gmac.hash_len = AES_GMAC_HASH_LEN;
				else
				s->gmac.hash_len = hash->cri_mlen;
				} else if (hash != NULL) {
				s->hmac.auth_hash = auth_hash;
				s->hmac.auth_mode = auth_mode;
				s->hmac.partial_digest_len = partial_digest_len;
				if (hash->cri_mlen == 0)
				s->hmac.hash_len = auth_hash->hashsize;
				else
				s->hmac.hash_len = hash->cri_mlen;
				ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
				hash->cri_klen);
				}
				if (cipher != NULL) {
				s->blkcipher.cipher_mode = cipher_mode;
				s->blkcipher.iv_len = iv_len;
				if (cipher->cri_key != NULL)
				ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
				cipher->cri_klen);
				}

				s->active = true;
				mtx_unlock(&sc->lock);

				*sidp = sess;
				return (0);
				}

				static int
				ccp_freesession(device_t dev, uint64_t tid)
				{
				struct ccp_softc *sc;
				uint32_t sid;
				int error;

				sc = device_get_softc(dev);
				sid = CRYPTO_SESID2LID(tid);
				mtx_lock(&sc->lock);
				if (sid >= sc->nsessions \|\| !sc->sessions[sid].active)
				error = EINVAL;
				else {
				if (sc->sessions[sid].pending != 0)
				device_printf(dev,
				"session %d freed with %d pending requests\n", sid,
				sc->sessions[sid].pending);
				sc->sessions[sid].active = false;
				error = 0;
				}
				mtx_unlock(&sc->lock);
				return (error);
				}

				static int
				ccp_process(device_t dev, struct cryptop *crp, int hint)
				{
				struct ccp_softc *sc;
				struct ccp_queue *qp;
				struct ccp_session *s;
				struct cryptodesc crd, crda, *crde;
				uint32_t sid;
				int error;
				bool qpheld;

				qpheld = false;
				qp = NULL;
				if (crp == NULL)
				return (EINVAL);

				crd = crp->crp_desc;
				sid = CRYPTO_SESID2LID(crp->crp_sid);
				sc = device_get_softc(dev);
				mtx_lock(&sc->lock);
				if (sid >= sc->nsessions \|\| !sc->sessions[sid].active) {
				mtx_unlock(&sc->lock);
				error = EINVAL;
				goto out;
				}

				s = &sc->sessions[sid];
				qp = &sc->queues[s->queue];
				mtx_unlock(&sc->lock);
				error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
				if (error != 0)
				goto out;
				qpheld = true;

				error = ccp_populate_sglist(qp->cq_sg_crp, crp);
				if (error != 0)
				goto out;

				switch (s->mode) {
				case HMAC:
				if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
				ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
				crd->crd_klen);
				error = ccp_hmac(qp, s, crp);
				break;
				case BLKCIPHER:
				if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crd->crd_alg,
				crd->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crd->crd_alg, crd->crd_key,
				crd->crd_klen);
				}
				error = ccp_blkcipher(qp, s, crp);
				break;
				case AUTHENC:
				error = 0;
				switch (crd->crd_alg) {
				case CRYPTO_AES_CBC:
				case CRYPTO_AES_ICM:
				case CRYPTO_AES_XTS:
				/* Only encrypt-then-authenticate supported. */
				crde = crd;
				crda = crd->crd_next;
				if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
				error = EINVAL;
				break;
				}
				s->cipher_first = true;
				break;
				default:
				crda = crd;
				crde = crd->crd_next;
				if (crde->crd_flags & CRD_F_ENCRYPT) {
				error = EINVAL;
				break;
				}
				s->cipher_first = false;
				break;
				}
				if (error != 0)
				break;
				if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
				ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
				crda->crd_klen);
				if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crde->crd_alg,
				crde->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
				crde->crd_klen);
				}
				error = ccp_authenc(qp, s, crp, crda, crde);
				break;
				case GCM:
				error = 0;
				if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
				crde = crd;
				crda = crd->crd_next;
				s->cipher_first = true;
				} else {
				crda = crd;
				crde = crd->crd_next;
				s->cipher_first = false;
				}
				if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
				error = ccp_aes_check_keylen(crde->crd_alg,
				crde->crd_klen);
				if (error != 0)
				break;
				ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
				crde->crd_klen);
				}
				if (crde->crd_len == 0) {
				mtx_unlock(&qp->cq_lock);
				ccp_gcm_soft(s, crp, crda, crde);
				return (0);
				}
				error = ccp_gcm(qp, s, crp, crda, crde);
				break;
				}

				if (error == 0)
				s->pending++;

				out:
				if (qpheld) {
				if (error != 0)
				ccp_queue_abort(qp);
				else
				ccp_queue_release(qp);
				}

				if (error != 0) {
				device_printf(dev, "XXX %s: early error:%d\n", __func__,
				error);
				crp->crp_etype = error;
				crypto_done(crp);
				}
				return (0);
				}

				static device_method_t ccp_methods[] = {
				DEVMETHOD(device_probe, ccp_probe),
				DEVMETHOD(device_attach, ccp_attach),
				DEVMETHOD(device_detach, ccp_detach),

				DEVMETHOD(cryptodev_newsession, ccp_newsession),
				DEVMETHOD(cryptodev_freesession, ccp_freesession),
				DEVMETHOD(cryptodev_process, ccp_process),

				DEVMETHOD_END
				};

				static driver_t ccp_driver = {
				"ccp",
				ccp_methods,
				sizeof(struct ccp_softc)
				markjUnsubmitted Done Inline Actions "error" is write-only. markj: "error" is write-only.
				};

				static devclass_t ccp_devclass;
				DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
				MODULE_VERSION(ccp, 1);
				MODULE_DEPEND(ccp, crypto, 1, 1, 1);
				MODULE_DEPEND(ccp, random_device, 1, 1, 1);

				static int
				ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
				{
				struct ccp_softc *sc;

				mtx_assert(&qp->cq_lock, MA_OWNED);
				sc = qp->cq_softc;

				if (n < 1 \|\| n >= (1 << sc->ring_size_order))
				return (EINVAL);

				markjUnsubmitted Not Done Inline Actions This function and the one below appear to be unused. markj: This function and the one below appear to be unused.
				cemAuthorUnsubmitted Not Done Inline Actions Yeah, I was thinking it would be nice to have a kernel API like ioat that doesn't go through opencrypto, but don't have that very concrete yet. cem: Yeah, I was thinking it would be nice to have a kernel API like ioat that doesn't go through…
				while (true) {
				if (ccp_queue_get_ring_space(qp) >= n)
				return (0);
				if ((mflags & M_WAITOK) == 0)
				return (EAGAIN);
				qp->cq_waiting = true;
				msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
				}
				}

				int
				ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
				{
				int error;

				mtx_lock(&qp->cq_lock);
				qp->cq_acq_tail = qp->cq_tail;
				error = ccp_queue_reserve_space(qp, n, mflags);
				if (error != 0)
				mtx_unlock(&qp->cq_lock);
				return (error);
				}

				void
				ccp_queue_release(struct ccp_queue *qp)
				{

				mtx_assert(&qp->cq_lock, MA_OWNED);
				if (qp->cq_tail != qp->cq_acq_tail) {
				wmb();
				ccp_queue_write_tail(qp);
				}
				mtx_unlock(&qp->cq_lock);
				}

				void
				ccp_queue_abort(struct ccp_queue *qp)
				{
				unsigned i;

				mtx_assert(&qp->cq_lock, MA_OWNED);

				/* Wipe out back any descriptors associated with this aborted txn. */
				for (i = qp->cq_acq_tail; i != qp->cq_tail;
				i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
				memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
				}
				qp->cq_tail = qp->cq_acq_tail;

				mtx_unlock(&qp->cq_lock);
				}

				#ifdef DDB
				#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)
				#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)
				static void
				db_show_ccp_sc(struct ccp_softc *sc)
				{

				db_printf("ccp softc at %p\n", sc);
				db_printf(" cid: %d\n", (int)sc->cid);
				db_printf(" nsessions: %d\n", sc->nsessions);

				db_printf(" lock: ");
				db_show_lock(&sc->lock);

				db_printf(" detaching: %d\n", (int)sc->detaching);
				db_printf(" ring_size_order: %u\n", sc->ring_size_order);

				db_printf(" hw_version: %d\n", (int)sc->hw_version);
				db_printf(" hw_features: %b\n", (int)sc->hw_features,
				"\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
				"\11SHA\0103DES\07AES");

				db_printf(" hw status:\n");
				db_ccp_show_hw(sc);
				}

				static void
				db_show_ccp_qp(struct ccp_queue *qp)
				{

				db_printf(" lock: ");
				db_show_lock(&qp->cq_lock);

				#if 0
				db_printf(" head: %u\n", sc->head);
				db_printf(" tail: %u\n", sc->tail);
				db_printf(" hw_head: %u\n", sc->hw_head);
				db_printf(" last_seen: 0x%lx\n", sc->last_seen);
				db_printf(" ring: %p\n", sc->ring);
				db_printf(" descriptors: %p\n", sc->hw_desc_ring);
				db_printf(" descriptors (phys): 0x%jx\n",
				(uintmax_t)sc->hw_desc_bus_addr);

				db_printf(" ring[%u] (tail):\n", sc->tail %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->tail)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->tail));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->tail)->generic.next);

				db_printf(" ring[%u] (head - 1):\n", (sc->head - 1) %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head - 1)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->head - 1));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->head - 1)->generic.next);

				db_printf(" ring[%u] (head):\n", (sc->head) %
				(1 << sc->ring_size_order));
				db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head)->id);
				db_printf(" addr: 0x%lx\n",
				RING_PHYS_ADDR(sc, sc->head));
				db_printf(" next: 0x%lx\n",
				ioat_get_descriptor(sc, sc->head)->generic.next);

				for (idx = 0; idx < (1 << sc->ring_size_order); idx++)
				if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK)
				== RING_PHYS_ADDR(sc, idx))
				db_printf(" ring[%u] == hardware tail\n", idx);
				#endif

				db_printf(" hw status:\n");
				db_ccp_show_queue_hw(qp);
				}

				DB_SHOW_COMMAND(ccp, db_show_ccp)
				{
				struct ccp_softc *sc;
				unsigned unit, qindex;

				if (!have_addr)
				goto usage;

				unit = (unsigned)addr;

				sc = devclass_get_softc(ccp_devclass, unit);
				if (sc == NULL) {
				db_printf("No such device ccp%u\n", unit);
				goto usage;
				}

				if (count == -1) {
				db_show_ccp_sc(sc);
				return;
				}

				qindex = (unsigned)count;
				if (qindex >= nitems(sc->queues)) {
				db_printf("No such queue %u\n", qindex);
				goto usage;
				}
				db_show_ccp_qp(&sc->queues[qindex]);
				return;

				usage:
				db_printf("usage: show ccp <unit>[,<qindex>]\n");
				return;
				}
				#endif /* DDB */

mitigations.7: explain RELROClosedPublicActions