D25457.id73676.diff
No OneTemporary
Actions

Size

28 KB

Referenced Files

None

Subscribers

None

D25457.id73676.diff
View Options

	Index: head/sys/dev/cxgbe/crypto/t4_crypto.c
	===================================================================
	--- head/sys/dev/cxgbe/crypto/t4_crypto.c
	+++ head/sys/dev/cxgbe/crypto/t4_crypto.c
	@@ -170,8 +170,9 @@
	};

	struct ccr_session {
	- bool active;
	+#ifdef INVARIANTS
	int pending;
	+#endif
	enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
	struct ccr_port *port;
	union {
	@@ -180,16 +181,7 @@
	struct ccr_session_ccm_mac ccm_mac;
	};
	struct ccr_session_blkcipher blkcipher;
	-};
	-
	-struct ccr_softc {
	- struct adapter *adapter;
	- device_t dev;
	- uint32_t cid;
	struct mtx lock;
	- bool detaching;
	- struct ccr_port ports[MAX_NPORTS];
	- u_int port_mask;

	/*
	* Pre-allocate S/G lists used when preparing a work request.
	@@ -205,7 +197,17 @@
	struct sglist *sg_output;
	struct sglist *sg_ulptx;
	struct sglist *sg_dsgl;
	+};

	+struct ccr_softc {
	+ struct adapter *adapter;
	+ device_t dev;
	+ uint32_t cid;
	+ struct mtx lock;
	+ bool detaching;
	+ struct ccr_port ports[MAX_NPORTS];
	+ u_int port_mask;
	+
	/*
	* Pre-allocate a dummy output buffer for the IV and AAD for
	* AEAD requests.
	@@ -214,24 +216,23 @@
	struct sglist *sg_iv_aad;

	/* Statistics. */
	- uint64_t stats_blkcipher_encrypt;
	- uint64_t stats_blkcipher_decrypt;
	- uint64_t stats_hash;
	- uint64_t stats_hmac;
	- uint64_t stats_eta_encrypt;
	- uint64_t stats_eta_decrypt;
	- uint64_t stats_gcm_encrypt;
	- uint64_t stats_gcm_decrypt;
	- uint64_t stats_ccm_encrypt;
	- uint64_t stats_ccm_decrypt;
	- uint64_t stats_wr_nomem;
	- uint64_t stats_inflight;
	- uint64_t stats_mac_error;
	- uint64_t stats_pad_error;
	- uint64_t stats_bad_session;
	- uint64_t stats_sglist_error;
	- uint64_t stats_process_error;
	- uint64_t stats_sw_fallback;
	+ counter_u64_t stats_blkcipher_encrypt;
	+ counter_u64_t stats_blkcipher_decrypt;
	+ counter_u64_t stats_hash;
	+ counter_u64_t stats_hmac;
	+ counter_u64_t stats_eta_encrypt;
	+ counter_u64_t stats_eta_decrypt;
	+ counter_u64_t stats_gcm_encrypt;
	+ counter_u64_t stats_gcm_decrypt;
	+ counter_u64_t stats_ccm_encrypt;
	+ counter_u64_t stats_ccm_decrypt;
	+ counter_u64_t stats_wr_nomem;
	+ counter_u64_t stats_inflight;
	+ counter_u64_t stats_mac_error;
	+ counter_u64_t stats_pad_error;
	+ counter_u64_t stats_sglist_error;
	+ counter_u64_t stats_process_error;
	+ counter_u64_t stats_sw_fallback;
	};

	/*
	@@ -307,8 +308,7 @@
	}

	static void
	-ccr_write_phys_dsgl(struct ccr_softc sc, struct ccr_session s, void *dst,
	- int nsegs)
	+ccr_write_phys_dsgl(struct ccr_session s, void dst, int nsegs)
	{
	struct sglist *sg;
	struct cpl_rx_phys_dsgl *cpl;
	@@ -317,7 +317,7 @@
	size_t seglen;
	u_int i, j;

	- sg = sc->sg_dsgl;
	+ sg = s->sg_dsgl;
	cpl = dst;
	cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) \|
	V_CPL_RX_PHYS_DSGL_ISRDMA(0));
	@@ -366,14 +366,14 @@
	}

	static void
	-ccr_write_ulptx_sgl(struct ccr_softc sc, void dst, int nsegs)
	+ccr_write_ulptx_sgl(struct ccr_session s, void dst, int nsegs)
	{
	struct ulptx_sgl *usgl;
	struct sglist *sg;
	struct sglist_seg *ss;
	int i;

	- sg = sc->sg_ulptx;
	+ sg = s->sg_ulptx;
	MPASS(nsegs == sg->sg_nseg);
	ss = &sg->sg_segs[0];
	usgl = dst;
	@@ -496,12 +496,12 @@
	sgl_len = 0;
	} else {
	imm_len = 0;
	- sglist_reset(sc->sg_ulptx);
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ sglist_reset(s->sg_ulptx);
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	- sgl_nsegs = sc->sg_ulptx->sg_nseg;
	+ sgl_nsegs = s->sg_ulptx->sg_nseg;
	sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
	}

	@@ -510,7 +510,7 @@
	return (EFBIG);
	wr = alloc_wrqe(wr_len, s->port->txq);
	if (wr == NULL) {
	- sc->stats_wr_nomem++;
	+ counter_u64_add(sc->stats_wr_nomem, 1);
	return (ENOMEM);
	}
	crwr = wrtod(wr);
	@@ -564,7 +564,7 @@
	crypto_copydata(crp, crp->crp_payload_start,
	crp->crp_payload_length, dst);
	else
	- ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
	+ ccr_write_ulptx_sgl(s, dst, sgl_nsegs);

	/* XXX: TODO backpressure */
	t4_wrq_tx(sc->adapter, wr);
	@@ -620,16 +620,16 @@
	else
	op_type = CHCR_DECRYPT_OP;

	- sglist_reset(sc->sg_dsgl);
	+ sglist_reset(s->sg_dsgl);
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_payload_output_start, crp->crp_payload_length);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	- dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
	+ dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
	return (EFBIG);
	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
	@@ -650,12 +650,12 @@
	sgl_len = 0;
	} else {
	imm_len = 0;
	- sglist_reset(sc->sg_ulptx);
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ sglist_reset(s->sg_ulptx);
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	- sgl_nsegs = sc->sg_ulptx->sg_nseg;
	+ sgl_nsegs = s->sg_ulptx->sg_nseg;
	sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
	}

	@@ -665,7 +665,7 @@
	return (EFBIG);
	wr = alloc_wrqe(wr_len, s->port->txq);
	if (wr == NULL) {
	- sc->stats_wr_nomem++;
	+ counter_u64_add(sc->stats_wr_nomem, 1);
	return (ENOMEM);
	}
	crwr = wrtod(wr);
	@@ -737,7 +737,7 @@
	}

	dst = (char *)(crwr + 1) + kctx_len;
	- ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
	+ ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
	memcpy(dst, iv, iv_len);
	dst += iv_len;
	@@ -745,7 +745,7 @@
	crypto_copydata(crp, crp->crp_payload_start,
	crp->crp_payload_length, dst);
	else
	- ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
	+ ccr_write_ulptx_sgl(s, dst, sgl_nsegs);

	/* XXX: TODO backpressure */
	t4_wrq_tx(sc->adapter, wr);
	@@ -846,30 +846,30 @@
	MAX_REQUEST_SIZE)
	return (EFBIG);
	}
	- sglist_reset(sc->sg_dsgl);
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0,
	+ sglist_reset(s->sg_dsgl);
	+ error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0,
	iv_len + crp->crp_aad_length);
	if (error)
	return (error);
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_payload_output_start, crp->crp_payload_length);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_ENCRYPT_OP) {
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_digest_start, hash_size_in_response);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
	+ dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
	return (EFBIG);
	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
	@@ -916,29 +916,29 @@
	sgl_len = 0;
	} else {
	imm_len = 0;
	- sglist_reset(sc->sg_ulptx);
	+ sglist_reset(s->sg_ulptx);
	if (crp->crp_aad_length != 0) {
	if (crp->crp_aad != NULL)
	- error = sglist_append(sc->sg_ulptx,
	+ error = sglist_append(s->sg_ulptx,
	crp->crp_aad, crp->crp_aad_length);
	else
	- error = sglist_append_sglist(sc->sg_ulptx,
	- sc->sg_input, crp->crp_aad_start,
	+ error = sglist_append_sglist(s->sg_ulptx,
	+ s->sg_input, crp->crp_aad_start,
	crp->crp_aad_length);
	if (error)
	return (error);
	}
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_DECRYPT_OP) {
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- sgl_nsegs = sc->sg_ulptx->sg_nseg;
	+ sgl_nsegs = s->sg_ulptx->sg_nseg;
	sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
	}

	@@ -966,7 +966,7 @@
	return (EFBIG);
	wr = alloc_wrqe(wr_len, s->port->txq);
	if (wr == NULL) {
	- sc->stats_wr_nomem++;
	+ counter_u64_add(sc->stats_wr_nomem, 1);
	return (ENOMEM);
	}
	crwr = wrtod(wr);
	@@ -1048,7 +1048,7 @@
	memcpy(dst, s->hmac.pads, iopad_size * 2);

	dst = (char *)(crwr + 1) + kctx_len;
	- ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
	+ ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
	memcpy(dst, iv, iv_len);
	dst += iv_len;
	@@ -1068,7 +1068,7 @@
	crypto_copydata(crp, crp->crp_digest_start,
	hash_size_in_response, dst);
	} else
	- ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
	+ ccr_write_ulptx_sgl(s, dst, sgl_nsegs);

	/* XXX: TODO backpressure */
	t4_wrq_tx(sc->adapter, wr);
	@@ -1168,30 +1168,30 @@
	MAX_REQUEST_SIZE)
	return (EFBIG);
	}
	- sglist_reset(sc->sg_dsgl);
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
	+ sglist_reset(s->sg_dsgl);
	+ error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
	crp->crp_aad_length);
	if (error)
	return (error);
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_payload_output_start, crp->crp_payload_length);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_ENCRYPT_OP) {
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_digest_start, hash_size_in_response);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
	+ dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
	return (EFBIG);
	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
	@@ -1225,29 +1225,29 @@
	sgl_len = 0;
	} else {
	imm_len = 0;
	- sglist_reset(sc->sg_ulptx);
	+ sglist_reset(s->sg_ulptx);
	if (crp->crp_aad_length != 0) {
	if (crp->crp_aad != NULL)
	- error = sglist_append(sc->sg_ulptx,
	+ error = sglist_append(s->sg_ulptx,
	crp->crp_aad, crp->crp_aad_length);
	else
	- error = sglist_append_sglist(sc->sg_ulptx,
	- sc->sg_input, crp->crp_aad_start,
	+ error = sglist_append_sglist(s->sg_ulptx,
	+ s->sg_input, crp->crp_aad_start,
	crp->crp_aad_length);
	if (error)
	return (error);
	}
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_DECRYPT_OP) {
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- sgl_nsegs = sc->sg_ulptx->sg_nseg;
	+ sgl_nsegs = s->sg_ulptx->sg_nseg;
	sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
	}

	@@ -1274,7 +1274,7 @@
	return (EFBIG);
	wr = alloc_wrqe(wr_len, s->port->txq);
	if (wr == NULL) {
	- sc->stats_wr_nomem++;
	+ counter_u64_add(sc->stats_wr_nomem, 1);
	return (ENOMEM);
	}
	crwr = wrtod(wr);
	@@ -1339,7 +1339,7 @@
	memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);

	dst = (char *)(crwr + 1) + kctx_len;
	- ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
	+ ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
	memcpy(dst, iv, iv_len);
	dst += iv_len;
	@@ -1359,7 +1359,7 @@
	crypto_copydata(crp, crp->crp_digest_start,
	hash_size_in_response, dst);
	} else
	- ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
	+ ccr_write_ulptx_sgl(s, dst, sgl_nsegs);

	/* XXX: TODO backpressure */
	t4_wrq_tx(sc->adapter, wr);
	@@ -1642,30 +1642,30 @@
	MAX_REQUEST_SIZE)
	return (EFBIG);
	}
	- sglist_reset(sc->sg_dsgl);
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
	+ sglist_reset(s->sg_dsgl);
	+ error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
	aad_len);
	if (error)
	return (error);
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_payload_output_start, crp->crp_payload_length);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_ENCRYPT_OP) {
	if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
	crp->crp_digest_start, hash_size_in_response);
	else
	- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
	+ dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
	return (EFBIG);
	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
	@@ -1701,29 +1701,29 @@
	/* Block 0 is passed as immediate data. */
	imm_len = b0_len;

	- sglist_reset(sc->sg_ulptx);
	+ sglist_reset(s->sg_ulptx);
	if (crp->crp_aad_length != 0) {
	if (crp->crp_aad != NULL)
	- error = sglist_append(sc->sg_ulptx,
	+ error = sglist_append(s->sg_ulptx,
	crp->crp_aad, crp->crp_aad_length);
	else
	- error = sglist_append_sglist(sc->sg_ulptx,
	- sc->sg_input, crp->crp_aad_start,
	+ error = sglist_append_sglist(s->sg_ulptx,
	+ s->sg_input, crp->crp_aad_start,
	crp->crp_aad_length);
	if (error)
	return (error);
	}
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_payload_start, crp->crp_payload_length);
	if (error)
	return (error);
	if (op_type == CHCR_DECRYPT_OP) {
	- error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
	+ error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
	crp->crp_digest_start, hash_size_in_response);
	if (error)
	return (error);
	}
	- sgl_nsegs = sc->sg_ulptx->sg_nseg;
	+ sgl_nsegs = s->sg_ulptx->sg_nseg;
	sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
	}

	@@ -1745,7 +1745,7 @@
	return (EFBIG);
	wr = alloc_wrqe(wr_len, s->port->txq);
	if (wr == NULL) {
	- sc->stats_wr_nomem++;
	+ counter_u64_add(sc->stats_wr_nomem, 1);
	return (ENOMEM);
	}
	crwr = wrtod(wr);
	@@ -1809,7 +1809,7 @@
	s->blkcipher.enckey, s->blkcipher.key_len);

	dst = (char *)(crwr + 1) + kctx_len;
	- ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
	+ ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
	memcpy(dst, iv, iv_len);
	dst += iv_len;
	@@ -1847,7 +1847,7 @@
	idata->len = htobe32(0);
	dst = (void *)(idata + 1);
	}
	- ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
	+ ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
	}

	/* XXX: TODO backpressure */
	@@ -2054,47 +2054,50 @@
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "statistics");
	children = SYSCTL_CHILDREN(oid);

	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
	- &sc->stats_hash, 0, "Hash requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
	- &sc->stats_hmac, 0, "HMAC requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD,
	- &sc->stats_blkcipher_encrypt, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
	+ &sc->stats_hash, "Hash requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
	+ &sc->stats_hmac, "HMAC requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt",
	+ CTLFLAG_RD, &sc->stats_blkcipher_encrypt,
	"Cipher encryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD,
	- &sc->stats_blkcipher_decrypt, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt",
	+ CTLFLAG_RD, &sc->stats_blkcipher_decrypt,
	"Cipher decryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD,
	- &sc->stats_eta_encrypt, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt",
	+ CTLFLAG_RD, &sc->stats_eta_encrypt,
	"Combined AES+HMAC encryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD,
	- &sc->stats_eta_decrypt, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt",
	+ CTLFLAG_RD, &sc->stats_eta_decrypt,
	"Combined AES+HMAC decryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD,
	- &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD,
	- &sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD,
	- &sc->stats_ccm_encrypt, 0, "AES-CCM encryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD,
	- &sc->stats_ccm_decrypt, 0, "AES-CCM decryption requests submitted");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
	- &sc->stats_wr_nomem, 0, "Work request memory allocation failures");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
	- &sc->stats_inflight, 0, "Requests currently pending");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
	- &sc->stats_mac_error, 0, "MAC errors");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
	- &sc->stats_pad_error, 0, "Padding errors");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD,
	- &sc->stats_bad_session, 0, "Requests with invalid session ID");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD,
	- &sc->stats_sglist_error, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt",
	+ CTLFLAG_RD, &sc->stats_gcm_encrypt,
	+ "AES-GCM encryption requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt",
	+ CTLFLAG_RD, &sc->stats_gcm_decrypt,
	+ "AES-GCM decryption requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt",
	+ CTLFLAG_RD, &sc->stats_ccm_encrypt,
	+ "AES-CCM encryption requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt",
	+ CTLFLAG_RD, &sc->stats_ccm_decrypt,
	+ "AES-CCM decryption requests submitted");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
	+ &sc->stats_wr_nomem, "Work request memory allocation failures");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
	+ &sc->stats_inflight, "Requests currently pending");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
	+ &sc->stats_mac_error, "MAC errors");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
	+ &sc->stats_pad_error, "Padding errors");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error",
	+ CTLFLAG_RD, &sc->stats_sglist_error,
	"Requests for which DMA mapping failed");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD,
	- &sc->stats_process_error, 0, "Requests failed during queueing");
	- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD,
	- &sc->stats_sw_fallback, 0,
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error",
	+ CTLFLAG_RD, &sc->stats_process_error,
	+ "Requests failed during queueing");
	+ SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback",
	+ CTLFLAG_RD, &sc->stats_sw_fallback,
	"Requests processed by falling back to software");

	/*
	@@ -2157,12 +2160,25 @@
	sc->adapter->ccr_softc = sc;

	mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
	- sc->sg_input = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
	- sc->sg_output = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
	- sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
	- sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK);
	sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK);
	sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
	+ sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_hash = counter_u64_alloc(M_WAITOK);
	+ sc->stats_hmac = counter_u64_alloc(M_WAITOK);
	+ sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK);
	+ sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK);
	+ sc->stats_inflight = counter_u64_alloc(M_WAITOK);
	+ sc->stats_mac_error = counter_u64_alloc(M_WAITOK);
	+ sc->stats_pad_error = counter_u64_alloc(M_WAITOK);
	+ sc->stats_sglist_error = counter_u64_alloc(M_WAITOK);
	+ sc->stats_process_error = counter_u64_alloc(M_WAITOK);
	+ sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK);
	ccr_sysctls(sc);

	return (0);
	@@ -2182,12 +2198,25 @@
	crypto_unregister_all(sc->cid);

	mtx_destroy(&sc->lock);
	+ counter_u64_free(sc->stats_blkcipher_encrypt);
	+ counter_u64_free(sc->stats_blkcipher_decrypt);
	+ counter_u64_free(sc->stats_hash);
	+ counter_u64_free(sc->stats_hmac);
	+ counter_u64_free(sc->stats_eta_encrypt);
	+ counter_u64_free(sc->stats_eta_decrypt);
	+ counter_u64_free(sc->stats_gcm_encrypt);
	+ counter_u64_free(sc->stats_gcm_decrypt);
	+ counter_u64_free(sc->stats_ccm_encrypt);
	+ counter_u64_free(sc->stats_ccm_decrypt);
	+ counter_u64_free(sc->stats_wr_nomem);
	+ counter_u64_free(sc->stats_inflight);
	+ counter_u64_free(sc->stats_mac_error);
	+ counter_u64_free(sc->stats_pad_error);
	+ counter_u64_free(sc->stats_sglist_error);
	+ counter_u64_free(sc->stats_process_error);
	+ counter_u64_free(sc->stats_sw_fallback);
	sglist_free(sc->sg_iv_aad);
	free(sc->iv_aad_buf, M_CCR);
	- sglist_free(sc->sg_dsgl);
	- sglist_free(sc->sg_ulptx);
	- sglist_free(sc->sg_output);
	- sglist_free(sc->sg_input);
	sc->adapter->ccr_softc = NULL;
	return (0);
	}
	@@ -2459,6 +2488,16 @@
	return (best);
	}

	+static void
	+ccr_delete_session(struct ccr_session *s)
	+{
	+ sglist_free(s->sg_input);
	+ sglist_free(s->sg_output);
	+ sglist_free(s->sg_ulptx);
	+ sglist_free(s->sg_dsgl);
	+ mtx_destroy(&s->lock);
	+}
	+
	static int
	ccr_newsession(device_t dev, crypto_session_t cses,
	const struct crypto_session_params *csp)
	@@ -2547,18 +2586,31 @@
	}
	#endif

	+ s = crypto_get_driver_session(cses);
	+ mtx_init(&s->lock, "ccr session", NULL, MTX_DEF);
	+ s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
	+ s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
	+ s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
	+ s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT);
	+ if (s->sg_input == NULL \|\| s->sg_output == NULL \|\|
	+ s->sg_ulptx == NULL \|\| s->sg_dsgl == NULL) {
	+ ccr_delete_session(s);
	+ return (ENOMEM);
	+ }
	+
	sc = device_get_softc(dev);

	mtx_lock(&sc->lock);
	if (sc->detaching) {
	mtx_unlock(&sc->lock);
	+ ccr_delete_session(s);
	return (ENXIO);
	}

	- s = crypto_get_driver_session(cses);
	s->port = ccr_choose_port(sc);
	if (s->port == NULL) {
	mtx_unlock(&sc->lock);
	+ ccr_delete_session(s);
	return (ENXIO);
	}

	@@ -2619,7 +2671,6 @@
	csp->csp_cipher_klen);
	}

	- s->active = true;
	s->port->active_sessions++;
	mtx_unlock(&sc->lock);
	return (0);
	@@ -2633,14 +2684,16 @@

	sc = device_get_softc(dev);
	s = crypto_get_driver_session(cses);
	- mtx_lock(&sc->lock);
	+#ifdef INVARIANTS
	if (s->pending != 0)
	device_printf(dev,
	"session %p freed with %d pending requests\n", s,
	s->pending);
	- s->active = false;
	+#endif
	+ mtx_lock(&sc->lock);
	s->port->active_sessions--;
	mtx_unlock(&sc->lock);
	+ ccr_delete_session(s);
	}

	static int
	@@ -2655,12 +2708,12 @@
	s = crypto_get_driver_session(crp->crp_session);
	sc = device_get_softc(dev);

	- mtx_lock(&sc->lock);
	- error = ccr_populate_sglist(sc->sg_input, &crp->crp_buf);
	+ mtx_lock(&s->lock);
	+ error = ccr_populate_sglist(s->sg_input, &crp->crp_buf);
	if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp))
	- error = ccr_populate_sglist(sc->sg_output, &crp->crp_obuf);
	+ error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf);
	if (error) {
	- sc->stats_sglist_error++;
	+ counter_u64_add(sc->stats_sglist_error, 1);
	goto out;
	}

	@@ -2668,7 +2721,7 @@
	case HASH:
	error = ccr_hash(sc, s, crp);
	if (error == 0)
	- sc->stats_hash++;
	+ counter_u64_add(sc->stats_hash, 1);
	break;
	case HMAC:
	if (crp->crp_auth_key != NULL)
	@@ -2677,7 +2730,7 @@
	csp->csp_auth_klen, s->hmac.pads);
	error = ccr_hash(sc, s, crp);
	if (error == 0)
	- sc->stats_hmac++;
	+ counter_u64_add(sc->stats_hmac, 1);
	break;
	case BLKCIPHER:
	if (crp->crp_cipher_key != NULL)
	@@ -2686,9 +2739,9 @@
	error = ccr_blkcipher(sc, s, crp);
	if (error == 0) {
	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
	- sc->stats_blkcipher_encrypt++;
	+ counter_u64_add(sc->stats_blkcipher_encrypt, 1);
	else
	- sc->stats_blkcipher_decrypt++;
	+ counter_u64_add(sc->stats_blkcipher_decrypt, 1);
	}
	break;
	case ETA:
	@@ -2702,9 +2755,9 @@
	error = ccr_eta(sc, s, crp);
	if (error == 0) {
	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
	- sc->stats_eta_encrypt++;
	+ counter_u64_add(sc->stats_eta_encrypt, 1);
	else
	- sc->stats_eta_decrypt++;
	+ counter_u64_add(sc->stats_eta_decrypt, 1);
	}
	break;
	case GCM:
	@@ -2715,22 +2768,22 @@
	csp->csp_cipher_klen);
	}
	if (crp->crp_payload_length == 0) {
	- mtx_unlock(&sc->lock);
	+ mtx_unlock(&s->lock);
	ccr_gcm_soft(s, crp);
	return (0);
	}
	error = ccr_gcm(sc, s, crp);
	if (error == EMSGSIZE) {
	- sc->stats_sw_fallback++;
	- mtx_unlock(&sc->lock);
	+ counter_u64_add(sc->stats_sw_fallback, 1);
	+ mtx_unlock(&s->lock);
	ccr_gcm_soft(s, crp);
	return (0);
	}
	if (error == 0) {
	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
	- sc->stats_gcm_encrypt++;
	+ counter_u64_add(sc->stats_gcm_encrypt, 1);
	else
	- sc->stats_gcm_decrypt++;
	+ counter_u64_add(sc->stats_gcm_decrypt, 1);
	}
	break;
	case CCM:
	@@ -2740,28 +2793,30 @@
	}
	error = ccr_ccm(sc, s, crp);
	if (error == EMSGSIZE) {
	- sc->stats_sw_fallback++;
	- mtx_unlock(&sc->lock);
	+ counter_u64_add(sc->stats_sw_fallback, 1);
	+ mtx_unlock(&s->lock);
	ccr_ccm_soft(s, crp);
	return (0);
	}
	if (error == 0) {
	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
	- sc->stats_ccm_encrypt++;
	+ counter_u64_add(sc->stats_ccm_encrypt, 1);
	else
	- sc->stats_ccm_decrypt++;
	+ counter_u64_add(sc->stats_ccm_decrypt, 1);
	}
	break;
	}

	if (error == 0) {
	+#ifdef INVARIANTS
	s->pending++;
	- sc->stats_inflight++;
	+#endif
	+ counter_u64_add(sc->stats_inflight, 1);
	} else
	- sc->stats_process_error++;
	+ counter_u64_add(sc->stats_process_error, 1);

	out:
	- mtx_unlock(&sc->lock);
	+ mtx_unlock(&s->lock);

	if (error) {
	crp->crp_etype = error;
	@@ -2795,9 +2850,12 @@
	else
	error = 0;

	- mtx_lock(&sc->lock);
	+#ifdef INVARIANTS
	+ mtx_lock(&s->lock);
	s->pending--;
	- sc->stats_inflight--;
	+ mtx_unlock(&s->lock);
	+#endif
	+ counter_u64_add(sc->stats_inflight, -1);

	switch (s->mode) {
	case HASH:
	@@ -2820,11 +2878,10 @@

	if (error == EBADMSG) {
	if (CHK_MAC_ERR_BIT(status))
	- sc->stats_mac_error++;
	+ counter_u64_add(sc->stats_mac_error, 1);
	if (CHK_PAD_ERR_BIT(status))
	- sc->stats_pad_error++;
	+ counter_u64_add(sc->stats_pad_error, 1);
	}
	- mtx_unlock(&sc->lock);
	crp->crp_etype = error;
	crypto_done(crp);
	m_freem(m);

File Metadata

Mime Type: text/plain
Expires: Thu, Jun 25, 5:57 AM (20 h, 24 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 34307283
Default Alt Text: D25457.id73676.diff (28 KB)

D25457.id73676.diffNo OneTemporaryActions

D25457.id73676.diffView Options

File Metadata

Event Timeline

D25457.id73676.diff
No OneTemporary
Actions

D25457.id73676.diff
View Options