Page MenuHomeFreeBSD
Differential D25457 Diff 73676 head/sys/dev/cxgbe/crypto/t4_crypto.c

Changeset View

Standalone View

View Options

head/sys/dev/cxgbe/crypto/t4_crypto.c

Show First 20 Lines • Show All 164 Lines • ▼ Show 20 Lines
struct ccr_port { struct ccr_port {
struct sge_wrq *txq; struct sge_wrq *txq;
struct sge_rxq *rxq; struct sge_rxq *rxq;
int tx_channel_id; int tx_channel_id;
u_int active_sessions; u_int active_sessions;
}; };
struct ccr_session { struct ccr_session {
bool active; #ifdef INVARIANTS
int pending; int pending;
#endif
enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
struct ccr_port *port; struct ccr_port *port;
union { union {
struct ccr_session_hmac hmac; struct ccr_session_hmac hmac;
struct ccr_session_gmac gmac; struct ccr_session_gmac gmac;
struct ccr_session_ccm_mac ccm_mac; struct ccr_session_ccm_mac ccm_mac;
}; };
struct ccr_session_blkcipher blkcipher; struct ccr_session_blkcipher blkcipher;
};
struct ccr_softc {
struct adapter *adapter;
device_t dev;
uint32_t cid;
struct mtx lock; struct mtx lock;
bool detaching;
struct ccr_port ports[MAX_NPORTS];
u_int port_mask;
/* /*
* Pre-allocate S/G lists used when preparing a work request. * Pre-allocate S/G lists used when preparing a work request.
* 'sg_input' contains an sglist describing the entire input * 'sg_input' contains an sglist describing the entire input
* buffer for a 'struct cryptop'. 'sg_output' contains an * buffer for a 'struct cryptop'. 'sg_output' contains an
* sglist describing the entire output buffer. 'sg_ulptx' is * sglist describing the entire output buffer. 'sg_ulptx' is
* used to describe the data the engine should DMA as input * used to describe the data the engine should DMA as input
* via ULPTX_SGL. 'sg_dsgl' is used to describe the * via ULPTX_SGL. 'sg_dsgl' is used to describe the
* destination that cipher text and a tag should be written * destination that cipher text and a tag should be written
* to. * to.
*/ */
struct sglist *sg_input; struct sglist *sg_input;
struct sglist *sg_output; struct sglist *sg_output;
struct sglist *sg_ulptx; struct sglist *sg_ulptx;
struct sglist *sg_dsgl; struct sglist *sg_dsgl;
};
struct ccr_softc {
struct adapter *adapter;
device_t dev;
uint32_t cid;
struct mtx lock;
bool detaching;
struct ccr_port ports[MAX_NPORTS];
u_int port_mask;
/* /*
* Pre-allocate a dummy output buffer for the IV and AAD for * Pre-allocate a dummy output buffer for the IV and AAD for
* AEAD requests. * AEAD requests.
*/ */
char *iv_aad_buf; char *iv_aad_buf;
struct sglist *sg_iv_aad; struct sglist *sg_iv_aad;
/* Statistics. */ /* Statistics. */
uint64_t stats_blkcipher_encrypt; counter_u64_t stats_blkcipher_encrypt;
uint64_t stats_blkcipher_decrypt; counter_u64_t stats_blkcipher_decrypt;
uint64_t stats_hash; counter_u64_t stats_hash;
uint64_t stats_hmac; counter_u64_t stats_hmac;
uint64_t stats_eta_encrypt; counter_u64_t stats_eta_encrypt;
uint64_t stats_eta_decrypt; counter_u64_t stats_eta_decrypt;
uint64_t stats_gcm_encrypt; counter_u64_t stats_gcm_encrypt;
uint64_t stats_gcm_decrypt; counter_u64_t stats_gcm_decrypt;
uint64_t stats_ccm_encrypt; counter_u64_t stats_ccm_encrypt;
uint64_t stats_ccm_decrypt; counter_u64_t stats_ccm_decrypt;
uint64_t stats_wr_nomem; counter_u64_t stats_wr_nomem;
uint64_t stats_inflight; counter_u64_t stats_inflight;
uint64_t stats_mac_error; counter_u64_t stats_mac_error;
uint64_t stats_pad_error; counter_u64_t stats_pad_error;
uint64_t stats_bad_session; counter_u64_t stats_sglist_error;
uint64_t stats_sglist_error; counter_u64_t stats_process_error;
uint64_t stats_process_error; counter_u64_t stats_sw_fallback;
uint64_t stats_sw_fallback;
}; };
/* /*
* Crypto requests involve two kind of scatter/gather lists. * Crypto requests involve two kind of scatter/gather lists.
* *
* Non-hash-only requests require a PHYS_DSGL that describes the * Non-hash-only requests require a PHYS_DSGL that describes the
* location to store the results of the encryption or decryption * location to store the results of the encryption or decryption
* operation. This SGL uses a different format (PHYS_DSGL) and should * operation. This SGL uses a different format (PHYS_DSGL) and should
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Linesccr_phys_dsgl_len(int nsegs)
if ((nsegs % 8) != 0) { if ((nsegs % 8) != 0) {
len += sizeof(uint16_t) * 8; len += sizeof(uint16_t) * 8;
len += roundup2(nsegs % 8, 2) * sizeof(uint64_t); len += roundup2(nsegs % 8, 2) * sizeof(uint64_t);
} }
return (len); return (len);
} }
static void static void
ccr_write_phys_dsgl(struct ccr_softc *sc, struct ccr_session *s, void *dst, ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs)
int nsegs)
{ {
struct sglist *sg; struct sglist *sg;
struct cpl_rx_phys_dsgl *cpl; struct cpl_rx_phys_dsgl *cpl;
struct phys_sge_pairs *sgl; struct phys_sge_pairs *sgl;
vm_paddr_t paddr; vm_paddr_t paddr;
size_t seglen; size_t seglen;
u_int i, j; u_int i, j;
sg = sc->sg_dsgl; sg = s->sg_dsgl;
cpl = dst; cpl = dst;
cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) |
V_CPL_RX_PHYS_DSGL_ISRDMA(0)); V_CPL_RX_PHYS_DSGL_ISRDMA(0));
cpl->pcirlxorder_to_noofsgentr = htobe32( cpl->pcirlxorder_to_noofsgentr = htobe32(
V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) | V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) |
V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) | V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) |
V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) | V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) |
V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs)); V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs));
Show All 32 Linesccr_ulptx_sgl_len(int nsegs)
u_int n; u_int n;
nsegs--; /* first segment is part of ulptx_sgl */ nsegs--; /* first segment is part of ulptx_sgl */
n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
return (roundup2(n, 16)); return (roundup2(n, 16));
} }
static void static void
ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs) ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs)
{ {
struct ulptx_sgl *usgl; struct ulptx_sgl *usgl;
struct sglist *sg; struct sglist *sg;
struct sglist_seg *ss; struct sglist_seg *ss;
int i; int i;
sg = sc->sg_ulptx; sg = s->sg_ulptx;
MPASS(nsegs == sg->sg_nseg); MPASS(nsegs == sg->sg_nseg);
ss = &sg->sg_segs[0]; ss = &sg->sg_segs[0];
usgl = dst; usgl = dst;
usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
V_ULPTX_NSGE(nsegs)); V_ULPTX_NSGE(nsegs));
usgl->len0 = htobe32(ss->ss_len); usgl->len0 = htobe32(ss->ss_len);
usgl->addr0 = htobe64(ss->ss_paddr); usgl->addr0 = htobe64(ss->ss_paddr);
ss++; ss++;
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines if (crp->crp_payload_length == 0) {
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) { } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) {
imm_len = crp->crp_payload_length; imm_len = crp->crp_payload_length;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len; wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
if (wr_len > SGE_MAX_WR_LEN) if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
memset(crwr, 0, wr_len); memset(crwr, 0, wr_len);
ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len,
hash_size_in_response, crp); hash_size_in_response, crp);
Show All 37 Lines if (crp->crp_payload_length == 0) {
dst[0] = 0x80; dst[0] = 0x80;
if (s->mode == HMAC) if (s->mode == HMAC)
*(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) =
htobe64(axf->blocksize << 3); htobe64(axf->blocksize << 3);
} else if (imm_len != 0) } else if (imm_len != 0)
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
else else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
return (0); return (0);
} }
static int static int
Show All 39 Linesccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
if (crp->crp_payload_length > MAX_REQUEST_SIZE) if (crp->crp_payload_length > MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP; op_type = CHCR_ENCRYPT_OP;
else else
op_type = CHCR_DECRYPT_OP; op_type = CHCR_DECRYPT_OP;
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
/* The 'key' must be 128-bit aligned. */ /* The 'key' must be 128-bit aligned. */
kctx_len = roundup2(s->blkcipher.key_len, 16); kctx_len = roundup2(s->blkcipher.key_len, 16);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
/* For AES-XTS we send a 16-byte IV in the work request. */ /* For AES-XTS we send a 16-byte IV in the work request. */
if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS) if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
iv_len = AES_BLOCK_LEN; iv_len = AES_BLOCK_LEN;
else else
iv_len = s->blkcipher.iv_len; iv_len = s->blkcipher.iv_len;
if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) { if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) {
imm_len = crp->crp_payload_length; imm_len = crp->crp_payload_length;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
wr_len = roundup2(transhdr_len, 16) + iv_len + wr_len = roundup2(transhdr_len, 16) + iv_len +
roundup2(imm_len, 16) + sgl_len; roundup2(imm_len, 16) + sgl_len;
if (wr_len > SGE_MAX_WR_LEN) if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
memset(crwr, 0, wr_len); memset(crwr, 0, wr_len);
crypto_read_iv(crp, iv); crypto_read_iv(crp, iv);
/* Zero the remainder of the IV for AES-XTS. */ /* Zero the remainder of the IV for AES-XTS. */
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
s->blkcipher.enckey, key_half); s->blkcipher.enckey, key_half);
else else
memcpy(crwr->key_ctx.key + key_half, memcpy(crwr->key_ctx.key + key_half,
s->blkcipher.deckey, key_half); s->blkcipher.deckey, key_half);
break; break;
} }
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
if (imm_len != 0) if (imm_len != 0)
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
else else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv, sizeof(iv));
return (0); return (0);
} }
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines if (op_type == CHCR_ENCRYPT_OP) {
if (iv_len + crp->crp_aad_length + crp->crp_payload_length + if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE) hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} else { } else {
if (iv_len + crp->crp_aad_length + crp->crp_payload_length > if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0,
iv_len + crp->crp_aad_length); iv_len + crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
/* PADs must be 128-bit aligned. */ /* PADs must be 128-bit aligned. */
iopad_size = roundup2(s->hmac.partial_digest_len, 16); iopad_size = roundup2(s->hmac.partial_digest_len, 16);
/* /*
Show All 30 Lines if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response; input_len += hash_size_in_response;
if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len; imm_len = input_len;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
/* Any AAD comes after the IV. */ /* Any AAD comes after the IV. */
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
aad_start = iv_len + 1; aad_start = iv_len + 1;
aad_stop = aad_start + crp->crp_aad_length - 1; aad_stop = aad_start + crp->crp_aad_length - 1;
} else { } else {
Show All 11 Lines else
auth_insert = 0; auth_insert = 0;
wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
sgl_len; sgl_len;
if (wr_len > SGE_MAX_WR_LEN) if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
memset(crwr, 0, wr_len); memset(crwr, 0, wr_len);
crypto_read_iv(crp, iv); crypto_read_iv(crp, iv);
/* Zero the remainder of the IV for AES-XTS. */ /* Zero the remainder of the IV for AES-XTS. */
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines else
s->blkcipher.deckey, key_half); s->blkcipher.deckey, key_half);
break; break;
} }
dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
memcpy(dst, s->hmac.pads, iopad_size * 2); memcpy(dst, s->hmac.pads, iopad_size * 2);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
if (imm_len != 0) { if (imm_len != 0) {
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
memcpy(dst, crp->crp_aad, crp->crp_aad_length); memcpy(dst, crp->crp_aad, crp->crp_aad_length);
else else
crypto_copydata(crp, crp->crp_aad_start, crypto_copydata(crp, crp->crp_aad_start,
crp->crp_aad_length, dst); crp->crp_aad_length, dst);
dst += crp->crp_aad_length; dst += crp->crp_aad_length;
} }
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
dst += crp->crp_payload_length; dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
} else } else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv, sizeof(iv));
return (0); return (0);
} }
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines if (op_type == CHCR_ENCRYPT_OP) {
if (iv_len + crp->crp_aad_length + crp->crp_payload_length + if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE) hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} else { } else {
if (iv_len + crp->crp_aad_length + crp->crp_payload_length > if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
/* /*
* The 'key' part of the key context consists of the key followed * The 'key' part of the key context consists of the key followed
* by the Galois hash key. * by the Galois hash key.
*/ */
Show All 17 Linesccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
if (input_len > MAX_REQUEST_SIZE) if (input_len > MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len; imm_len = input_len;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
aad_start = iv_len + 1; aad_start = iv_len + 1;
aad_stop = aad_start + crp->crp_aad_length - 1; aad_stop = aad_start + crp->crp_aad_length - 1;
} else { } else {
aad_start = 0; aad_start = 0;
Show All 10 Lines else
auth_insert = 0; auth_insert = 0;
wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
sgl_len; sgl_len;
if (wr_len > SGE_MAX_WR_LEN) if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
memset(crwr, 0, wr_len); memset(crwr, 0, wr_len);
memcpy(iv, crp->crp_iv, s->blkcipher.iv_len); memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
if (s->blkcipher.iv_len == 12) if (s->blkcipher.iv_len == 12)
*(uint32_t *)&iv[12] = htobe32(1); *(uint32_t *)&iv[12] = htobe32(1);
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len);
dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
if (imm_len != 0) { if (imm_len != 0) {
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
memcpy(dst, crp->crp_aad, crp->crp_aad_length); memcpy(dst, crp->crp_aad, crp->crp_aad_length);
else else
crypto_copydata(crp, crp->crp_aad_start, crypto_copydata(crp, crp->crp_aad_start,
crp->crp_aad_length, dst); crp->crp_aad_length, dst);
dst += crp->crp_aad_length; dst += crp->crp_aad_length;
} }
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
dst += crp->crp_payload_length; dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
} else } else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv, sizeof(iv));
return (0); return (0);
} }
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines if (op_type == CHCR_ENCRYPT_OP) {
if (iv_len + aad_len + crp->crp_payload_length + if (iv_len + aad_len + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE) hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} else { } else {
if (iv_len + aad_len + crp->crp_payload_length > if (iv_len + aad_len + crp->crp_payload_length >
MAX_REQUEST_SIZE) MAX_REQUEST_SIZE)
return (EFBIG); return (EFBIG);
} }
sglist_reset(sc->sg_dsgl); sglist_reset(s->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
aad_len); aad_len);
if (error) if (error)
return (error); return (error);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_payload_output_start, crp->crp_payload_length); crp->crp_payload_output_start, crp->crp_payload_length);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_ENCRYPT_OP) { if (op_type == CHCR_ENCRYPT_OP) {
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
else else
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
return (EFBIG); return (EFBIG);
dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
/* /*
* The 'key' part of the key context consists of two copies of * The 'key' part of the key context consists of two copies of
* the AES key. * the AES key.
*/ */
Show All 19 Linesccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len; imm_len = input_len;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
/* Block 0 is passed as immediate data. */ /* Block 0 is passed as immediate data. */
imm_len = b0_len; imm_len = b0_len;
sglist_reset(sc->sg_ulptx); sglist_reset(s->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
error = sglist_append(sc->sg_ulptx, error = sglist_append(s->sg_ulptx,
crp->crp_aad, crp->crp_aad_length); crp->crp_aad, crp->crp_aad_length);
else else
error = sglist_append_sglist(sc->sg_ulptx, error = sglist_append_sglist(s->sg_ulptx,
sc->sg_input, crp->crp_aad_start, s->sg_input, crp->crp_aad_start,
crp->crp_aad_length); crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = s->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
aad_start = iv_len + 1; aad_start = iv_len + 1;
aad_stop = aad_start + aad_len - 1; aad_stop = aad_start + aad_len - 1;
cipher_start = aad_stop + 1; cipher_start = aad_stop + 1;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
cipher_stop = hash_size_in_response; cipher_stop = hash_size_in_response;
else else
cipher_stop = 0; cipher_stop = 0;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
auth_insert = hash_size_in_response; auth_insert = hash_size_in_response;
else else
auth_insert = 0; auth_insert = 0;
wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
sgl_len; sgl_len;
if (wr_len > SGE_MAX_WR_LEN) if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG); return (EFBIG);
wr = alloc_wrqe(wr_len, s->port->txq); wr = alloc_wrqe(wr_len, s->port->txq);
if (wr == NULL) { if (wr == NULL) {
sc->stats_wr_nomem++; counter_u64_add(sc->stats_wr_nomem, 1);
return (ENOMEM); return (ENOMEM);
} }
crwr = wrtod(wr); crwr = wrtod(wr);
memset(crwr, 0, wr_len); memset(crwr, 0, wr_len);
/* /*
* Read the nonce from the request. Use the nonce to generate * Read the nonce from the request. Use the nonce to generate
* the full IV with the counter set to 0. * the full IV with the counter set to 0.
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len);
memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16), memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16),
s->blkcipher.enckey, s->blkcipher.key_len); s->blkcipher.enckey, s->blkcipher.key_len);
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
generate_ccm_b0(crp, hash_size_in_response, iv, dst); generate_ccm_b0(crp, hash_size_in_response, iv, dst);
if (sgl_nsegs == 0) { if (sgl_nsegs == 0) {
dst += b0_len; dst += b0_len;
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL) if (crp->crp_aad != NULL)
Show All 21 Lines if (b0_len > CCM_B0_SIZE) {
("b0_len mismatch")); ("b0_len mismatch"));
memset(dst + CCM_AAD_FIELD_SIZE, 0, memset(dst + CCM_AAD_FIELD_SIZE, 0,
8 - CCM_AAD_FIELD_SIZE); 8 - CCM_AAD_FIELD_SIZE);
idata = (void *)(dst + 8); idata = (void *)(dst + 8);
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
idata->len = htobe32(0); idata->len = htobe32(0);
dst = (void *)(idata + 1); dst = (void *)(idata + 1);
} }
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
} }
/* XXX: TODO backpressure */ /* XXX: TODO backpressure */
t4_wrq_tx(sc->adapter, wr); t4_wrq_tx(sc->adapter, wr);
explicit_bzero(iv, sizeof(iv)); explicit_bzero(iv, sizeof(iv));
return (0); return (0);
} }
▲ Show 20 Lines • Show All 190 Lines • ▼ Show 20 Linesccr_sysctls(struct ccr_softc *sc)
/* /*
* dev.ccr.X.stats. * dev.ccr.X.stats.
*/ */
oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
children = SYSCTL_CHILDREN(oid); children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
&sc->stats_hash, 0, "Hash requests submitted"); &sc->stats_hash, "Hash requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
&sc->stats_hmac, 0, "HMAC requests submitted"); &sc->stats_hmac, "HMAC requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt",
&sc->stats_blkcipher_encrypt, 0, CTLFLAG_RD, &sc->stats_blkcipher_encrypt,
"Cipher encryption requests submitted"); "Cipher encryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt",
&sc->stats_blkcipher_decrypt, 0, CTLFLAG_RD, &sc->stats_blkcipher_decrypt,
"Cipher decryption requests submitted"); "Cipher decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt",
&sc->stats_eta_encrypt, 0, CTLFLAG_RD, &sc->stats_eta_encrypt,
"Combined AES+HMAC encryption requests submitted"); "Combined AES+HMAC encryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt",
&sc->stats_eta_decrypt, 0, CTLFLAG_RD, &sc->stats_eta_decrypt,
"Combined AES+HMAC decryption requests submitted"); "Combined AES+HMAC decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt",
&sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted"); CTLFLAG_RD, &sc->stats_gcm_encrypt,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, "AES-GCM encryption requests submitted");
&sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt",
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD, CTLFLAG_RD, &sc->stats_gcm_decrypt,
&sc->stats_ccm_encrypt, 0, "AES-CCM encryption requests submitted"); "AES-GCM decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt",
&sc->stats_ccm_decrypt, 0, "AES-CCM decryption requests submitted"); CTLFLAG_RD, &sc->stats_ccm_encrypt,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, "AES-CCM encryption requests submitted");
&sc->stats_wr_nomem, 0, "Work request memory allocation failures"); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt",
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, CTLFLAG_RD, &sc->stats_ccm_decrypt,
&sc->stats_inflight, 0, "Requests currently pending"); "AES-CCM decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
&sc->stats_mac_error, 0, "MAC errors"); &sc->stats_wr_nomem, "Work request memory allocation failures");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
&sc->stats_pad_error, 0, "Padding errors"); &sc->stats_inflight, "Requests currently pending");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
&sc->stats_bad_session, 0, "Requests with invalid session ID"); &sc->stats_mac_error, "MAC errors");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
&sc->stats_sglist_error, 0, &sc->stats_pad_error, "Padding errors");
SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error",
CTLFLAG_RD, &sc->stats_sglist_error,
"Requests for which DMA mapping failed"); "Requests for which DMA mapping failed");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error",
&sc->stats_process_error, 0, "Requests failed during queueing"); CTLFLAG_RD, &sc->stats_process_error,
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, "Requests failed during queueing");
&sc->stats_sw_fallback, 0, SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback",
CTLFLAG_RD, &sc->stats_sw_fallback,
"Requests processed by falling back to software"); "Requests processed by falling back to software");
/* /*
* dev.ccr.X.stats.port * dev.ccr.X.stats.port
*/ */
port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port", port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port",
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics"); CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics");
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Linesccr_attach(device_t dev)
if (cid < 0) { if (cid < 0) {
device_printf(dev, "could not get crypto driver id\n"); device_printf(dev, "could not get crypto driver id\n");
return (ENXIO); return (ENXIO);
} }
sc->cid = cid; sc->cid = cid;
sc->adapter->ccr_softc = sc; sc->adapter->ccr_softc = sc;
mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
sc->sg_input = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_output = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK);
sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK);
sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_hash = counter_u64_alloc(M_WAITOK);
sc->stats_hmac = counter_u64_alloc(M_WAITOK);
sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK);
sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK);
sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK);
sc->stats_inflight = counter_u64_alloc(M_WAITOK);
sc->stats_mac_error = counter_u64_alloc(M_WAITOK);
sc->stats_pad_error = counter_u64_alloc(M_WAITOK);
sc->stats_sglist_error = counter_u64_alloc(M_WAITOK);
sc->stats_process_error = counter_u64_alloc(M_WAITOK);
sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK);
ccr_sysctls(sc); ccr_sysctls(sc);
return (0); return (0);
} }
static int static int
ccr_detach(device_t dev) ccr_detach(device_t dev)
{ {
struct ccr_softc *sc; struct ccr_softc *sc;
sc = device_get_softc(dev); sc = device_get_softc(dev);
mtx_lock(&sc->lock); mtx_lock(&sc->lock);
sc->detaching = true; sc->detaching = true;
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
crypto_unregister_all(sc->cid); crypto_unregister_all(sc->cid);
mtx_destroy(&sc->lock); mtx_destroy(&sc->lock);
counter_u64_free(sc->stats_blkcipher_encrypt);
counter_u64_free(sc->stats_blkcipher_decrypt);
counter_u64_free(sc->stats_hash);
counter_u64_free(sc->stats_hmac);
counter_u64_free(sc->stats_eta_encrypt);
counter_u64_free(sc->stats_eta_decrypt);
counter_u64_free(sc->stats_gcm_encrypt);
counter_u64_free(sc->stats_gcm_decrypt);
counter_u64_free(sc->stats_ccm_encrypt);
counter_u64_free(sc->stats_ccm_decrypt);
counter_u64_free(sc->stats_wr_nomem);
counter_u64_free(sc->stats_inflight);
counter_u64_free(sc->stats_mac_error);
counter_u64_free(sc->stats_pad_error);
counter_u64_free(sc->stats_sglist_error);
counter_u64_free(sc->stats_process_error);
counter_u64_free(sc->stats_sw_fallback);
sglist_free(sc->sg_iv_aad); sglist_free(sc->sg_iv_aad);
free(sc->iv_aad_buf, M_CCR); free(sc->iv_aad_buf, M_CCR);
sglist_free(sc->sg_dsgl);
sglist_free(sc->sg_ulptx);
sglist_free(sc->sg_output);
sglist_free(sc->sg_input);
sc->adapter->ccr_softc = NULL; sc->adapter->ccr_softc = NULL;
return (0); return (0);
} }
static void static void
ccr_init_hash_digest(struct ccr_session *s) ccr_init_hash_digest(struct ccr_session *s)
{ {
union authctx auth_ctx; union authctx auth_ctx;
▲ Show 20 Lines • Show All 255 Lines • ▼ Show 20 Lines for (i = 0; i < nitems(sc->ports); i++) {
if (best == NULL || if (best == NULL ||
p->active_sessions < best->active_sessions) p->active_sessions < best->active_sessions)
best = p; best = p;
} }
return (best); return (best);
} }
static void
ccr_delete_session(struct ccr_session *s)
{
sglist_free(s->sg_input);
sglist_free(s->sg_output);
sglist_free(s->sg_ulptx);
sglist_free(s->sg_dsgl);
mtx_destroy(&s->lock);
}
static int static int
ccr_newsession(device_t dev, crypto_session_t cses, ccr_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp) const struct crypto_session_params *csp)
{ {
struct ccr_softc *sc; struct ccr_softc *sc;
struct ccr_session *s; struct ccr_session *s;
struct auth_hash *auth_hash; struct auth_hash *auth_hash;
unsigned int auth_mode, cipher_mode, mk_size; unsigned int auth_mode, cipher_mode, mk_size;
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines case CSP_MODE_ETA:
if (auth_mode == SCMD_AUTH_MODE_NOP) if (auth_mode == SCMD_AUTH_MODE_NOP)
panic("invalid auth algo"); panic("invalid auth algo");
break; break;
default: default:
panic("invalid csp mode"); panic("invalid csp mode");
} }
#endif #endif
s = crypto_get_driver_session(cses);
mtx_init(&s->lock, "ccr session", NULL, MTX_DEF);
s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT);
if (s->sg_input == NULL || s->sg_output == NULL ||
s->sg_ulptx == NULL || s->sg_dsgl == NULL) {
ccr_delete_session(s);
return (ENOMEM);
}
sc = device_get_softc(dev); sc = device_get_softc(dev);
mtx_lock(&sc->lock); mtx_lock(&sc->lock);
if (sc->detaching) { if (sc->detaching) {
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
return (ENXIO); return (ENXIO);
} }
s = crypto_get_driver_session(cses);
s->port = ccr_choose_port(sc); s->port = ccr_choose_port(sc);
if (s->port == NULL) { if (s->port == NULL) {
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
return (ENXIO); return (ENXIO);
} }
switch (csp->csp_mode) { switch (csp->csp_mode) {
case CSP_MODE_AEAD: case CSP_MODE_AEAD:
if (cipher_mode == SCMD_CIPH_MODE_AES_CCM) if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
s->mode = CCM; s->mode = CCM;
else else
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines#endif
if (cipher_mode != SCMD_CIPH_MODE_NOP) { if (cipher_mode != SCMD_CIPH_MODE_NOP) {
s->blkcipher.cipher_mode = cipher_mode; s->blkcipher.cipher_mode = cipher_mode;
s->blkcipher.iv_len = csp->csp_ivlen; s->blkcipher.iv_len = csp->csp_ivlen;
if (csp->csp_cipher_key != NULL) if (csp->csp_cipher_key != NULL)
ccr_aes_setkey(s, csp->csp_cipher_key, ccr_aes_setkey(s, csp->csp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
s->active = true;
s->port->active_sessions++; s->port->active_sessions++;
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
return (0); return (0);
} }
static void static void
ccr_freesession(device_t dev, crypto_session_t cses) ccr_freesession(device_t dev, crypto_session_t cses)
{ {
struct ccr_softc *sc; struct ccr_softc *sc;
struct ccr_session *s; struct ccr_session *s;
sc = device_get_softc(dev); sc = device_get_softc(dev);
s = crypto_get_driver_session(cses); s = crypto_get_driver_session(cses);
mtx_lock(&sc->lock); #ifdef INVARIANTS
if (s->pending != 0) if (s->pending != 0)
device_printf(dev, device_printf(dev,
"session %p freed with %d pending requests\n", s, "session %p freed with %d pending requests\n", s,
s->pending); s->pending);
s->active = false; #endif
mtx_lock(&sc->lock);
s->port->active_sessions--; s->port->active_sessions--;
mtx_unlock(&sc->lock); mtx_unlock(&sc->lock);
ccr_delete_session(s);
} }
static int static int
ccr_process(device_t dev, struct cryptop *crp, int hint) ccr_process(device_t dev, struct cryptop *crp, int hint)
{ {
const struct crypto_session_params *csp; const struct crypto_session_params *csp;
struct ccr_softc *sc; struct ccr_softc *sc;
struct ccr_session *s; struct ccr_session *s;
int error; int error;
csp = crypto_get_params(crp->crp_session); csp = crypto_get_params(crp->crp_session);
s = crypto_get_driver_session(crp->crp_session); s = crypto_get_driver_session(crp->crp_session);
sc = device_get_softc(dev); sc = device_get_softc(dev);
mtx_lock(&sc->lock); mtx_lock(&s->lock);
error = ccr_populate_sglist(sc->sg_input, &crp->crp_buf); error = ccr_populate_sglist(s->sg_input, &crp->crp_buf);
if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp))
error = ccr_populate_sglist(sc->sg_output, &crp->crp_obuf); error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf);
if (error) { if (error) {
sc->stats_sglist_error++; counter_u64_add(sc->stats_sglist_error, 1);
goto out; goto out;
} }
switch (s->mode) { switch (s->mode) {
case HASH: case HASH:
error = ccr_hash(sc, s, crp); error = ccr_hash(sc, s, crp);
if (error == 0) if (error == 0)
sc->stats_hash++; counter_u64_add(sc->stats_hash, 1);
break; break;
case HMAC: case HMAC:
if (crp->crp_auth_key != NULL) if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash, t4_init_hmac_digest(s->hmac.auth_hash,
s->hmac.partial_digest_len, crp->crp_auth_key, s->hmac.partial_digest_len, crp->crp_auth_key,
csp->csp_auth_klen, s->hmac.pads); csp->csp_auth_klen, s->hmac.pads);
error = ccr_hash(sc, s, crp); error = ccr_hash(sc, s, crp);
if (error == 0) if (error == 0)
sc->stats_hmac++; counter_u64_add(sc->stats_hmac, 1);
break; break;
case BLKCIPHER: case BLKCIPHER:
if (crp->crp_cipher_key != NULL) if (crp->crp_cipher_key != NULL)
ccr_aes_setkey(s, crp->crp_cipher_key, ccr_aes_setkey(s, crp->crp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
error = ccr_blkcipher(sc, s, crp); error = ccr_blkcipher(sc, s, crp);
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_blkcipher_encrypt++; counter_u64_add(sc->stats_blkcipher_encrypt, 1);
else else
sc->stats_blkcipher_decrypt++; counter_u64_add(sc->stats_blkcipher_decrypt, 1);
} }
break; break;
case ETA: case ETA:
if (crp->crp_auth_key != NULL) if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash, t4_init_hmac_digest(s->hmac.auth_hash,
s->hmac.partial_digest_len, crp->crp_auth_key, s->hmac.partial_digest_len, crp->crp_auth_key,
csp->csp_auth_klen, s->hmac.pads); csp->csp_auth_klen, s->hmac.pads);
if (crp->crp_cipher_key != NULL) if (crp->crp_cipher_key != NULL)
ccr_aes_setkey(s, crp->crp_cipher_key, ccr_aes_setkey(s, crp->crp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
error = ccr_eta(sc, s, crp); error = ccr_eta(sc, s, crp);
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_eta_encrypt++; counter_u64_add(sc->stats_eta_encrypt, 1);
else else
sc->stats_eta_decrypt++; counter_u64_add(sc->stats_eta_decrypt, 1);
} }
break; break;
case GCM: case GCM:
if (crp->crp_cipher_key != NULL) { if (crp->crp_cipher_key != NULL) {
t4_init_gmac_hash(crp->crp_cipher_key, t4_init_gmac_hash(crp->crp_cipher_key,
csp->csp_cipher_klen, s->gmac.ghash_h); csp->csp_cipher_klen, s->gmac.ghash_h);
ccr_aes_setkey(s, crp->crp_cipher_key, ccr_aes_setkey(s, crp->crp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
if (crp->crp_payload_length == 0) { if (crp->crp_payload_length == 0) {
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_gcm_soft(s, crp); ccr_gcm_soft(s, crp);
return (0); return (0);
} }
error = ccr_gcm(sc, s, crp); error = ccr_gcm(sc, s, crp);
if (error == EMSGSIZE) { if (error == EMSGSIZE) {
sc->stats_sw_fallback++; counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_gcm_soft(s, crp); ccr_gcm_soft(s, crp);
return (0); return (0);
} }
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_gcm_encrypt++; counter_u64_add(sc->stats_gcm_encrypt, 1);
else else
sc->stats_gcm_decrypt++; counter_u64_add(sc->stats_gcm_decrypt, 1);
} }
break; break;
case CCM: case CCM:
if (crp->crp_cipher_key != NULL) { if (crp->crp_cipher_key != NULL) {
ccr_aes_setkey(s, crp->crp_cipher_key, ccr_aes_setkey(s, crp->crp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
error = ccr_ccm(sc, s, crp); error = ccr_ccm(sc, s, crp);
if (error == EMSGSIZE) { if (error == EMSGSIZE) {
sc->stats_sw_fallback++; counter_u64_add(sc->stats_sw_fallback, 1);
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
ccr_ccm_soft(s, crp); ccr_ccm_soft(s, crp);
return (0); return (0);
} }
if (error == 0) { if (error == 0) {
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_ccm_encrypt++; counter_u64_add(sc->stats_ccm_encrypt, 1);
else else
sc->stats_ccm_decrypt++; counter_u64_add(sc->stats_ccm_decrypt, 1);
} }
break; break;
} }
if (error == 0) { if (error == 0) {
#ifdef INVARIANTS
s->pending++; s->pending++;
sc->stats_inflight++; #endif
counter_u64_add(sc->stats_inflight, 1);
} else } else
sc->stats_process_error++; counter_u64_add(sc->stats_process_error, 1);
out: out:
mtx_unlock(&sc->lock); mtx_unlock(&s->lock);
if (error) { if (error) {
crp->crp_etype = error; crp->crp_etype = error;
crypto_done(crp); crypto_done(crp);
} }
return (0); return (0);
} }
Show All 17 Linesdo_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]); crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]);
s = crypto_get_driver_session(crp->crp_session); s = crypto_get_driver_session(crp->crp_session);
status = be64toh(cpl->data[0]); status = be64toh(cpl->data[0]);
if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status)) if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status))
error = EBADMSG; error = EBADMSG;
else else
error = 0; error = 0;
mtx_lock(&sc->lock); #ifdef INVARIANTS
mtx_lock(&s->lock);
s->pending--; s->pending--;
sc->stats_inflight--; mtx_unlock(&s->lock);
#endif
counter_u64_add(sc->stats_inflight, -1);
switch (s->mode) { switch (s->mode) {
case HASH: case HASH:
case HMAC: case HMAC:
error = ccr_hash_done(sc, s, crp, cpl, error); error = ccr_hash_done(sc, s, crp, cpl, error);
break; break;
case BLKCIPHER: case BLKCIPHER:
error = ccr_blkcipher_done(sc, s, crp, cpl, error); error = ccr_blkcipher_done(sc, s, crp, cpl, error);
break; break;
case ETA: case ETA:
error = ccr_eta_done(sc, s, crp, cpl, error); error = ccr_eta_done(sc, s, crp, cpl, error);
break; break;
case GCM: case GCM:
error = ccr_gcm_done(sc, s, crp, cpl, error); error = ccr_gcm_done(sc, s, crp, cpl, error);
break; break;
case CCM: case CCM:
error = ccr_ccm_done(sc, s, crp, cpl, error); error = ccr_ccm_done(sc, s, crp, cpl, error);
break; break;
} }
if (error == EBADMSG) { if (error == EBADMSG) {
if (CHK_MAC_ERR_BIT(status)) if (CHK_MAC_ERR_BIT(status))
sc->stats_mac_error++; counter_u64_add(sc->stats_mac_error, 1);
if (CHK_PAD_ERR_BIT(status)) if (CHK_PAD_ERR_BIT(status))
sc->stats_pad_error++; counter_u64_add(sc->stats_pad_error, 1);
} }
mtx_unlock(&sc->lock);
crp->crp_etype = error; crp->crp_etype = error;
crypto_done(crp); crypto_done(crp);
m_freem(m); m_freem(m);
return (0); return (0);
} }
static int static int
ccr_modevent(module_t mod, int cmd, void *arg) ccr_modevent(module_t mod, int cmd, void *arg)
Show All 40 Lines