Index: sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- sys/dev/cxgbe/crypto/t4_crypto.c +++ sys/dev/cxgbe/crypto/t4_crypto.c @@ -170,8 +170,9 @@ }; struct ccr_session { - bool active; +#ifdef INVARIANTS int pending; +#endif enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode; struct ccr_port *port; union { @@ -180,16 +181,7 @@ struct ccr_session_ccm_mac ccm_mac; }; struct ccr_session_blkcipher blkcipher; -}; - -struct ccr_softc { - struct adapter *adapter; - device_t dev; - uint32_t cid; struct mtx lock; - bool detaching; - struct ccr_port ports[MAX_NPORTS]; - u_int port_mask; /* * Pre-allocate S/G lists used when preparing a work request. @@ -205,6 +197,16 @@ struct sglist *sg_output; struct sglist *sg_ulptx; struct sglist *sg_dsgl; +}; + +struct ccr_softc { + struct adapter *adapter; + device_t dev; + uint32_t cid; + struct mtx lock; + bool detaching; + struct ccr_port ports[MAX_NPORTS]; + u_int port_mask; /* * Pre-allocate a dummy output buffer for the IV and AAD for @@ -214,24 +216,23 @@ struct sglist *sg_iv_aad; /* Statistics. */ - uint64_t stats_blkcipher_encrypt; - uint64_t stats_blkcipher_decrypt; - uint64_t stats_hash; - uint64_t stats_hmac; - uint64_t stats_eta_encrypt; - uint64_t stats_eta_decrypt; - uint64_t stats_gcm_encrypt; - uint64_t stats_gcm_decrypt; - uint64_t stats_ccm_encrypt; - uint64_t stats_ccm_decrypt; - uint64_t stats_wr_nomem; - uint64_t stats_inflight; - uint64_t stats_mac_error; - uint64_t stats_pad_error; - uint64_t stats_bad_session; - uint64_t stats_sglist_error; - uint64_t stats_process_error; - uint64_t stats_sw_fallback; + counter_u64_t stats_blkcipher_encrypt; + counter_u64_t stats_blkcipher_decrypt; + counter_u64_t stats_hash; + counter_u64_t stats_hmac; + counter_u64_t stats_eta_encrypt; + counter_u64_t stats_eta_decrypt; + counter_u64_t stats_gcm_encrypt; + counter_u64_t stats_gcm_decrypt; + counter_u64_t stats_ccm_encrypt; + counter_u64_t stats_ccm_decrypt; + counter_u64_t stats_wr_nomem; + counter_u64_t stats_inflight; + counter_u64_t stats_mac_error; + counter_u64_t stats_pad_error; + counter_u64_t stats_sglist_error; + counter_u64_t stats_process_error; + counter_u64_t stats_sw_fallback; }; /* @@ -307,8 +308,7 @@ } static void -ccr_write_phys_dsgl(struct ccr_softc *sc, struct ccr_session *s, void *dst, - int nsegs) +ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs) { struct sglist *sg; struct cpl_rx_phys_dsgl *cpl; @@ -317,7 +317,7 @@ size_t seglen; u_int i, j; - sg = sc->sg_dsgl; + sg = s->sg_dsgl; cpl = dst; cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | V_CPL_RX_PHYS_DSGL_ISRDMA(0)); @@ -366,14 +366,14 @@ } static void -ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs) +ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs) { struct ulptx_sgl *usgl; struct sglist *sg; struct sglist_seg *ss; int i; - sg = sc->sg_ulptx; + sg = s->sg_ulptx; MPASS(nsegs == sg->sg_nseg); ss = &sg->sg_segs[0]; usgl = dst; @@ -496,12 +496,12 @@ sgl_len = 0; } else { imm_len = 0; - sglist_reset(sc->sg_ulptx); - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + sglist_reset(s->sg_ulptx); + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); - sgl_nsegs = sc->sg_ulptx->sg_nseg; + sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } @@ -510,7 +510,7 @@ return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { - sc->stats_wr_nomem++; + counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); @@ -564,7 +564,7 @@ crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else - ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); + ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); @@ -620,16 +620,16 @@ else op_type = CHCR_DECRYPT_OP; - sglist_reset(sc->sg_dsgl); + sglist_reset(s->sg_dsgl); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); - dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); + dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); @@ -650,12 +650,12 @@ sgl_len = 0; } else { imm_len = 0; - sglist_reset(sc->sg_ulptx); - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + sglist_reset(s->sg_ulptx); + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); - sgl_nsegs = sc->sg_ulptx->sg_nseg; + sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } @@ -665,7 +665,7 @@ return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { - sc->stats_wr_nomem++; + counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); @@ -737,7 +737,7 @@ } dst = (char *)(crwr + 1) + kctx_len; - ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); + ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; @@ -745,7 +745,7 @@ crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, dst); else - ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); + ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); @@ -846,30 +846,30 @@ MAX_REQUEST_SIZE) return (EFBIG); } - sglist_reset(sc->sg_dsgl); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, + sglist_reset(s->sg_dsgl); + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); + dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); @@ -916,29 +916,29 @@ sgl_len = 0; } else { imm_len = 0; - sglist_reset(sc->sg_ulptx); + sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) - error = sglist_append(sc->sg_ulptx, + error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else - error = sglist_append_sglist(sc->sg_ulptx, - sc->sg_input, crp->crp_aad_start, + error = sglist_append_sglist(s->sg_ulptx, + s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - sgl_nsegs = sc->sg_ulptx->sg_nseg; + sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } @@ -966,7 +966,7 @@ return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { - sc->stats_wr_nomem++; + counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); @@ -1048,7 +1048,7 @@ memcpy(dst, s->hmac.pads, iopad_size * 2); dst = (char *)(crwr + 1) + kctx_len; - ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); + ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; @@ -1068,7 +1068,7 @@ crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else - ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); + ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); @@ -1168,30 +1168,30 @@ MAX_REQUEST_SIZE) return (EFBIG); } - sglist_reset(sc->sg_dsgl); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + + sglist_reset(s->sg_dsgl); + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crp->crp_aad_length); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); + dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); @@ -1225,29 +1225,29 @@ sgl_len = 0; } else { imm_len = 0; - sglist_reset(sc->sg_ulptx); + sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) - error = sglist_append(sc->sg_ulptx, + error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else - error = sglist_append_sglist(sc->sg_ulptx, - sc->sg_input, crp->crp_aad_start, + error = sglist_append_sglist(s->sg_ulptx, + s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - sgl_nsegs = sc->sg_ulptx->sg_nseg; + sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } @@ -1274,7 +1274,7 @@ return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { - sc->stats_wr_nomem++; + counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); @@ -1339,7 +1339,7 @@ memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); dst = (char *)(crwr + 1) + kctx_len; - ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); + ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; @@ -1359,7 +1359,7 @@ crypto_copydata(crp, crp->crp_digest_start, hash_size_in_response, dst); } else - ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); + ccr_write_ulptx_sgl(s, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); @@ -1642,30 +1642,30 @@ MAX_REQUEST_SIZE) return (EFBIG); } - sglist_reset(sc->sg_dsgl); - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + + sglist_reset(s->sg_dsgl); + error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len + aad_len); if (error) return (error); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_payload_output_start, crp->crp_payload_length); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_output, + error = sglist_append_sglist(s->sg_dsgl, s->sg_output, crp->crp_digest_start, hash_size_in_response); else - error = sglist_append_sglist(sc->sg_dsgl, sc->sg_input, + error = sglist_append_sglist(s->sg_dsgl, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); + dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); @@ -1701,29 +1701,29 @@ /* Block 0 is passed as immediate data. */ imm_len = b0_len; - sglist_reset(sc->sg_ulptx); + sglist_reset(s->sg_ulptx); if (crp->crp_aad_length != 0) { if (crp->crp_aad != NULL) - error = sglist_append(sc->sg_ulptx, + error = sglist_append(s->sg_ulptx, crp->crp_aad, crp->crp_aad_length); else - error = sglist_append_sglist(sc->sg_ulptx, - sc->sg_input, crp->crp_aad_start, + error = sglist_append_sglist(s->sg_ulptx, + s->sg_input, crp->crp_aad_start, crp->crp_aad_length); if (error) return (error); } - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_payload_start, crp->crp_payload_length); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { - error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, + error = sglist_append_sglist(s->sg_ulptx, s->sg_input, crp->crp_digest_start, hash_size_in_response); if (error) return (error); } - sgl_nsegs = sc->sg_ulptx->sg_nseg; + sgl_nsegs = s->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } @@ -1745,7 +1745,7 @@ return (EFBIG); wr = alloc_wrqe(wr_len, s->port->txq); if (wr == NULL) { - sc->stats_wr_nomem++; + counter_u64_add(sc->stats_wr_nomem, 1); return (ENOMEM); } crwr = wrtod(wr); @@ -1809,7 +1809,7 @@ s->blkcipher.enckey, s->blkcipher.key_len); dst = (char *)(crwr + 1) + kctx_len; - ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); + ccr_write_phys_dsgl(s, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; @@ -1847,7 +1847,7 @@ idata->len = htobe32(0); dst = (void *)(idata + 1); } - ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); + ccr_write_ulptx_sgl(s, dst, sgl_nsegs); } /* XXX: TODO backpressure */ @@ -2054,47 +2054,50 @@ CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, - &sc->stats_hash, 0, "Hash requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, - &sc->stats_hmac, 0, "HMAC requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, - &sc->stats_blkcipher_encrypt, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, + &sc->stats_hash, "Hash requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, + &sc->stats_hmac, "HMAC requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt", + CTLFLAG_RD, &sc->stats_blkcipher_encrypt, "Cipher encryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, - &sc->stats_blkcipher_decrypt, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt", + CTLFLAG_RD, &sc->stats_blkcipher_decrypt, "Cipher decryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD, - &sc->stats_eta_encrypt, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt", + CTLFLAG_RD, &sc->stats_eta_encrypt, "Combined AES+HMAC encryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD, - &sc->stats_eta_decrypt, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt", + CTLFLAG_RD, &sc->stats_eta_decrypt, "Combined AES+HMAC decryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, - &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, - &sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD, - &sc->stats_ccm_encrypt, 0, "AES-CCM encryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD, - &sc->stats_ccm_decrypt, 0, "AES-CCM decryption requests submitted"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, - &sc->stats_wr_nomem, 0, "Work request memory allocation failures"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, - &sc->stats_inflight, 0, "Requests currently pending"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, - &sc->stats_mac_error, 0, "MAC errors"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, - &sc->stats_pad_error, 0, "Padding errors"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD, - &sc->stats_bad_session, 0, "Requests with invalid session ID"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, - &sc->stats_sglist_error, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt", + CTLFLAG_RD, &sc->stats_gcm_encrypt, + "AES-GCM encryption requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt", + CTLFLAG_RD, &sc->stats_gcm_decrypt, + "AES-GCM decryption requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt", + CTLFLAG_RD, &sc->stats_ccm_encrypt, + "AES-CCM encryption requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt", + CTLFLAG_RD, &sc->stats_ccm_decrypt, + "AES-CCM decryption requests submitted"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, + &sc->stats_wr_nomem, "Work request memory allocation failures"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, + &sc->stats_inflight, "Requests currently pending"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, + &sc->stats_mac_error, "MAC errors"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, + &sc->stats_pad_error, "Padding errors"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error", + CTLFLAG_RD, &sc->stats_sglist_error, "Requests for which DMA mapping failed"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, - &sc->stats_process_error, 0, "Requests failed during queueing"); - SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, - &sc->stats_sw_fallback, 0, + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error", + CTLFLAG_RD, &sc->stats_process_error, + "Requests failed during queueing"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback", + CTLFLAG_RD, &sc->stats_sw_fallback, "Requests processed by falling back to software"); /* @@ -2157,12 +2160,25 @@ sc->adapter->ccr_softc = sc; mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); - sc->sg_input = sglist_alloc(TX_SGL_SEGS, M_WAITOK); - sc->sg_output = sglist_alloc(TX_SGL_SEGS, M_WAITOK); - sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK); - sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); + sc->stats_blkcipher_encrypt = counter_u64_alloc(M_WAITOK); + sc->stats_blkcipher_decrypt = counter_u64_alloc(M_WAITOK); + sc->stats_hash = counter_u64_alloc(M_WAITOK); + sc->stats_hmac = counter_u64_alloc(M_WAITOK); + sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK); + sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK); + sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK); + sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK); + sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK); + sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK); + sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK); + sc->stats_inflight = counter_u64_alloc(M_WAITOK); + sc->stats_mac_error = counter_u64_alloc(M_WAITOK); + sc->stats_pad_error = counter_u64_alloc(M_WAITOK); + sc->stats_sglist_error = counter_u64_alloc(M_WAITOK); + sc->stats_process_error = counter_u64_alloc(M_WAITOK); + sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK); ccr_sysctls(sc); return (0); @@ -2182,12 +2198,25 @@ crypto_unregister_all(sc->cid); mtx_destroy(&sc->lock); + counter_u64_free(sc->stats_blkcipher_encrypt); + counter_u64_free(sc->stats_blkcipher_decrypt); + counter_u64_free(sc->stats_hash); + counter_u64_free(sc->stats_hmac); + counter_u64_free(sc->stats_eta_encrypt); + counter_u64_free(sc->stats_eta_decrypt); + counter_u64_free(sc->stats_gcm_encrypt); + counter_u64_free(sc->stats_gcm_decrypt); + counter_u64_free(sc->stats_ccm_encrypt); + counter_u64_free(sc->stats_ccm_decrypt); + counter_u64_free(sc->stats_wr_nomem); + counter_u64_free(sc->stats_inflight); + counter_u64_free(sc->stats_mac_error); + counter_u64_free(sc->stats_pad_error); + counter_u64_free(sc->stats_sglist_error); + counter_u64_free(sc->stats_process_error); + counter_u64_free(sc->stats_sw_fallback); sglist_free(sc->sg_iv_aad); free(sc->iv_aad_buf, M_CCR); - sglist_free(sc->sg_dsgl); - sglist_free(sc->sg_ulptx); - sglist_free(sc->sg_output); - sglist_free(sc->sg_input); sc->adapter->ccr_softc = NULL; return (0); } @@ -2459,6 +2488,16 @@ return (best); } +static void +ccr_delete_session(struct ccr_session *s) +{ + sglist_free(s->sg_input); + sglist_free(s->sg_output); + sglist_free(s->sg_ulptx); + sglist_free(s->sg_dsgl); + mtx_destroy(&s->lock); +} + static int ccr_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) @@ -2547,18 +2586,31 @@ } #endif + s = crypto_get_driver_session(cses); + mtx_init(&s->lock, "ccr session", NULL, MTX_DEF); + s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); + s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); + s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT); + s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT); + if (s->sg_input == NULL || s->sg_output == NULL || + s->sg_ulptx == NULL || s->sg_dsgl == NULL) { + ccr_delete_session(s); + return (ENOMEM); + } + sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); + ccr_delete_session(s); return (ENXIO); } - s = crypto_get_driver_session(cses); s->port = ccr_choose_port(sc); if (s->port == NULL) { mtx_unlock(&sc->lock); + ccr_delete_session(s); return (ENXIO); } @@ -2619,7 +2671,6 @@ csp->csp_cipher_klen); } - s->active = true; s->port->active_sessions++; mtx_unlock(&sc->lock); return (0); @@ -2633,14 +2684,16 @@ sc = device_get_softc(dev); s = crypto_get_driver_session(cses); - mtx_lock(&sc->lock); +#ifdef INVARIANTS if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); - s->active = false; +#endif + mtx_lock(&sc->lock); s->port->active_sessions--; mtx_unlock(&sc->lock); + ccr_delete_session(s); } static int @@ -2655,12 +2708,12 @@ s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); - mtx_lock(&sc->lock); - error = ccr_populate_sglist(sc->sg_input, &crp->crp_buf); + mtx_lock(&s->lock); + error = ccr_populate_sglist(s->sg_input, &crp->crp_buf); if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) - error = ccr_populate_sglist(sc->sg_output, &crp->crp_obuf); + error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf); if (error) { - sc->stats_sglist_error++; + counter_u64_add(sc->stats_sglist_error, 1); goto out; } @@ -2668,7 +2721,7 @@ case HASH: error = ccr_hash(sc, s, crp); if (error == 0) - sc->stats_hash++; + counter_u64_add(sc->stats_hash, 1); break; case HMAC: if (crp->crp_auth_key != NULL) @@ -2677,7 +2730,7 @@ csp->csp_auth_klen, s->hmac.pads); error = ccr_hash(sc, s, crp); if (error == 0) - sc->stats_hmac++; + counter_u64_add(sc->stats_hmac, 1); break; case BLKCIPHER: if (crp->crp_cipher_key != NULL) @@ -2686,9 +2739,9 @@ error = ccr_blkcipher(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) - sc->stats_blkcipher_encrypt++; + counter_u64_add(sc->stats_blkcipher_encrypt, 1); else - sc->stats_blkcipher_decrypt++; + counter_u64_add(sc->stats_blkcipher_decrypt, 1); } break; case ETA: @@ -2702,9 +2755,9 @@ error = ccr_eta(sc, s, crp); if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) - sc->stats_eta_encrypt++; + counter_u64_add(sc->stats_eta_encrypt, 1); else - sc->stats_eta_decrypt++; + counter_u64_add(sc->stats_eta_decrypt, 1); } break; case GCM: @@ -2715,22 +2768,22 @@ csp->csp_cipher_klen); } if (crp->crp_payload_length == 0) { - mtx_unlock(&sc->lock); + mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } error = ccr_gcm(sc, s, crp); if (error == EMSGSIZE) { - sc->stats_sw_fallback++; - mtx_unlock(&sc->lock); + counter_u64_add(sc->stats_sw_fallback, 1); + mtx_unlock(&s->lock); ccr_gcm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) - sc->stats_gcm_encrypt++; + counter_u64_add(sc->stats_gcm_encrypt, 1); else - sc->stats_gcm_decrypt++; + counter_u64_add(sc->stats_gcm_decrypt, 1); } break; case CCM: @@ -2740,28 +2793,30 @@ } error = ccr_ccm(sc, s, crp); if (error == EMSGSIZE) { - sc->stats_sw_fallback++; - mtx_unlock(&sc->lock); + counter_u64_add(sc->stats_sw_fallback, 1); + mtx_unlock(&s->lock); ccr_ccm_soft(s, crp); return (0); } if (error == 0) { if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) - sc->stats_ccm_encrypt++; + counter_u64_add(sc->stats_ccm_encrypt, 1); else - sc->stats_ccm_decrypt++; + counter_u64_add(sc->stats_ccm_decrypt, 1); } break; } if (error == 0) { +#ifdef INVARIANTS s->pending++; - sc->stats_inflight++; +#endif + counter_u64_add(sc->stats_inflight, 1); } else - sc->stats_process_error++; + counter_u64_add(sc->stats_process_error, 1); out: - mtx_unlock(&sc->lock); + mtx_unlock(&s->lock); if (error) { crp->crp_etype = error; @@ -2795,9 +2850,12 @@ else error = 0; - mtx_lock(&sc->lock); +#ifdef INVARIANTS + mtx_lock(&s->lock); s->pending--; - sc->stats_inflight--; + mtx_unlock(&s->lock); +#endif + counter_u64_add(sc->stats_inflight, -1); switch (s->mode) { case HASH: @@ -2820,11 +2878,10 @@ if (error == EBADMSG) { if (CHK_MAC_ERR_BIT(status)) - sc->stats_mac_error++; + counter_u64_add(sc->stats_mac_error, 1); if (CHK_PAD_ERR_BIT(status)) - sc->stats_pad_error++; + counter_u64_add(sc->stats_pad_error, 1); } - mtx_unlock(&sc->lock); crp->crp_etype = error; crypto_done(crp); m_freem(m);