Index: head/sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- head/sys/dev/cxgbe/crypto/t4_crypto.c (revision 340485) +++ head/sys/dev/cxgbe/crypto/t4_crypto.c (revision 340486) @@ -1,2384 +1,2384 @@ /*- * Copyright (c) 2017 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include "common/common.h" #include "crypto/t4_crypto.h" /* * Requests consist of: * * +-------------------------------+ * | struct fw_crypto_lookaside_wr | * +-------------------------------+ * | struct ulp_txpkt | * +-------------------------------+ * | struct ulptx_idata | * +-------------------------------+ * | struct cpl_tx_sec_pdu | * +-------------------------------+ * | struct cpl_tls_tx_scmd_fmt | * +-------------------------------+ * | key context header | * +-------------------------------+ * | AES key | ----- For requests with AES * +-------------------------------+ * | Hash state | ----- For hash-only requests * +-------------------------------+ - * | IPAD (16-byte aligned) | \ * +-------------------------------+ +---- For requests with HMAC * | OPAD (16-byte aligned) | / * +-------------------------------+ - * | GMAC H | ----- For AES-GCM * +-------------------------------+ - * | struct cpl_rx_phys_dsgl | \ * +-------------------------------+ +---- Destination buffer for * | PHYS_DSGL entries | / non-hash-only requests * +-------------------------------+ - * | 16 dummy bytes | ----- Only for HMAC/hash-only requests * +-------------------------------+ * | IV | ----- If immediate IV * +-------------------------------+ * | Payload | ----- If immediate Payload * +-------------------------------+ - * | struct ulptx_sgl | \ * +-------------------------------+ +---- If payload via SGL * | SGL entries | / * +-------------------------------+ - * * Note that the key context must be padded to ensure 16-byte alignment. * For HMAC requests, the key consists of the partial hash of the IPAD * followed by the partial hash of the OPAD. * * Replies consist of: * * +-------------------------------+ * | struct cpl_fw6_pld | * +-------------------------------+ * | hash digest | ----- For HMAC request with * +-------------------------------+ 'hash_size' set in work request * * A 32-bit big-endian error status word is supplied in the last 4 * bytes of data[0] in the CPL_FW6_PLD message. bit 0 indicates a * "MAC" error and bit 1 indicates a "PAD" error. * * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message * in the request is returned in data[1] of the CPL_FW6_PLD message. * * For block cipher replies, the updated IV is supplied in data[2] and * data[3] of the CPL_FW6_PLD message. * * For hash replies where the work request set 'hash_size' to request * a copy of the hash in the reply, the hash digest is supplied * immediately following the CPL_FW6_PLD message. */ /* * The crypto engine supports a maximum AAD size of 511 bytes. */ #define MAX_AAD_LEN 511 /* * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG * entries. While the CPL includes a 16-bit length field, the T6 can * sometimes hang if an error occurs while processing a request with a * single DSGL entry larger than 2k. */ #define MAX_RX_PHYS_DSGL_SGE 32 #define DSGL_SGE_MAXLEN 2048 /* * The adapter only supports requests with a total input or output * length of 64k-1 or smaller. Longer requests either result in hung * requests or incorrect results. */ #define MAX_REQUEST_SIZE 65535 static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto"); struct ccr_session_hmac { struct auth_hash *auth_hash; int hash_len; unsigned int partial_digest_len; unsigned int auth_mode; unsigned int mk_size; char ipad[CHCR_HASH_MAX_BLOCK_SIZE_128]; char opad[CHCR_HASH_MAX_BLOCK_SIZE_128]; }; struct ccr_session_gmac { int hash_len; char ghash_h[GMAC_BLOCK_LEN]; }; struct ccr_session_blkcipher { unsigned int cipher_mode; unsigned int key_len; unsigned int iv_len; __be32 key_ctx_hdr; char enckey[CHCR_AES_MAX_KEY_LEN]; char deckey[CHCR_AES_MAX_KEY_LEN]; }; struct ccr_session { bool active; int pending; enum { HASH, HMAC, BLKCIPHER, AUTHENC, GCM } mode; union { struct ccr_session_hmac hmac; struct ccr_session_gmac gmac; }; struct ccr_session_blkcipher blkcipher; }; struct ccr_softc { struct adapter *adapter; device_t dev; uint32_t cid; int tx_channel_id; struct mtx lock; bool detaching; struct sge_wrq *txq; struct sge_rxq *rxq; /* * Pre-allocate S/G lists used when preparing a work request. * 'sg_crp' contains an sglist describing the entire buffer * for a 'struct cryptop'. 'sg_ulptx' is used to describe * the data the engine should DMA as input via ULPTX_SGL. * 'sg_dsgl' is used to describe the destination that cipher * text and a tag should be written to. */ struct sglist *sg_crp; struct sglist *sg_ulptx; struct sglist *sg_dsgl; /* * Pre-allocate a dummy output buffer for the IV and AAD for * AEAD requests. */ char *iv_aad_buf; struct sglist *sg_iv_aad; /* Statistics. */ uint64_t stats_blkcipher_encrypt; uint64_t stats_blkcipher_decrypt; uint64_t stats_hash; uint64_t stats_hmac; uint64_t stats_authenc_encrypt; uint64_t stats_authenc_decrypt; uint64_t stats_gcm_encrypt; uint64_t stats_gcm_decrypt; uint64_t stats_wr_nomem; uint64_t stats_inflight; uint64_t stats_mac_error; uint64_t stats_pad_error; uint64_t stats_bad_session; uint64_t stats_sglist_error; uint64_t stats_process_error; uint64_t stats_sw_fallback; }; /* * Crypto requests involve two kind of scatter/gather lists. * * Non-hash-only requests require a PHYS_DSGL that describes the * location to store the results of the encryption or decryption * operation. This SGL uses a different format (PHYS_DSGL) and should * exclude the crd_skip bytes at the start of the data as well as * any AAD or IV. For authenticated encryption requests it should * cover include the destination of the hash or tag. * * The input payload may either be supplied inline as immediate data, * or via a standard ULP_TX SGL. This SGL should include AAD, * ciphertext, and the hash or tag for authenticated decryption * requests. * * These scatter/gather lists can describe different subsets of the * buffer described by the crypto operation. ccr_populate_sglist() * generates a scatter/gather list that covers the entire crypto * operation buffer that is then used to construct the other * scatter/gather lists. */ static int ccr_populate_sglist(struct sglist *sg, struct cryptop *crp) { int error; sglist_reset(sg); if (crp->crp_flags & CRYPTO_F_IMBUF) error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf); else if (crp->crp_flags & CRYPTO_F_IOV) error = sglist_append_uio(sg, (struct uio *)crp->crp_buf); else error = sglist_append(sg, crp->crp_buf, crp->crp_ilen); return (error); } /* * Segments in 'sg' larger than 'maxsegsize' are counted as multiple * segments. */ static int ccr_count_sgl(struct sglist *sg, int maxsegsize) { int i, nsegs; nsegs = 0; for (i = 0; i < sg->sg_nseg; i++) nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize); return (nsegs); } /* These functions deal with PHYS_DSGL for the reply buffer. */ static inline int ccr_phys_dsgl_len(int nsegs) { int len; len = (nsegs / 8) * sizeof(struct phys_sge_pairs); if ((nsegs % 8) != 0) { len += sizeof(uint16_t) * 8; len += roundup2(nsegs % 8, 2) * sizeof(uint64_t); } return (len); } static void ccr_write_phys_dsgl(struct ccr_softc *sc, void *dst, int nsegs) { struct sglist *sg; struct cpl_rx_phys_dsgl *cpl; struct phys_sge_pairs *sgl; vm_paddr_t paddr; size_t seglen; u_int i, j; sg = sc->sg_dsgl; cpl = dst; cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) | V_CPL_RX_PHYS_DSGL_ISRDMA(0)); cpl->pcirlxorder_to_noofsgentr = htobe32( V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) | V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) | V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) | V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs)); cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; cpl->rss_hdr_int.qid = htobe16(sc->rxq->iq.abs_id); cpl->rss_hdr_int.hash_val = 0; sgl = (struct phys_sge_pairs *)(cpl + 1); j = 0; for (i = 0; i < sg->sg_nseg; i++) { seglen = sg->sg_segs[i].ss_len; paddr = sg->sg_segs[i].ss_paddr; do { sgl->addr[j] = htobe64(paddr); if (seglen > DSGL_SGE_MAXLEN) { sgl->len[j] = htobe16(DSGL_SGE_MAXLEN); paddr += DSGL_SGE_MAXLEN; seglen -= DSGL_SGE_MAXLEN; } else { sgl->len[j] = htobe16(seglen); seglen = 0; } j++; if (j == 8) { sgl++; j = 0; } } while (seglen != 0); } MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs); } /* These functions deal with the ULPTX_SGL for input payload. */ static inline int ccr_ulptx_sgl_len(int nsegs) { u_int n; nsegs--; /* first segment is part of ulptx_sgl */ n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); return (roundup2(n, 16)); } static void ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs) { struct ulptx_sgl *usgl; struct sglist *sg; struct sglist_seg *ss; int i; sg = sc->sg_ulptx; MPASS(nsegs == sg->sg_nseg); ss = &sg->sg_segs[0]; usgl = dst; usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); usgl->len0 = htobe32(ss->ss_len); usgl->addr0 = htobe64(ss->ss_paddr); ss++; for (i = 0; i < sg->sg_nseg - 1; i++) { usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr); ss++; } } static bool ccr_use_imm_data(u_int transhdr_len, u_int input_len) { if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN) return (false); if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) > SGE_MAX_WR_LEN) return (false); return (true); } static void ccr_populate_wreq(struct ccr_softc *sc, struct chcr_wr *crwr, u_int kctx_len, u_int wr_len, u_int imm_len, u_int sgl_len, u_int hash_size, struct cryptop *crp) { u_int cctx_size; cctx_size = sizeof(struct _key_ctx) + kctx_len; crwr->wreq.op_to_cctx_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) | V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) | V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) | V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4)); crwr->wreq.len16_pkd = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16)); crwr->wreq.session_id = 0; crwr->wreq.rx_chid_to_rx_q_id = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(sc->tx_channel_id) | V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) | V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) | V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) | V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) | V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) | V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(sc->rxq->iq.abs_id)); crwr->wreq.key_addr = 0; crwr->wreq.pld_size_hash_size = htobe32( V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) | V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size)); crwr->wreq.cookie = htobe64((uintptr_t)crp); crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DATAMODIFY(0) | V_ULP_TXPKT_CHANNELID(sc->tx_channel_id) | V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(0) | V_ULP_TXPKT_RO(1)); crwr->ulptx.len = htobe32( ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16)); crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | V_ULP_TX_SC_MORE(imm_len != 0 ? 0 : 1)); crwr->sc_imm.len = htobe32(wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len); } static int ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { struct chcr_wr *crwr; struct wrqe *wr; struct auth_hash *axf; struct cryptodesc *crd; char *dst; u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len; u_int hmac_ctrl, imm_len, iopad_size; int error, sgl_nsegs, sgl_len, use_opad; crd = crp->crp_desc; /* Reject requests with too large of an input buffer. */ if (crd->crd_len > MAX_REQUEST_SIZE) return (EFBIG); axf = s->hmac.auth_hash; if (s->mode == HMAC) { use_opad = 1; - hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; } else { use_opad = 0; - hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; + hmac_ctrl = SCMD_HMAC_CTRL_NOP; } /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the context includes the aligned IPAD and * OPAD. */ kctx_len = iopad_size; if (use_opad) kctx_len += iopad_size; hash_size_in_response = axf->hashsize; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); if (crd->crd_len == 0) { imm_len = axf->blocksize; sgl_nsegs = 0; sgl_len = 0; } else if (ccr_use_imm_data(transhdr_len, crd->crd_len)) { imm_len = crd->crd_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crd->crd_skip, crd->crd_len); if (error) return (error); sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, sc->txq); if (wr == NULL) { sc->stats_wr_nomem++; return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, hash_size_in_response, crp); /* XXX: Hardcodes SGE loopback channel of 0. */ crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(0)); crwr->sec_cpl.pldlen = htobe32(crd->crd_len == 0 ? axf->blocksize : crd->crd_len); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | - V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) | - V_SCMD_CIPH_MODE(CHCR_SCMD_CIPHER_MODE_NOP) | + V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | + V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_LAST_FRAG(0) | V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1)); memcpy(crwr->key_ctx.key, s->hmac.ipad, s->hmac.partial_digest_len); if (use_opad) memcpy(crwr->key_ctx.key + iopad_size, s->hmac.opad, s->hmac.partial_digest_len); /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */ kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_OPAD_PRESENT(use_opad) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) | V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1)); dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES; if (crd->crd_len == 0) { dst[0] = 0x80; *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) = htobe64(axf->blocksize << 3); } else if (imm_len != 0) crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip, crd->crd_len, dst); else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { struct cryptodesc *crd; crd = crp->crp_desc; if (error == 0) { crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, s->hmac.hash_len, (c_caddr_t)(cpl + 1)); } return (error); } static int ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; struct cryptodesc *crd; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int imm_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; crd = crp->crp_desc; if (s->blkcipher.key_len == 0 || crd->crd_len == 0) return (EINVAL); if (crd->crd_alg == CRYPTO_AES_CBC && (crd->crd_len % AES_BLOCK_LEN) != 0) return (EINVAL); /* Reject requests with too large of an input buffer. */ if (crd->crd_len > MAX_REQUEST_SIZE) return (EFBIG); if (crd->crd_flags & CRD_F_ENCRYPT) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; sglist_reset(sc->sg_dsgl); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip, crd->crd_len); if (error) return (error); dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* The 'key' must be 128-bit aligned. */ kctx_len = roundup2(s->blkcipher.key_len, 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); if (ccr_use_imm_data(transhdr_len, crd->crd_len + s->blkcipher.iv_len)) { imm_len = crd->crd_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crd->crd_skip, crd->crd_len); if (error) return (error); sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, sc->txq); if (wr == NULL) { sc->stats_wr_nomem++; return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); /* * Read the existing IV from the request or generate a random * one if none is provided. Optionally copy the generated IV * into the output buffer if requested. */ if (op_type == CHCR_ENCRYPT_OP) { if (crd->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crd->crd_iv, s->blkcipher.iv_len); else arc4rand(iv, s->blkcipher.iv_len, 0); if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, s->blkcipher.iv_len, iv); } else { if (crd->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crd->crd_iv, s->blkcipher.iv_len); else crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_inject, s->blkcipher.iv_len, iv); } ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); /* XXX: Hardcodes SGE loopback channel of 0. */ crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | - V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) | + V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | - V_SCMD_AUTH_MODE(CHCR_SCMD_AUTH_MODE_NOP) | - V_SCMD_HMAC_CTRL(CHCR_SCMD_HMAC_CTRL_NOP) | + V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | + V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (crd->crd_alg) { case CRYPTO_AES_CBC: if (crd->crd_flags & CRD_F_ENCRYPT) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case CRYPTO_AES_ICM: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case CRYPTO_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (crd->crd_flags & CRD_F_ENCRYPT) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, s->blkcipher.iv_len); dst += s->blkcipher.iv_len; if (imm_len != 0) crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip, crd->crd_len, dst); else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. */ return (error); } /* * 'hashsize' is the length of a full digest. 'authsize' is the * requested digest length for this operation which may be less * than 'hashsize'. */ static int ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize) { if (authsize == 10) - return (CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366); + return (SCMD_HMAC_CTRL_TRUNC_RFC4366); if (authsize == 12) - return (CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT); + return (SCMD_HMAC_CTRL_IPSEC_96BIT); if (authsize == hashsize / 2) - return (CHCR_SCMD_HMAC_CTRL_DIV2); - return (CHCR_SCMD_HMAC_CTRL_NO_TRUNC); + return (SCMD_HMAC_CTRL_DIV2); + return (SCMD_HMAC_CTRL_NO_TRUNC); } static int ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; struct auth_hash *axf; char *dst; u_int kctx_len, key_half, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len, iopad_size; u_int aad_start, aad_len, aad_stop; u_int auth_start, auth_stop, auth_insert; u_int cipher_start, cipher_stop; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; /* * If there is a need in the future, requests with an empty * payload could be supported as HMAC-only requests. */ if (s->blkcipher.key_len == 0 || crde->crd_len == 0) return (EINVAL); if (crde->crd_alg == CRYPTO_AES_CBC && (crde->crd_len % AES_BLOCK_LEN) != 0) return (EINVAL); /* * Compute the length of the AAD (data covered by the * authentication descriptor but not the encryption * descriptor). To simplify the logic, AAD is only permitted * before the cipher/plain text, not after. This is true of * all currently-generated requests. */ if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip) return (EINVAL); if (crda->crd_skip < crde->crd_skip) { if (crda->crd_skip + crda->crd_len > crde->crd_skip) aad_len = (crde->crd_skip - crda->crd_skip); else aad_len = crda->crd_len; } else aad_len = 0; if (aad_len + s->blkcipher.iv_len > MAX_AAD_LEN) return (EINVAL); axf = s->hmac.auth_hash; hash_size_in_response = s->hmac.hash_len; if (crde->crd_flags & CRD_F_ENCRYPT) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The output buffer consists of the cipher text followed by * the hash when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (s->blkcipher.iv_len + aad_len + crde->crd_len + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (s->blkcipher.iv_len + aad_len + crde->crd_len > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(sc->sg_dsgl); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, s->blkcipher.iv_len + aad_len); if (error) return (error); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip, crde->crd_len); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crda->crd_inject, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* PADs must be 128-bit aligned. */ iopad_size = roundup2(s->hmac.partial_digest_len, 16); /* * The 'key' part of the key context consists of the key followed * by the IPAD and OPAD. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = aad_len + crde->crd_len; /* * The firmware hangs if sent a request which is a * bit smaller than MAX_REQUEST_SIZE. In particular, the * firmware appears to require 512 - 16 bytes of spare room * along with the size of the hash even if the hash isn't * included in the input buffer. */ if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) > MAX_REQUEST_SIZE) return (EFBIG); if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); if (aad_len != 0) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crda->crd_skip, aad_len); if (error) return (error); } error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crde->crd_skip, crde->crd_len); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crda->crd_inject, hash_size_in_response); if (error) return (error); } sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } /* * Any auth-only data before the cipher region is marked as AAD. * Auth-data that overlaps with the cipher region is placed in * the auth section. */ if (aad_len != 0) { aad_start = s->blkcipher.iv_len + 1; aad_stop = aad_start + aad_len - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = s->blkcipher.iv_len + aad_len + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (aad_len == crda->crd_len) { auth_start = 0; auth_stop = 0; } else { if (aad_len != 0) auth_start = cipher_start; else auth_start = s->blkcipher.iv_len + crda->crd_skip - crde->crd_skip + 1; auth_stop = (crde->crd_skip + crde->crd_len) - (crda->crd_skip + crda->crd_len) + cipher_stop; } if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, sc->txq); if (wr == NULL) { sc->stats_wr_nomem++; return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); /* * Read the existing IV from the request or generate a random * one if none is provided. Optionally copy the generated IV * into the output buffer if requested. */ if (op_type == CHCR_ENCRYPT_OP) { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); else arc4rand(iv, s->blkcipher.iv_len, 0); if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, crde->crd_inject, s->blkcipher.iv_len, iv); } else { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); else crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_inject, s->blkcipher.iv_len, iv); } ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp); /* XXX: Hardcodes SGE loopback channel of 0. */ crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len); crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | - V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) | + V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) | V_SCMD_AUTH_MODE(s->hmac.auth_mode) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; switch (crde->crd_alg) { case CRYPTO_AES_CBC: if (crde->crd_flags & CRD_F_ENCRYPT) memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); else memcpy(crwr->key_ctx.key, s->blkcipher.deckey, s->blkcipher.key_len); break; case CRYPTO_AES_ICM: memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); break; case CRYPTO_AES_XTS: key_half = s->blkcipher.key_len / 2; memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half, key_half); if (crde->crd_flags & CRD_F_ENCRYPT) memcpy(crwr->key_ctx.key + key_half, s->blkcipher.enckey, key_half); else memcpy(crwr->key_ctx.key + key_half, s->blkcipher.deckey, key_half); break; } dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->hmac.ipad, s->hmac.partial_digest_len); memcpy(dst + iopad_size, s->hmac.opad, s->hmac.partial_digest_len); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, s->blkcipher.iv_len); dst += s->blkcipher.iv_len; if (imm_len != 0) { if (aad_len != 0) { crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip, aad_len, dst); dst += aad_len; } crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip, crde->crd_len, dst); dst += crde->crd_len; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { struct cryptodesc *crd; /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * For a decryption request, the hardware may do a verification * of the HMAC which will fail if the existing HMAC isn't in the * buffer. If that happens, clear the error and copy the HMAC * from the CPL reply into the buffer. * * For encryption requests, crd should be the cipher request * which will have CRD_F_ENCRYPT set. For decryption * requests, crp_desc will be the HMAC request which should * not have this flag set. */ crd = crp->crp_desc; if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) && !(crd->crd_flags & CRD_F_ENCRYPT)) { crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, s->hmac.hash_len, (c_caddr_t)(cpl + 1)); error = 0; } return (error); } static int ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde) { char iv[CHCR_MAX_CRYPTO_IV_LEN]; struct chcr_wr *crwr; struct wrqe *wr; char *dst; u_int iv_len, kctx_len, op_type, transhdr_len, wr_len; u_int hash_size_in_response, imm_len; u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert; u_int hmac_ctrl, input_len; int dsgl_nsegs, dsgl_len; int sgl_nsegs, sgl_len; int error; if (s->blkcipher.key_len == 0) return (EINVAL); /* * The crypto engine doesn't handle GCM requests with an empty * payload, so handle those in software instead. */ if (crde->crd_len == 0) return (EMSGSIZE); /* * AAD is only permitted before the cipher/plain text, not * after. */ if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip) return (EMSGSIZE); if (crda->crd_len + AES_BLOCK_LEN > MAX_AAD_LEN) return (EMSGSIZE); hash_size_in_response = s->gmac.hash_len; if (crde->crd_flags & CRD_F_ENCRYPT) op_type = CHCR_ENCRYPT_OP; else op_type = CHCR_DECRYPT_OP; /* * The IV handling for GCM in OCF is a bit more complicated in * that IPSec provides a full 16-byte IV (including the * counter), whereas the /dev/crypto interface sometimes * provides a full 16-byte IV (if no IV is provided in the * ioctl) and sometimes a 12-byte IV (if the IV was explicit). * * When provided a 12-byte IV, assume the IV is really 16 bytes * with a counter in the last 4 bytes initialized to 1. * * While iv_len is checked below, the value is currently * always set to 12 when creating a GCM session in this driver * due to limitations in OCF (there is no way to know what the * IV length of a given request will be). This means that the * driver always assumes as 12-byte IV for now. */ if (s->blkcipher.iv_len == 12) iv_len = AES_BLOCK_LEN; else iv_len = s->blkcipher.iv_len; /* * The output buffer consists of the cipher text followed by * the tag when encrypting. For decryption it only contains * the plain text. * * Due to a firmware bug, the output buffer must include a * dummy output buffer for the IV and AAD prior to the real * output buffer. */ if (op_type == CHCR_ENCRYPT_OP) { if (iv_len + crda->crd_len + crde->crd_len + hash_size_in_response > MAX_REQUEST_SIZE) return (EFBIG); } else { if (iv_len + crda->crd_len + crde->crd_len > MAX_REQUEST_SIZE) return (EFBIG); } sglist_reset(sc->sg_dsgl); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len + crda->crd_len); if (error) return (error); error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip, crde->crd_len); if (error) return (error); if (op_type == CHCR_ENCRYPT_OP) { error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crda->crd_inject, hash_size_in_response); if (error) return (error); } dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN); if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE) return (EFBIG); dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs); /* * The 'key' part of the key context consists of the key followed * by the Galois hash key. */ kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len); /* * The input buffer consists of the IV, any AAD, and then the * cipher/plain text. For decryption requests the hash is * appended after the cipher text. * * The IV is always stored at the start of the input buffer * even though it may be duplicated in the payload. The * crypto engine doesn't work properly if the IV offset points * inside of the AAD region, so a second copy is always * required. */ input_len = crda->crd_len + crde->crd_len; if (op_type == CHCR_DECRYPT_OP) input_len += hash_size_in_response; if (input_len > MAX_REQUEST_SIZE) return (EFBIG); if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { imm_len = input_len; sgl_nsegs = 0; sgl_len = 0; } else { imm_len = 0; sglist_reset(sc->sg_ulptx); if (crda->crd_len != 0) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crda->crd_skip, crda->crd_len); if (error) return (error); } error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crde->crd_skip, crde->crd_len); if (error) return (error); if (op_type == CHCR_DECRYPT_OP) { error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp, crda->crd_inject, hash_size_in_response); if (error) return (error); } sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); } if (crda->crd_len != 0) { aad_start = iv_len + 1; aad_stop = aad_start + crda->crd_len - 1; } else { aad_start = 0; aad_stop = 0; } cipher_start = iv_len + crda->crd_len + 1; if (op_type == CHCR_DECRYPT_OP) cipher_stop = hash_size_in_response; else cipher_stop = 0; if (op_type == CHCR_DECRYPT_OP) auth_insert = hash_size_in_response; else auth_insert = 0; wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) + sgl_len; if (wr_len > SGE_MAX_WR_LEN) return (EFBIG); wr = alloc_wrqe(wr_len, sc->txq); if (wr == NULL) { sc->stats_wr_nomem++; return (ENOMEM); } crwr = wrtod(wr); memset(crwr, 0, wr_len); /* * Read the existing IV from the request or generate a random * one if none is provided. Optionally copy the generated IV * into the output buffer if requested. * * If the input IV is 12 bytes, append an explicit 4-byte * counter of 1. */ if (op_type == CHCR_ENCRYPT_OP) { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); else arc4rand(iv, s->blkcipher.iv_len, 0); if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, crde->crd_inject, s->blkcipher.iv_len, iv); } else { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, s->blkcipher.iv_len); else crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_inject, s->blkcipher.iv_len, iv); } if (s->blkcipher.iv_len == 12) *(uint32_t *)&iv[12] = htobe32(1); ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0, crp); /* XXX: Hardcodes SGE loopback channel of 0. */ crwr->sec_cpl.op_ivinsrtofst = htobe32( V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) | V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | V_CPL_TX_SEC_PDU_IVINSRTOFST(1)); crwr->sec_cpl.pldlen = htobe32(iv_len + input_len); /* * NB: cipherstop is explicitly set to 0. On encrypt it * should normally be set to 0 anyway (as the encrypt crd ends * at the end of the input). However, for decrypt the cipher * ends before the tag in the AUTHENC case (and authstop is * set to stop before the tag), but for GCM the cipher still * runs to the end of the buffer. Not sure if this is * intentional or a firmware quirk, but it is required for * working tag validation with GCM decryption. */ crwr->sec_cpl.aadstart_cipherstop_hi = htobe32( V_CPL_TX_SEC_PDU_AADSTART(aad_start) | V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0)); crwr->sec_cpl.cipherstop_lo_authinsert = htobe32( V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) | V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) | V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) | V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response); crwr->sec_cpl.seqno_numivs = htobe32( V_SCMD_SEQ_NO_CTRL(0) | - V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) | + V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(op_type) | V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) | - V_SCMD_CIPH_MODE(CHCR_SCMD_CIPHER_MODE_AES_GCM) | - V_SCMD_AUTH_MODE(CHCR_SCMD_AUTH_MODE_GHASH) | + V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) | + V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) | V_SCMD_HMAC_CTRL(hmac_ctrl) | V_SCMD_IV_SIZE(iv_len / 2) | V_SCMD_NUM_IVS(0)); crwr->sec_cpl.ivgen_hdrlen = htobe32( V_SCMD_IV_GEN_CTRL(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) | V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len)); crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr; memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len); dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16); memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN); dst = (char *)(crwr + 1) + kctx_len; ccr_write_phys_dsgl(sc, dst, dsgl_nsegs); dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; memcpy(dst, iv, iv_len); dst += iv_len; if (imm_len != 0) { if (crda->crd_len != 0) { crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip, crda->crd_len, dst); dst += crda->crd_len; } crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip, crde->crd_len, dst); dst += crde->crd_len; if (op_type == CHCR_DECRYPT_OP) crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, hash_size_in_response, dst); } else ccr_write_ulptx_sgl(sc, dst, sgl_nsegs); /* XXX: TODO backpressure */ t4_wrq_tx(sc->adapter, wr); return (0); } static int ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error) { /* * The updated IV to permit chained requests is at * cpl->data[2], but OCF doesn't permit chained requests. * * Note that the hardware should always verify the GMAC hash. */ return (error); } /* * Handle a GCM request that is not supported by the crypto engine by * performing the operation in software. Derived from swcr_authenc(). */ static void ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde) { struct auth_hash *axf; struct enc_xform *exf; void *auth_ctx; uint8_t *kschedule; char block[GMAC_BLOCK_LEN]; char digest[GMAC_DIGEST_LEN]; char iv[AES_BLOCK_LEN]; int error, i, len; auth_ctx = NULL; kschedule = NULL; /* Initialize the MAC. */ switch (s->blkcipher.key_len) { case 16: axf = &auth_hash_nist_gmac_aes_128; break; case 24: axf = &auth_hash_nist_gmac_aes_192; break; case 32: axf = &auth_hash_nist_gmac_aes_256; break; default: error = EINVAL; goto out; } auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT); if (auth_ctx == NULL) { error = ENOMEM; goto out; } axf->Init(auth_ctx); axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len); /* Initialize the cipher. */ exf = &enc_xform_aes_nist_gcm; error = exf->setkey(&kschedule, s->blkcipher.enckey, s->blkcipher.key_len); if (error) goto out; /* * This assumes a 12-byte IV from the crp. See longer comment * above in ccr_gcm() for more details. */ if (crde->crd_flags & CRD_F_ENCRYPT) { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, 12); else arc4rand(iv, 12, 0); if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, crde->crd_inject, 12, iv); } else { if (crde->crd_flags & CRD_F_IV_EXPLICIT) memcpy(iv, crde->crd_iv, 12); else crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_inject, 12, iv); } *(uint32_t *)&iv[12] = htobe32(1); axf->Reinit(auth_ctx, iv, sizeof(iv)); /* MAC the AAD. */ for (i = 0; i < crda->crd_len; i += sizeof(block)) { len = imin(crda->crd_len - i, sizeof(block)); crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip + i, len, block); bzero(block + len, sizeof(block) - len); axf->Update(auth_ctx, block, sizeof(block)); } exf->reinit(kschedule, iv); /* Do encryption with MAC */ for (i = 0; i < crde->crd_len; i += sizeof(block)) { len = imin(crde->crd_len - i, sizeof(block)); crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip + i, len, block); bzero(block + len, sizeof(block) - len); if (crde->crd_flags & CRD_F_ENCRYPT) { exf->encrypt(kschedule, block); axf->Update(auth_ctx, block, len); crypto_copyback(crp->crp_flags, crp->crp_buf, crde->crd_skip + i, len, block); } else { axf->Update(auth_ctx, block, len); } } /* Length block. */ bzero(block, sizeof(block)); ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8); ((uint32_t *)block)[3] = htobe32(crde->crd_len * 8); axf->Update(auth_ctx, block, sizeof(block)); /* Finalize MAC. */ axf->Final(digest, auth_ctx); /* Inject or validate tag. */ if (crde->crd_flags & CRD_F_ENCRYPT) { crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject, sizeof(digest), digest); error = 0; } else { char digest2[GMAC_DIGEST_LEN]; crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, sizeof(digest2), digest2); if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) { error = 0; /* Tag matches, decrypt data. */ for (i = 0; i < crde->crd_len; i += sizeof(block)) { len = imin(crde->crd_len - i, sizeof(block)); crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip + i, len, block); bzero(block + len, sizeof(block) - len); exf->decrypt(kschedule, block); crypto_copyback(crp->crp_flags, crp->crp_buf, crde->crd_skip + i, len, block); } } else error = EBADMSG; } exf->zerokey(&kschedule); out: if (auth_ctx != NULL) { memset(auth_ctx, 0, axf->ctxsize); free(auth_ctx, M_CCR); } crp->crp_etype = error; crypto_done(crp); } static void ccr_identify(driver_t *driver, device_t parent) { struct adapter *sc; sc = device_get_softc(parent); if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE && device_find_child(parent, "ccr", -1) == NULL) device_add_child(parent, "ccr", -1); } static int ccr_probe(device_t dev) { device_set_desc(dev, "Chelsio Crypto Accelerator"); return (BUS_PROBE_DEFAULT); } static void ccr_sysctls(struct ccr_softc *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; ctx = device_get_sysctl_ctx(sc->dev); /* * dev.ccr.X. */ oid = device_get_sysctl_tree(sc->dev); children = SYSCTL_CHILDREN(oid); /* * dev.ccr.X.stats. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD, &sc->stats_hash, 0, "Hash requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD, &sc->stats_hmac, 0, "HMAC requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD, &sc->stats_blkcipher_encrypt, 0, "Cipher encryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD, &sc->stats_blkcipher_decrypt, 0, "Cipher decryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD, &sc->stats_authenc_encrypt, 0, "Combined AES+HMAC encryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD, &sc->stats_authenc_decrypt, 0, "Combined AES+HMAC decryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD, &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD, &sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD, &sc->stats_wr_nomem, 0, "Work request memory allocation failures"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD, &sc->stats_inflight, 0, "Requests currently pending"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD, &sc->stats_mac_error, 0, "MAC errors"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD, &sc->stats_pad_error, 0, "Padding errors"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD, &sc->stats_bad_session, 0, "Requests with invalid session ID"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD, &sc->stats_sglist_error, 0, "Requests for which DMA mapping failed"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD, &sc->stats_process_error, 0, "Requests failed during queueing"); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD, &sc->stats_sw_fallback, 0, "Requests processed by falling back to software"); } static int ccr_attach(device_t dev) { struct ccr_softc *sc; int32_t cid; /* * TODO: Crypto requests will panic if the parent device isn't * initialized so that the queues are up and running. Need to * figure out how to handle that correctly, maybe just reject * requests if the adapter isn't fully initialized? */ sc = device_get_softc(dev); sc->dev = dev; sc->adapter = device_get_softc(device_get_parent(dev)); sc->txq = &sc->adapter->sge.ctrlq[0]; sc->rxq = &sc->adapter->sge.rxq[0]; cid = crypto_get_driverid(dev, sizeof(struct ccr_session), CRYPTOCAP_F_HARDWARE); if (cid < 0) { device_printf(dev, "could not get crypto driver id\n"); return (ENXIO); } sc->cid = cid; sc->adapter->ccr_softc = sc; /* XXX: TODO? */ sc->tx_channel_id = 0; mtx_init(&sc->lock, "ccr", NULL, MTX_DEF); sc->sg_crp = sglist_alloc(TX_SGL_SEGS, M_WAITOK); sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK); sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK); sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK); sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK); ccr_sysctls(sc); crypto_register(cid, CRYPTO_SHA1, 0, 0); crypto_register(cid, CRYPTO_SHA2_224, 0, 0); crypto_register(cid, CRYPTO_SHA2_256, 0, 0); crypto_register(cid, CRYPTO_SHA2_384, 0, 0); crypto_register(cid, CRYPTO_SHA2_512, 0, 0); crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0); crypto_register(cid, CRYPTO_SHA2_224_HMAC, 0, 0); crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0); crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0); crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0); crypto_register(cid, CRYPTO_AES_CBC, 0, 0); crypto_register(cid, CRYPTO_AES_ICM, 0, 0); crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0); crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); crypto_register(cid, CRYPTO_AES_XTS, 0, 0); return (0); } static int ccr_detach(device_t dev) { struct ccr_softc *sc; sc = device_get_softc(dev); mtx_lock(&sc->lock); sc->detaching = true; mtx_unlock(&sc->lock); crypto_unregister_all(sc->cid); mtx_destroy(&sc->lock); sglist_free(sc->sg_iv_aad); free(sc->iv_aad_buf, M_CCR); sglist_free(sc->sg_dsgl); sglist_free(sc->sg_ulptx); sglist_free(sc->sg_crp); sc->adapter->ccr_softc = NULL; return (0); } static void ccr_copy_partial_hash(void *dst, int cri_alg, union authctx *auth_ctx) { uint32_t *u32; uint64_t *u64; u_int i; u32 = (uint32_t *)dst; u64 = (uint64_t *)dst; switch (cri_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: for (i = 0; i < SHA1_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); break; case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha224ctx.state[i]); break; case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); break; case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); break; case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) u64[i] = htobe64(auth_ctx->sha512ctx.state[i]); break; } } static void ccr_init_hash_digest(struct ccr_session *s, int cri_alg) { union authctx auth_ctx; struct auth_hash *axf; axf = s->hmac.auth_hash; axf->Init(&auth_ctx); ccr_copy_partial_hash(s->hmac.ipad, cri_alg, &auth_ctx); } static void ccr_init_hmac_digest(struct ccr_session *s, int cri_alg, char *key, int klen) { union authctx auth_ctx; struct auth_hash *axf; u_int i; /* * If the key is larger than the block size, use the digest of * the key as the key instead. */ axf = s->hmac.auth_hash; klen /= 8; if (klen > axf->blocksize) { axf->Init(&auth_ctx); axf->Update(&auth_ctx, key, klen); axf->Final(s->hmac.ipad, &auth_ctx); klen = axf->hashsize; } else memcpy(s->hmac.ipad, key, klen); memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); for (i = 0; i < axf->blocksize; i++) { s->hmac.ipad[i] ^= HMAC_IPAD_VAL; s->hmac.opad[i] ^= HMAC_OPAD_VAL; } /* * Hash the raw ipad and opad and store the partial result in * the same buffer. */ axf->Init(&auth_ctx); axf->Update(&auth_ctx, s->hmac.ipad, axf->blocksize); ccr_copy_partial_hash(s->hmac.ipad, cri_alg, &auth_ctx); axf->Init(&auth_ctx); axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize); ccr_copy_partial_hash(s->hmac.opad, cri_alg, &auth_ctx); } /* * Borrowed from AES_GMAC_Setkey(). */ static void ccr_init_gmac_hash(struct ccr_session *s, char *key, int klen) { static char zeroes[GMAC_BLOCK_LEN]; uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; int rounds; rounds = rijndaelKeySetupEnc(keysched, key, klen); rijndaelEncrypt(keysched, rounds, zeroes, s->gmac.ghash_h); } static int ccr_aes_check_keylen(int alg, int klen) { switch (klen) { case 128: case 192: if (alg == CRYPTO_AES_XTS) return (EINVAL); break; case 256: break; case 512: if (alg != CRYPTO_AES_XTS) return (EINVAL); break; default: return (EINVAL); } return (0); } static void ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen) { unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size; unsigned int opad_present; if (alg == CRYPTO_AES_XTS) kbits = klen / 2; else kbits = klen; switch (kbits) { case 128: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; break; case 192: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; break; case 256: ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; break; default: panic("should not get here"); } s->blkcipher.key_len = klen / 8; memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); switch (alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_XTS: t4_aes_getdeckey(s->blkcipher.deckey, key, kbits); break; } kctx_len = roundup2(s->blkcipher.key_len, 16); switch (s->mode) { case AUTHENC: mk_size = s->hmac.mk_size; opad_present = 1; iopad_size = roundup2(s->hmac.partial_digest_len, 16); kctx_len += iopad_size * 2; break; case GCM: mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; opad_present = 0; kctx_len += GMAC_BLOCK_LEN; break; default: mk_size = CHCR_KEYCTX_NO_KEY; opad_present = 0; break; } kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16; s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) | V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) | V_KEY_CONTEXT_OPAD_PRESENT(opad_present) | V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) | V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1)); } static int ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) { struct ccr_softc *sc; struct ccr_session *s; struct auth_hash *auth_hash; struct cryptoini *c, *hash, *cipher; unsigned int auth_mode, cipher_mode, iv_len, mk_size; unsigned int partial_digest_len; int error; bool gcm_hash, hmac; if (cri == NULL) return (EINVAL); gcm_hash = false; hmac = false; cipher = NULL; hash = NULL; auth_hash = NULL; - auth_mode = CHCR_SCMD_AUTH_MODE_NOP; - cipher_mode = CHCR_SCMD_CIPHER_MODE_NOP; + auth_mode = SCMD_AUTH_MODE_NOP; + cipher_mode = SCMD_CIPH_MODE_NOP; iv_len = 0; mk_size = 0; partial_digest_len = 0; for (c = cri; c != NULL; c = c->cri_next) { switch (c->cri_alg) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: case CRYPTO_SHA2_384: case CRYPTO_SHA2_512: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: if (hash) return (EINVAL); hash = c; switch (c->cri_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: auth_hash = &auth_hash_hmac_sha1; - auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; + auth_mode = SCMD_AUTH_MODE_SHA1; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; partial_digest_len = SHA1_HASH_LEN; break; case CRYPTO_SHA2_224: case CRYPTO_SHA2_224_HMAC: auth_hash = &auth_hash_hmac_sha2_224; - auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; + auth_mode = SCMD_AUTH_MODE_SHA224; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: auth_hash = &auth_hash_hmac_sha2_256; - auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; + auth_mode = SCMD_AUTH_MODE_SHA256; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; partial_digest_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: auth_hash = &auth_hash_hmac_sha2_384; - auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; + auth_mode = SCMD_AUTH_MODE_SHA512_384; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: auth_hash = &auth_hash_hmac_sha2_512; - auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; + auth_mode = SCMD_AUTH_MODE_SHA512_512; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; partial_digest_len = SHA2_512_HASH_LEN; break; case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: gcm_hash = true; - auth_mode = CHCR_SCMD_AUTH_MODE_GHASH; + auth_mode = SCMD_AUTH_MODE_GHASH; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; break; } switch (c->cri_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: hmac = true; break; } break; case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_XTS: if (cipher) return (EINVAL); cipher = c; switch (c->cri_alg) { case CRYPTO_AES_CBC: - cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; + cipher_mode = SCMD_CIPH_MODE_AES_CBC; iv_len = AES_BLOCK_LEN; break; case CRYPTO_AES_ICM: - cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; + cipher_mode = SCMD_CIPH_MODE_AES_CTR; iv_len = AES_BLOCK_LEN; break; case CRYPTO_AES_NIST_GCM_16: - cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_GCM; + cipher_mode = SCMD_CIPH_MODE_AES_GCM; iv_len = AES_GCM_IV_LEN; break; case CRYPTO_AES_XTS: - cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; + cipher_mode = SCMD_CIPH_MODE_AES_XTS; iv_len = AES_BLOCK_LEN; break; } if (c->cri_key != NULL) { error = ccr_aes_check_keylen(c->cri_alg, c->cri_klen); if (error) return (error); } break; default: return (EINVAL); } } - if (gcm_hash != (cipher_mode == CHCR_SCMD_CIPHER_MODE_AES_GCM)) + if (gcm_hash != (cipher_mode == SCMD_CIPH_MODE_AES_GCM)) return (EINVAL); if (hash == NULL && cipher == NULL) return (EINVAL); if (hash != NULL) { if ((hmac || gcm_hash) && hash->cri_key == NULL) return (EINVAL); if (!(hmac || gcm_hash) && hash->cri_key != NULL) return (EINVAL); } sc = device_get_softc(dev); mtx_lock(&sc->lock); if (sc->detaching) { mtx_unlock(&sc->lock); return (ENXIO); } s = crypto_get_driver_session(cses); if (gcm_hash) s->mode = GCM; else if (hash != NULL && cipher != NULL) s->mode = AUTHENC; else if (hash != NULL) { if (hmac) s->mode = HMAC; else s->mode = HASH; } else { MPASS(cipher != NULL); s->mode = BLKCIPHER; } if (gcm_hash) { if (hash->cri_mlen == 0) s->gmac.hash_len = AES_GMAC_HASH_LEN; else s->gmac.hash_len = hash->cri_mlen; ccr_init_gmac_hash(s, hash->cri_key, hash->cri_klen); } else if (hash != NULL) { s->hmac.auth_hash = auth_hash; s->hmac.auth_mode = auth_mode; s->hmac.mk_size = mk_size; s->hmac.partial_digest_len = partial_digest_len; if (hash->cri_mlen == 0) s->hmac.hash_len = auth_hash->hashsize; else s->hmac.hash_len = hash->cri_mlen; if (hmac) ccr_init_hmac_digest(s, hash->cri_alg, hash->cri_key, hash->cri_klen); else ccr_init_hash_digest(s, hash->cri_alg); } if (cipher != NULL) { s->blkcipher.cipher_mode = cipher_mode; s->blkcipher.iv_len = iv_len; if (cipher->cri_key != NULL) ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key, cipher->cri_klen); } s->active = true; mtx_unlock(&sc->lock); return (0); } static void ccr_freesession(device_t dev, crypto_session_t cses) { struct ccr_softc *sc; struct ccr_session *s; sc = device_get_softc(dev); s = crypto_get_driver_session(cses); mtx_lock(&sc->lock); if (s->pending != 0) device_printf(dev, "session %p freed with %d pending requests\n", s, s->pending); s->active = false; mtx_unlock(&sc->lock); } static int ccr_process(device_t dev, struct cryptop *crp, int hint) { struct ccr_softc *sc; struct ccr_session *s; struct cryptodesc *crd, *crda, *crde; int error; if (crp == NULL) return (EINVAL); crd = crp->crp_desc; s = crypto_get_driver_session(crp->crp_session); sc = device_get_softc(dev); mtx_lock(&sc->lock); error = ccr_populate_sglist(sc->sg_crp, crp); if (error) { sc->stats_sglist_error++; goto out; } switch (s->mode) { case HASH: error = ccr_hash(sc, s, crp); if (error == 0) sc->stats_hash++; break; case HMAC: if (crd->crd_flags & CRD_F_KEY_EXPLICIT) ccr_init_hmac_digest(s, crd->crd_alg, crd->crd_key, crd->crd_klen); error = ccr_hash(sc, s, crp); if (error == 0) sc->stats_hmac++; break; case BLKCIPHER: if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { error = ccr_aes_check_keylen(crd->crd_alg, crd->crd_klen); if (error) break; ccr_aes_setkey(s, crd->crd_alg, crd->crd_key, crd->crd_klen); } error = ccr_blkcipher(sc, s, crp); if (error == 0) { if (crd->crd_flags & CRD_F_ENCRYPT) sc->stats_blkcipher_encrypt++; else sc->stats_blkcipher_decrypt++; } break; case AUTHENC: error = 0; switch (crd->crd_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: /* Only encrypt-then-authenticate supported. */ crde = crd; crda = crd->crd_next; if (!(crde->crd_flags & CRD_F_ENCRYPT)) { error = EINVAL; break; } break; default: crda = crd; crde = crd->crd_next; if (crde->crd_flags & CRD_F_ENCRYPT) { error = EINVAL; break; } break; } if (error) break; if (crda->crd_flags & CRD_F_KEY_EXPLICIT) ccr_init_hmac_digest(s, crda->crd_alg, crda->crd_key, crda->crd_klen); if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { error = ccr_aes_check_keylen(crde->crd_alg, crde->crd_klen); if (error) break; ccr_aes_setkey(s, crde->crd_alg, crde->crd_key, crde->crd_klen); } error = ccr_authenc(sc, s, crp, crda, crde); if (error == 0) { if (crde->crd_flags & CRD_F_ENCRYPT) sc->stats_authenc_encrypt++; else sc->stats_authenc_decrypt++; } break; case GCM: error = 0; if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) { crde = crd; crda = crd->crd_next; } else { crda = crd; crde = crd->crd_next; } if (crda->crd_flags & CRD_F_KEY_EXPLICIT) ccr_init_gmac_hash(s, crda->crd_key, crda->crd_klen); if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { error = ccr_aes_check_keylen(crde->crd_alg, crde->crd_klen); if (error) break; ccr_aes_setkey(s, crde->crd_alg, crde->crd_key, crde->crd_klen); } if (crde->crd_len == 0) { mtx_unlock(&sc->lock); ccr_gcm_soft(s, crp, crda, crde); return (0); } error = ccr_gcm(sc, s, crp, crda, crde); if (error == EMSGSIZE) { sc->stats_sw_fallback++; mtx_unlock(&sc->lock); ccr_gcm_soft(s, crp, crda, crde); return (0); } if (error == 0) { if (crde->crd_flags & CRD_F_ENCRYPT) sc->stats_gcm_encrypt++; else sc->stats_gcm_decrypt++; } break; } if (error == 0) { s->pending++; sc->stats_inflight++; } else sc->stats_process_error++; out: mtx_unlock(&sc->lock); if (error) { crp->crp_etype = error; crypto_done(crp); } return (0); } static int do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct ccr_softc *sc = iq->adapter->ccr_softc; struct ccr_session *s; const struct cpl_fw6_pld *cpl; struct cryptop *crp; uint32_t status; int error; if (m != NULL) cpl = mtod(m, const void *); else cpl = (const void *)(rss + 1); crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]); s = crypto_get_driver_session(crp->crp_session); status = be64toh(cpl->data[0]); if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status)) error = EBADMSG; else error = 0; mtx_lock(&sc->lock); s->pending--; sc->stats_inflight--; switch (s->mode) { case HASH: case HMAC: error = ccr_hash_done(sc, s, crp, cpl, error); break; case BLKCIPHER: error = ccr_blkcipher_done(sc, s, crp, cpl, error); break; case AUTHENC: error = ccr_authenc_done(sc, s, crp, cpl, error); break; case GCM: error = ccr_gcm_done(sc, s, crp, cpl, error); break; } if (error == EBADMSG) { if (CHK_MAC_ERR_BIT(status)) sc->stats_mac_error++; if (CHK_PAD_ERR_BIT(status)) sc->stats_pad_error++; } mtx_unlock(&sc->lock); crp->crp_etype = error; crypto_done(crp); m_freem(m); return (0); } static int ccr_modevent(module_t mod, int cmd, void *arg) { switch (cmd) { case MOD_LOAD: t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld); return (0); case MOD_UNLOAD: t4_register_cpl_handler(CPL_FW6_PLD, NULL); return (0); default: return (EOPNOTSUPP); } } static device_method_t ccr_methods[] = { DEVMETHOD(device_identify, ccr_identify), DEVMETHOD(device_probe, ccr_probe), DEVMETHOD(device_attach, ccr_attach), DEVMETHOD(device_detach, ccr_detach), DEVMETHOD(cryptodev_newsession, ccr_newsession), DEVMETHOD(cryptodev_freesession, ccr_freesession), DEVMETHOD(cryptodev_process, ccr_process), DEVMETHOD_END }; static driver_t ccr_driver = { "ccr", ccr_methods, sizeof(struct ccr_softc) }; static devclass_t ccr_devclass; DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL); MODULE_VERSION(ccr, 1); MODULE_DEPEND(ccr, crypto, 1, 1, 1); MODULE_DEPEND(ccr, t6nex, 1, 1, 1); Index: head/sys/dev/cxgbe/crypto/t4_crypto.h =================================================================== --- head/sys/dev/cxgbe/crypto/t4_crypto.h (revision 340485) +++ head/sys/dev/cxgbe/crypto/t4_crypto.h (revision 340486) @@ -1,186 +1,191 @@ /*- * Copyright (c) 2017 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __T4_CRYPTO_H__ #define __T4_CRYPTO_H__ /* From chr_core.h */ #define PAD_ERROR_BIT 1 #define CHK_PAD_ERR_BIT(x) (((x) >> PAD_ERROR_BIT) & 1) #define MAC_ERROR_BIT 0 #define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1) #define MAX_SALT 4 struct _key_ctx { __be32 ctx_hdr; u8 salt[MAX_SALT]; __be64 reserverd; unsigned char key[0]; }; struct chcr_wr { struct fw_crypto_lookaside_wr wreq; struct ulp_txpkt ulptx; struct ulptx_idata sc_imm; struct cpl_tx_sec_pdu sec_cpl; struct _key_ctx key_ctx; }; /* From chr_algo.h */ /* Crypto key context */ #define S_KEY_CONTEXT_CTX_LEN 24 #define M_KEY_CONTEXT_CTX_LEN 0xff #define V_KEY_CONTEXT_CTX_LEN(x) ((x) << S_KEY_CONTEXT_CTX_LEN) #define G_KEY_CONTEXT_CTX_LEN(x) \ (((x) >> S_KEY_CONTEXT_CTX_LEN) & M_KEY_CONTEXT_CTX_LEN) #define S_KEY_CONTEXT_DUAL_CK 12 #define M_KEY_CONTEXT_DUAL_CK 0x1 #define V_KEY_CONTEXT_DUAL_CK(x) ((x) << S_KEY_CONTEXT_DUAL_CK) #define G_KEY_CONTEXT_DUAL_CK(x) \ (((x) >> S_KEY_CONTEXT_DUAL_CK) & M_KEY_CONTEXT_DUAL_CK) #define F_KEY_CONTEXT_DUAL_CK V_KEY_CONTEXT_DUAL_CK(1U) #define S_KEY_CONTEXT_OPAD_PRESENT 11 #define M_KEY_CONTEXT_OPAD_PRESENT 0x1 #define V_KEY_CONTEXT_OPAD_PRESENT(x) ((x) << S_KEY_CONTEXT_OPAD_PRESENT) #define G_KEY_CONTEXT_OPAD_PRESENT(x) \ (((x) >> S_KEY_CONTEXT_OPAD_PRESENT) & \ M_KEY_CONTEXT_OPAD_PRESENT) #define F_KEY_CONTEXT_OPAD_PRESENT V_KEY_CONTEXT_OPAD_PRESENT(1U) #define S_KEY_CONTEXT_SALT_PRESENT 10 #define M_KEY_CONTEXT_SALT_PRESENT 0x1 #define V_KEY_CONTEXT_SALT_PRESENT(x) ((x) << S_KEY_CONTEXT_SALT_PRESENT) #define G_KEY_CONTEXT_SALT_PRESENT(x) \ (((x) >> S_KEY_CONTEXT_SALT_PRESENT) & \ M_KEY_CONTEXT_SALT_PRESENT) #define F_KEY_CONTEXT_SALT_PRESENT V_KEY_CONTEXT_SALT_PRESENT(1U) #define S_KEY_CONTEXT_CK_SIZE 6 #define M_KEY_CONTEXT_CK_SIZE 0xf #define V_KEY_CONTEXT_CK_SIZE(x) ((x) << S_KEY_CONTEXT_CK_SIZE) #define G_KEY_CONTEXT_CK_SIZE(x) \ (((x) >> S_KEY_CONTEXT_CK_SIZE) & M_KEY_CONTEXT_CK_SIZE) #define S_KEY_CONTEXT_MK_SIZE 2 #define M_KEY_CONTEXT_MK_SIZE 0xf #define V_KEY_CONTEXT_MK_SIZE(x) ((x) << S_KEY_CONTEXT_MK_SIZE) #define G_KEY_CONTEXT_MK_SIZE(x) \ (((x) >> S_KEY_CONTEXT_MK_SIZE) & M_KEY_CONTEXT_MK_SIZE) #define S_KEY_CONTEXT_VALID 0 #define M_KEY_CONTEXT_VALID 0x1 #define V_KEY_CONTEXT_VALID(x) ((x) << S_KEY_CONTEXT_VALID) #define G_KEY_CONTEXT_VALID(x) \ (((x) >> S_KEY_CONTEXT_VALID) & \ M_KEY_CONTEXT_VALID) #define F_KEY_CONTEXT_VALID V_KEY_CONTEXT_VALID(1U) #define CHCR_HASH_MAX_DIGEST_SIZE 64 #define DUMMY_BYTES 16 #define TRANSHDR_SIZE(kctx_len)\ (sizeof(struct chcr_wr) +\ kctx_len) #define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \ (TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\ sizeof(struct cpl_rx_phys_dsgl)) #define HASH_TRANSHDR_SIZE(kctx_len)\ (TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES) #define CRYPTO_MAX_IMM_TX_PKT_LEN 256 struct phys_sge_pairs { __be16 len[8]; __be64 addr[8]; }; /* From chr_crypto.h */ #define CHCR_AES_MAX_KEY_LEN (AES_XTS_MAX_KEY) #define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */ #define CHCR_ENCRYPT_OP 0 #define CHCR_DECRYPT_OP 1 -#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 +#define SCMD_ENCDECCTRL_ENCRYPT 0 +#define SCMD_ENCDECCTRL_DECRYPT 1 -#define CHCR_SCMD_CIPHER_MODE_NOP 0 -#define CHCR_SCMD_CIPHER_MODE_AES_CBC 1 -#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 -#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 -#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES 4 -#define CHCR_SCMD_CIPHER_MODE_AES_XTS 6 -#define CHCR_SCMD_CIPHER_MODE_AES_CCM 7 +#define SCMD_PROTO_VERSION_TLS_1_2 0 +#define SCMD_PROTO_VERSION_TLS_1_1 1 +#define SCMD_PROTO_VERSION_GENERIC 4 -#define CHCR_SCMD_AUTH_MODE_NOP 0 -#define CHCR_SCMD_AUTH_MODE_SHA1 1 -#define CHCR_SCMD_AUTH_MODE_SHA224 2 -#define CHCR_SCMD_AUTH_MODE_SHA256 3 -#define CHCR_SCMD_AUTH_MODE_GHASH 4 -#define CHCR_SCMD_AUTH_MODE_SHA512_224 5 -#define CHCR_SCMD_AUTH_MODE_SHA512_256 6 -#define CHCR_SCMD_AUTH_MODE_SHA512_384 7 -#define CHCR_SCMD_AUTH_MODE_SHA512_512 8 -#define CHCR_SCMD_AUTH_MODE_CBCMAC 9 -#define CHCR_SCMD_AUTH_MODE_CMAC 10 +#define SCMD_CIPH_MODE_NOP 0 +#define SCMD_CIPH_MODE_AES_CBC 1 +#define SCMD_CIPH_MODE_AES_GCM 2 +#define SCMD_CIPH_MODE_AES_CTR 3 +#define SCMD_CIPH_MODE_GENERIC_AES 4 +#define SCMD_CIPH_MODE_AES_XTS 6 +#define SCMD_CIPH_MODE_AES_CCM 7 -#define CHCR_SCMD_HMAC_CTRL_NOP 0 -#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC 1 -#define CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366 2 -#define CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT 3 -#define CHCR_SCMD_HMAC_CTRL_PL1 4 -#define CHCR_SCMD_HMAC_CTRL_PL2 5 -#define CHCR_SCMD_HMAC_CTRL_PL3 6 -#define CHCR_SCMD_HMAC_CTRL_DIV2 7 +#define SCMD_AUTH_MODE_NOP 0 +#define SCMD_AUTH_MODE_SHA1 1 +#define SCMD_AUTH_MODE_SHA224 2 +#define SCMD_AUTH_MODE_SHA256 3 +#define SCMD_AUTH_MODE_GHASH 4 +#define SCMD_AUTH_MODE_SHA512_224 5 +#define SCMD_AUTH_MODE_SHA512_256 6 +#define SCMD_AUTH_MODE_SHA512_384 7 +#define SCMD_AUTH_MODE_SHA512_512 8 +#define SCMD_AUTH_MODE_CBCMAC 9 +#define SCMD_AUTH_MODE_CMAC 10 + +#define SCMD_HMAC_CTRL_NOP 0 +#define SCMD_HMAC_CTRL_NO_TRUNC 1 +#define SCMD_HMAC_CTRL_TRUNC_RFC4366 2 +#define SCMD_HMAC_CTRL_IPSEC_96BIT 3 +#define SCMD_HMAC_CTRL_PL1 4 +#define SCMD_HMAC_CTRL_PL2 5 +#define SCMD_HMAC_CTRL_PL3 6 +#define SCMD_HMAC_CTRL_DIV2 7 /* This are not really mac key size. They are intermediate values * of sha engine and its size */ #define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 #define CHCR_KEYCTX_MAC_KEY_SIZE_160 1 #define CHCR_KEYCTX_MAC_KEY_SIZE_192 2 #define CHCR_KEYCTX_MAC_KEY_SIZE_256 3 #define CHCR_KEYCTX_MAC_KEY_SIZE_512 4 #define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 #define CHCR_KEYCTX_CIPHER_KEY_SIZE_192 1 #define CHCR_KEYCTX_CIPHER_KEY_SIZE_256 2 #define CHCR_KEYCTX_NO_KEY 15 #define IV_NOP 0 #define IV_IMMEDIATE 1 #define IV_DSGL 2 #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 #endif /* !__T4_CRYPTO_H__ */ Index: head/sys/dev/cxgbe/tom/t4_tls.c =================================================================== --- head/sys/dev/cxgbe/tom/t4_tls.c (revision 340485) +++ head/sys/dev/cxgbe/tom/t4_tls.c (revision 340486) @@ -1,1643 +1,1644 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017-2018 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_inet.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #ifdef TCP_OFFLOAD #include "common/common.h" #include "common/t4_tcb.h" +#include "crypto/t4_crypto.h" #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" /* * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while * the mbuf is in the ulp_pdu_reclaimq. */ #define tls_tcp_seq PH_loc.thirtytwo[0] /* * Handshake lock used for the handshake timer. Having a global lock * is perhaps not ideal, but it avoids having to use callout_drain() * in tls_uninit_toep() which can't block. Also, the timer shouldn't * actually fire for most connections. */ static struct mtx tls_handshake_lock; static void t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val) { struct adapter *sc = td_adapter(toep->td); t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0); } /* TLS and DTLS common routines */ bool can_tls_offload(struct adapter *sc) { return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); } int tls_tx_key(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; return (tls_ofld->tx_key_addr >= 0); } int tls_rx_key(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; return (tls_ofld->rx_key_addr >= 0); } static int key_size(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); } /* Set TLS Key-Id in TCB */ static void t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) { t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), V_TCB_RX_TLS_KEY_TAG(key_id)); } /* Clear TF_RX_QUIESCE to re-enable receive. */ static void t4_clear_rx_quiesce(struct toepcb *toep) { t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); } static void tls_clr_ofld_mode(struct toepcb *toep) { tls_stop_handshake_timer(toep); /* Operate in PDU extraction mode only. */ t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); t4_clear_rx_quiesce(toep); } static void tls_clr_quiesce(struct toepcb *toep) { tls_stop_handshake_timer(toep); t4_clear_rx_quiesce(toep); } /* * Calculate the TLS data expansion size */ static int tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, unsigned short *pdus_per_ulp) { struct tls_ofld_info *tls_ofld = &toep->tls; struct tls_scmd *scmd = &tls_ofld->scmd0; int expn_size = 0, frag_count = 0, pad_per_pdu = 0, pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; int exp_per_pdu = 0; int hdr_len = TLS_HEADER_LENGTH; do { max_frag_size = tls_ofld->k_ctx.frag_size; if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == SCMD_CIPH_MODE_AES_GCM) { frag_count = (data_len / max_frag_size); exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + hdr_len; expn_size = frag_count * exp_per_pdu; if (full_pdus_only) { *pdus_per_ulp = data_len / (exp_per_pdu + max_frag_size); if (*pdus_per_ulp > 32) *pdus_per_ulp = 32; else if(!*pdus_per_ulp) *pdus_per_ulp = 1; expn_size = (*pdus_per_ulp) * exp_per_pdu; break; } if ((last_frag_size = data_len % max_frag_size) > 0) { frag_count += 1; expn_size += exp_per_pdu; } break; } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != SCMD_CIPH_MODE_NOP) { /* Calculate the number of fragments we can make */ frag_count = (data_len / max_frag_size); if (frag_count > 0) { pad_per_pdu = (((howmany((max_frag_size + tls_ofld->mac_length), CIPHER_BLOCK_SIZE)) * CIPHER_BLOCK_SIZE) - (max_frag_size + tls_ofld->mac_length)); if (!pad_per_pdu) pad_per_pdu = CIPHER_BLOCK_SIZE; exp_per_pdu = pad_per_pdu + tls_ofld->mac_length + hdr_len + CIPHER_BLOCK_SIZE; expn_size = frag_count * exp_per_pdu; } if (full_pdus_only) { *pdus_per_ulp = data_len / (exp_per_pdu + max_frag_size); if (*pdus_per_ulp > 32) *pdus_per_ulp = 32; else if (!*pdus_per_ulp) *pdus_per_ulp = 1; expn_size = (*pdus_per_ulp) * exp_per_pdu; break; } /* Consider the last fragment */ if ((last_frag_size = data_len % max_frag_size) > 0) { pad_last_pdu = (((howmany((last_frag_size + tls_ofld->mac_length), CIPHER_BLOCK_SIZE)) * CIPHER_BLOCK_SIZE) - (last_frag_size + tls_ofld->mac_length)); if (!pad_last_pdu) pad_last_pdu = CIPHER_BLOCK_SIZE; expn_size += (pad_last_pdu + tls_ofld->mac_length + hdr_len + CIPHER_BLOCK_SIZE); } } } while (0); return (expn_size); } /* Copy Key to WR */ static void tls_copy_tx_key(struct toepcb *toep, void *dst) { struct tls_ofld_info *tls_ofld = &toep->tls; struct ulptx_sc_memrd *sc_memrd; struct ulptx_idata *sc; if (tls_ofld->k_ctx.tx_key_info_size <= 0) return; if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { sc = dst; sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); sc->len = htobe32(0); sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | V_ULP_TX_SC_MORE(1) | V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { memcpy(dst, &tls_ofld->k_ctx.tx, tls_ofld->k_ctx.tx_key_info_size); } } /* TLS/DTLS content type for CPL SFO */ static inline unsigned char tls_content_type(unsigned char content_type) { /* * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and * default to "CUSTOM" for all other types including * heartbeat? */ switch (content_type) { case CONTENT_TYPE_CCS: return CPL_TX_TLS_SFO_TYPE_CCS; case CONTENT_TYPE_ALERT: return CPL_TX_TLS_SFO_TYPE_ALERT; case CONTENT_TYPE_HANDSHAKE: return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; case CONTENT_TYPE_HEARTBEAT: return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; } return CPL_TX_TLS_SFO_TYPE_DATA; } static unsigned char get_cipher_key_size(unsigned int ck_size) { switch (ck_size) { case AES_NOP: /* NOP */ return 15; case AES_128: /* AES128 */ return CH_CK_SIZE_128; case AES_192: /* AES192 */ return CH_CK_SIZE_192; case AES_256: /* AES256 */ return CH_CK_SIZE_256; default: return CH_CK_SIZE_256; } } static unsigned char get_mac_key_size(unsigned int mk_size) { switch (mk_size) { case SHA_NOP: /* NOP */ return CH_MK_SIZE_128; case SHA_GHASH: /* GHASH */ case SHA_512: /* SHA512 */ return CH_MK_SIZE_512; case SHA_224: /* SHA2-224 */ return CH_MK_SIZE_192; case SHA_256: /* SHA2-256*/ return CH_MK_SIZE_256; case SHA_384: /* SHA384 */ return CH_MK_SIZE_512; case SHA1: /* SHA1 */ default: return CH_MK_SIZE_160; } } static unsigned int get_proto_ver(int proto_ver) { switch (proto_ver) { case TLS1_2_VERSION: return TLS_1_2_VERSION; case TLS1_1_VERSION: return TLS_1_1_VERSION; case DTLS1_2_VERSION: return DTLS_1_2_VERSION; default: return TLS_VERSION_MAX; } } static void tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) { if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { kwr->u.rxhdr.ivinsert_to_authinsrt = htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); kwr->u.rxhdr.ivpresent_to_rxmk_size &= ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); kwr->u.rxhdr.authmode_to_rxvalid &= ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); } else { kwr->u.rxhdr.ivinsert_to_authinsrt = htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); } } /* Rx key */ static void prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) { unsigned int ck_size = kctx->cipher_secret_size; unsigned int mk_size = kctx->mac_secret_size; int proto_ver = kctx->proto_ver; kwr->u.rxhdr.flitcnt_hmacctrl = ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; kwr->u.rxhdr.protover_ciphmode = V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); kwr->u.rxhdr.authmode_to_rxvalid = V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | V_TLS_KEYCTX_TX_WR_RXVALID(1); kwr->u.rxhdr.ivpresent_to_rxmk_size = V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); tls_rxkey_flit1(kwr, kctx); /* No key reversal for GCM */ if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, (kctx->cipher_secret_size << 3)); memcpy(kwr->keys.edkey + kctx->cipher_secret_size, kctx->rx.key + kctx->cipher_secret_size, (IPAD_SIZE + OPAD_SIZE)); } else { memcpy(kwr->keys.edkey, kctx->rx.key, (kctx->tx_key_info_size - SALT_SIZE)); memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); } } /* Tx key */ static void prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) { unsigned int ck_size = kctx->cipher_secret_size; unsigned int mk_size = kctx->mac_secret_size; kwr->u.txhdr.ctxlen = (kctx->tx_key_info_size >> 4); kwr->u.txhdr.dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | V_TLS_KEYCTX_TX_WR_TXVALID(1); memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); kwr->u.txhdr.dualck_to_txvalid &= ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); } kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); } /* TLS Key memory management */ static int get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx) { struct adapter *sc = td_adapter(toep->td); vmem_addr_t addr; if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, &addr) != 0) return (-1); return (addr); } static void free_keyid(struct toepcb *toep, int keyid) { struct adapter *sc = td_adapter(toep->td); vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); } static void clear_tls_keyid(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; if (tls_ofld->rx_key_addr >= 0) { free_keyid(toep, tls_ofld->rx_key_addr); tls_ofld->rx_key_addr = -1; } if (tls_ofld->tx_key_addr >= 0) { free_keyid(toep, tls_ofld->tx_key_addr); tls_ofld->tx_key_addr = -1; } } static int get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) { return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1)); } static int get_tp_plen_max(struct tls_ofld_info *tls_ofld) { int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); } /* Send request to get the key-id */ static int tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) { struct tls_ofld_info *tls_ofld = &toep->tls; struct adapter *sc = td_adapter(toep->td); struct ofld_tx_sdesc *txsd; int kwrlen, kctxlen, keyid, len; struct wrqe *wr; struct tls_key_req *kwr; struct tls_keyctx *kctx; kwrlen = sizeof(*kwr); kctxlen = roundup2(sizeof(*kctx), 32); len = roundup2(kwrlen + kctxlen, 16); if (toep->txsd_avail == 0) return (EAGAIN); /* Dont initialize key for re-neg */ if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { if ((keyid = get_new_keyid(toep, k_ctx)) < 0) { return (ENOSPC); } } else { keyid = get_keyid(tls_ofld, k_ctx->l_p_key); } wr = alloc_wrqe(len, toep->ofld_txq); if (wr == NULL) { free_keyid(toep, keyid); return (ENOMEM); } kwr = wrtod(wr); memset(kwr, 0, kwrlen); kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | F_FW_WR_ATOMIC); kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | V_FW_WR_FLOWID(toep->tid)); kwr->protocol = get_proto_ver(k_ctx->proto_ver); kwr->mfs = htons(k_ctx->frag_size); kwr->reneg_to_write_rx = k_ctx->l_p_key; /* master command */ kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); kwr->len16 = htobe32((toep->tid << 8) | DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); /* sub command */ kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); kwr->sc_len = htobe32(kctxlen); kctx = (struct tls_keyctx *)(kwr + 1); memset(kctx, 0, kctxlen); if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { tls_ofld->tx_key_addr = keyid; prepare_txkey_wr(kctx, k_ctx); } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { tls_ofld->rx_key_addr = keyid; prepare_rxkey_wr(kctx, k_ctx); } txsd = &toep->txsd[toep->txsd_pidx]; txsd->tx_credits = DIV_ROUND_UP(len, 16); txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; t4_wrq_tx(sc, wr); return (0); } /* Store a key received from SSL in DDR. */ static int program_key_context(struct tcpcb *tp, struct toepcb *toep, struct tls_key_context *uk_ctx) { struct adapter *sc = td_adapter(toep->td); struct tls_ofld_info *tls_ofld = &toep->tls; struct tls_key_context *k_ctx; int error, key_offset; if (tp->t_state != TCPS_ESTABLISHED) { /* * XXX: Matches Linux driver, but not sure this is a * very appropriate error. */ return (ENOENT); } /* Stop timer on handshake completion */ tls_stop_handshake_timer(toep); toep->flags &= ~TPF_FORCE_CREDITS; CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : "KEY_WRITE_TX", uk_ctx->proto_ver); if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && toep->ulp_mode != ULP_MODE_TLS) return (EOPNOTSUPP); /* Don't copy the 'tx' and 'rx' fields. */ k_ctx = &tls_ofld->k_ctx; memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { tls_ofld->rx_key_addr = -1; t4_clear_rx_quiesce(toep); } else { tls_ofld->tx_key_addr = -1; } return (0); } if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { k_ctx->iv_size = 4; k_ctx->mac_first = 0; k_ctx->hmac_ctrl = 0; } else { k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ k_ctx->mac_first = 1; } tls_ofld->scmd0.seqno_numivs = (V_SCMD_SEQ_NO_CTRL(3) | V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | V_SCMD_IV_SIZE(k_ctx->iv_size)); tls_ofld->scmd0.ivgen_hdrlen = (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | V_SCMD_KEY_CTX_INLINE(0) | V_SCMD_TLS_FRAG_ENABLE(1)); tls_ofld->mac_length = k_ctx->mac_secret_size; if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { k_ctx->rx = uk_ctx->rx; /* Dont initialize key for re-neg */ if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) tls_ofld->rx_key_addr = -1; } else { k_ctx->tx = uk_ctx->tx; /* Dont initialize key for re-neg */ if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) tls_ofld->tx_key_addr = -1; } /* Flush pending data before new Tx key becomes active */ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { struct sockbuf *sb; /* XXX: This might not drain everything. */ t4_push_frames(sc, toep, 0); sb = &toep->inp->inp_socket->so_snd; SOCKBUF_LOCK(sb); /* XXX: This asserts that everything has been pushed. */ MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); sb->sb_sndptr = NULL; tls_ofld->sb_off = sbavail(sb); SOCKBUF_UNLOCK(sb); tls_ofld->tx_seq_no = 0; } if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { error = tls_program_key_id(toep, k_ctx); if (error) { /* XXX: Only clear quiesce for KEY_WRITE_RX? */ t4_clear_rx_quiesce(toep); return (error); } } if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { /* * RX key tags are an index into the key portion of MA * memory stored as an offset from the base address in * units of 64 bytes. */ key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; t4_set_tls_keyid(toep, key_offset / 64); t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | V_TF_TLS_CONTROL(1) | V_TF_TLS_ACTIVE(1) | V_TF_TLS_ENABLE(1)))); t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), V_TCB_TLS_SEQ(0)); t4_clear_rx_quiesce(toep); } else { unsigned short pdus_per_ulp; if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) tls_ofld->tx_key_addr = 1; tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); tls_ofld->expn_per_ulp = tls_expansion_size(toep, tls_ofld->fcplenmax, 1, &pdus_per_ulp); tls_ofld->pdus_per_ulp = pdus_per_ulp; tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + tls_ofld->k_ctx.frag_size); } return (0); } /* * In some cases a client connection can hang without sending the * ServerHelloDone message from the NIC to the host. Send a dummy * RX_DATA_ACK with RX_MODULATE to unstick the connection. */ static void tls_send_handshake_ack(void *arg) { struct toepcb *toep = arg; struct tls_ofld_info *tls_ofld = &toep->tls; struct adapter *sc = td_adapter(toep->td); /* * XXX: Does not have the t4_get_tcb() checks to refine the * workaround. */ callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); send_rx_modulate(sc, toep); } static void tls_start_handshake_timer(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; mtx_lock(&tls_handshake_lock); callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, tls_send_handshake_ack, toep); mtx_unlock(&tls_handshake_lock); } void tls_stop_handshake_timer(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; mtx_lock(&tls_handshake_lock); callout_stop(&tls_ofld->handshake_timer); mtx_unlock(&tls_handshake_lock); } int t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) { struct tls_key_context uk_ctx; struct inpcb *inp; struct tcpcb *tp; struct toepcb *toep; int error, optval; error = 0; if (sopt->sopt_dir == SOPT_SET && sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), sizeof(uk_ctx)); if (error) return (error); } inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); INP_WLOCK(inp); if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { INP_WUNLOCK(inp); return (ECONNRESET); } tp = intotcpcb(inp); toep = tp->t_toe; switch (sopt->sopt_dir) { case SOPT_SET: switch (sopt->sopt_name) { case TCP_TLSOM_SET_TLS_CONTEXT: error = program_key_context(tp, toep, &uk_ctx); INP_WUNLOCK(inp); break; case TCP_TLSOM_CLR_TLS_TOM: if (toep->ulp_mode == ULP_MODE_TLS) { CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", __func__, toep->tid); tls_clr_ofld_mode(toep); } else error = EOPNOTSUPP; INP_WUNLOCK(inp); break; case TCP_TLSOM_CLR_QUIES: if (toep->ulp_mode == ULP_MODE_TLS) { CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", __func__, toep->tid); tls_clr_quiesce(toep); } else error = EOPNOTSUPP; INP_WUNLOCK(inp); break; default: INP_WUNLOCK(inp); error = EOPNOTSUPP; break; } break; case SOPT_GET: switch (sopt->sopt_name) { case TCP_TLSOM_GET_TLS_TOM: /* * TLS TX is permitted on any TOE socket, but * TLS RX requires a TLS ULP mode. */ optval = TLS_TOM_NONE; if (can_tls_offload(td_adapter(toep->td))) { switch (toep->ulp_mode) { case ULP_MODE_NONE: case ULP_MODE_TCPDDP: optval = TLS_TOM_TXONLY; break; case ULP_MODE_TLS: optval = TLS_TOM_BOTH; break; } } CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", __func__, toep->tid, optval); INP_WUNLOCK(inp); error = sooptcopyout(sopt, &optval, sizeof(optval)); break; default: INP_WUNLOCK(inp); error = EOPNOTSUPP; break; } break; } return (error); } void tls_init_toep(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; tls_ofld->rx_key_addr = -1; tls_ofld->tx_key_addr = -1; if (toep->ulp_mode == ULP_MODE_TLS) callout_init_mtx(&tls_ofld->handshake_timer, &tls_handshake_lock, 0); } void tls_establish(struct toepcb *toep) { /* * Enable PDU extraction. * * XXX: Supposedly this should be done by the firmware when * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but * in practice this seems to be required. */ CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); toep->flags |= TPF_FORCE_CREDITS; tls_start_handshake_timer(toep); } void tls_uninit_toep(struct toepcb *toep) { if (toep->ulp_mode == ULP_MODE_TLS) tls_stop_handshake_timer(toep); clear_tls_keyid(toep); } #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) #define MIN_OFLD_TLSTX_CREDITS(toep) \ (howmany(sizeof(struct fw_tlstx_data_wr) + \ sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ CIPHER_BLOCK_SIZE + 1, 16)) static inline u_int max_imm_tls_space(int tx_credits) { const int n = 2; /* Use only up to 2 desc for imm. data WR */ int space; KASSERT(tx_credits >= 0 && tx_credits <= MAX_OFLD_TX_CREDITS, ("%s: %d credits", __func__, tx_credits)); if (tx_credits >= (n * EQ_ESIZE) / 16) space = (n * EQ_ESIZE); else space = tx_credits * 16; return (space); } static int count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) { int max_nsegs_1mbuf, n, nsegs; while (skip >= m->m_len) { skip -= m->m_len; m = m->m_next; } nsegs = 0; max_nsegs_1mbuf = 0; while (len > 0) { n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); if (n > max_nsegs_1mbuf) max_nsegs_1mbuf = n; nsegs += n; len -= m->m_len - skip; skip = 0; m = m->m_next; } *max_nsegs_1mbufp = max_nsegs_1mbuf; return (nsegs); } static void write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, unsigned int immdlen, unsigned int plen, unsigned int expn, unsigned int pdus, uint8_t credits, int shove, int imm_ivs) { struct tls_ofld_info *tls_ofld = &toep->tls; unsigned int len = plen + expn; txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | V_FW_TLSTX_DATA_WR_COMPL(1) | V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | V_FW_TLSTX_DATA_WR_LEN16(credits)); txwr->plen = htobe32(len); txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | V_FW_TLSTX_DATA_WR_EXP(expn) | V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); txwr->adjustedplen_pkd = htobe16( V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); txwr->expinplenmax_pkd = htobe16( V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); txwr->pdusinplenmax_pkd = htobe16( V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp)); } static void write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) { struct tls_ofld_info *tls_ofld = &toep->tls; int data_type, seglen; if (plen < tls_ofld->k_ctx.frag_size) seglen = plen; else seglen = tls_ofld->k_ctx.frag_size; data_type = tls_content_type(tls_hdr->type); cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); cpl->pld_len = htobe32(plen); if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) cpl->type_protover = htobe32( V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | V_SCMD_NUM_IVS(pdus)); cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); tls_ofld->tx_seq_no += pdus; } /* * Similar to write_tx_sgl() except that it accepts an optional * trailer buffer for IVs. */ static void write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, void *iv_buffer, int iv_len, int nsegs, int n) { struct mbuf *m; struct ulptx_sgl *usgl = dst; int i, j, rc; struct sglist sg; struct sglist_seg segs[n]; KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); sglist_init(&sg, n, segs); usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(nsegs)); for (m = start; skip >= m->m_len; m = m->m_next) skip -= m->m_len; i = -1; for (m = start; plen > 0; m = m->m_next) { rc = sglist_append(&sg, mtod(m, char *) + skip, m->m_len - skip); if (__predict_false(rc != 0)) panic("%s: sglist_append %d", __func__, rc); plen -= m->m_len - skip; skip = 0; for (j = 0; j < sg.sg_nseg; i++, j++) { if (i < 0) { usgl->len0 = htobe32(segs[j].ss_len); usgl->addr0 = htobe64(segs[j].ss_paddr); } else { usgl->sge[i / 2].len[i & 1] = htobe32(segs[j].ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(segs[j].ss_paddr); } #ifdef INVARIANTS nsegs--; #endif } sglist_reset(&sg); } if (iv_buffer != NULL) { rc = sglist_append(&sg, iv_buffer, iv_len); if (__predict_false(rc != 0)) panic("%s: sglist_append %d", __func__, rc); for (j = 0; j < sg.sg_nseg; i++, j++) { if (i < 0) { usgl->len0 = htobe32(segs[j].ss_len); usgl->addr0 = htobe64(segs[j].ss_paddr); } else { usgl->sge[i / 2].len[i & 1] = htobe32(segs[j].ss_len); usgl->sge[i / 2].addr[i & 1] = htobe64(segs[j].ss_paddr); } #ifdef INVARIANTS nsegs--; #endif } } if (i & 1) usgl->sge[i / 2].len[1] = htobe32(0); KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", __func__, nsegs, start, iv_buffer)); } /* * Similar to t4_push_frames() but handles TLS sockets when TLS offload * is enabled. Rather than transmitting bulk data, the socket buffer * contains TLS records. The work request requires a full TLS record, * so batch mbufs up until a full TLS record is seen. This requires * reading the TLS header out of the start of each record to determine * its length. */ void t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) { struct tls_hdr thdr; struct mbuf *sndptr; struct fw_tlstx_data_wr *txwr; struct cpl_tx_tls_sfo *cpl; struct wrqe *wr; u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; u_int expn_size, iv_len, pdus, sndptroff; struct tls_ofld_info *tls_ofld = &toep->tls; struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; struct ofld_tx_sdesc *txsd; bool imm_ivs, imm_payload; void *iv_buffer, *iv_dst, *buf; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); KASSERT(toep->ulp_mode == ULP_MODE_NONE || toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); KASSERT(tls_tx_key(toep), ("%s: TX key not set for toep %p", __func__, toep)); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", __func__, toep->tid, toep->flags, tp->t_flags); #endif if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; #ifdef RATELIMIT if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; } #endif /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { KASSERT(drop == 0, ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; } txsd = &toep->txsd[toep->txsd_pidx]; for (;;) { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); space = max_imm_tls_space(tx_credits); wr_len = sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + key_size(toep); if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d tx_credits %d min_wr %d space %d", __func__, toep->tid, tx_credits, wr_len + CIPHER_BLOCK_SIZE + 1, space); #endif return; } SOCKBUF_LOCK(sb); sowwakeup = drop; if (drop) { sbdrop_locked(sb, drop); MPASS(tls_ofld->sb_off >= drop); tls_ofld->sb_off -= drop; drop = 0; } /* * Send a FIN if requested, but only if there's no * more data to send. */ if (sbavail(sb) == tls_ofld->sb_off && toep->flags & TPF_SEND_FIN) { if (sowwakeup) sowwakeup_locked(so); else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); t4_close_conn(sc, toep); return; } if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { /* * A full TLS header is not yet queued, stop * for now until more data is added to the * socket buffer. However, if the connection * has been closed, we will never get the rest * of the header so just discard the partial * header and close the connection. */ #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s", __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN"); #endif if (sowwakeup) sowwakeup_locked(so); else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); if (toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); return; } /* Read the header of the next TLS record. */ sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); MPASS(!IS_AIOTX_MBUF(sndptr)); m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); tls_size = htons(thdr.length); plen = TLS_HEADER_LENGTH + tls_size; pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); iv_len = pdus * CIPHER_BLOCK_SIZE; if (sbavail(sb) < tls_ofld->sb_off + plen) { /* * The full TLS record is not yet queued, stop * for now until more data is added to the * socket buffer. However, if the connection * has been closed, we will never get the rest * of the record so just discard the partial * record and close the connection. */ #ifdef VERBOSE_TRACES CTR6(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d plen %d%s", __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, plen, toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN"); #endif if (sowwakeup) sowwakeup_locked(so); else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); if (toep->flags & TPF_SEND_FIN) t4_close_conn(sc, toep); return; } /* Shove if there is no additional data pending. */ shove = (sbavail(sb) == tls_ofld->sb_off + plen) && !(tp->t_flags & TF_MORETOCOME); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf && sb->sb_hiwat < V_tcp_autosndbuf_max && sbused(sb) >= sb->sb_hiwat * 7 / 8) { int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, V_tcp_autosndbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; else sowwakeup = 1; /* room available */ } if (sowwakeup) sowwakeup_locked(so); else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); /* Determine whether to use immediate vs SGL. */ imm_payload = false; imm_ivs = false; if (wr_len + iv_len <= space) { imm_ivs = true; wr_len += iv_len; if (wr_len + tls_size <= space) { wr_len += tls_size; imm_payload = true; } } /* Allocate space for IVs if needed. */ if (!imm_ivs) { iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); if (iv_buffer == NULL) { /* * XXX: How to restart this? */ if (sowwakeup) sowwakeup_locked(so); else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); CTR3(KTR_CXGBE, "%s: tid %d failed to alloc IV space len %d", __func__, toep->tid, iv_len); return; } } else iv_buffer = NULL; /* Determine size of SGL. */ nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ if (!imm_payload) { nsegs = count_mbuf_segs(sndptr, sndptroff + TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); if (!imm_ivs) { int n = sglist_count(iv_buffer, iv_len); nsegs += n; if (n > max_nsegs_1mbuf) max_nsegs_1mbuf = n; } /* Account for SGL in work request length. */ wr_len += sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; } wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; return; } #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", __func__, toep->tid, thdr.type, tls_size, pdus); #endif txwr = wrtod(wr); cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); memset(txwr, 0, roundup2(wr_len, 16)); credits = howmany(wr_len, 16); expn_size = tls_expansion_size(toep, tls_size, 0, NULL); write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); tls_copy_tx_key(toep, cpl + 1); /* Generate random IVs */ buf = (char *)(cpl + 1) + key_size(toep); if (imm_ivs) { MPASS(iv_buffer == NULL); iv_dst = buf; buf = (char *)iv_dst + iv_len; } else iv_dst = iv_buffer; arc4rand(iv_dst, iv_len, 0); if (imm_payload) { m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, tls_size, buf); } else { write_tlstx_sgl(buf, sndptr, sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, iv_len, nsegs, max_nsegs_1mbuf); } KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); toep->tx_credits -= credits; tp->snd_nxt += plen; tp->snd_max += plen; SOCKBUF_LOCK(sb); sbsndptr_adv(sb, sb->sb_sndptr, plen); tls_ofld->sb_off += plen; SOCKBUF_UNLOCK(sb); toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) toep->flags |= TPF_TX_SUSPENDED; KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; txsd->tx_credits = credits; txsd->iv_buffer = iv_buffer; txsd++; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { toep->txsd_pidx = 0; txsd = &toep->txsd[0]; } toep->txsd_avail--; atomic_add_long(&toep->vi->pi->tx_tls_records, 1); atomic_add_long(&toep->vi->pi->tx_tls_octets, plen); t4_l2t_send(sc, wr, toep->l2te); } } /* * For TLS data we place received mbufs received via CPL_TLS_DATA into * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is * received, the completed PDUs are placed into the socket receive * buffer. * * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. */ static int do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_tls_data *cpl = mtod(m, const void *); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp; int len; /* XXX: Should this match do_rx_data instead? */ KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); /* strip off CPL header */ m_adj(m, sizeof(*cpl)); len = m->m_pkthdr.len; atomic_add_long(&toep->vi->pi->rx_tls_octets, len); KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), ("%s: payload length mismatch", __func__)); INP_WLOCK(inp); if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", __func__, tid, len, inp->inp_flags); INP_WUNLOCK(inp); m_freem(m); return (0); } /* Save TCP sequence number. */ m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { #ifdef INVARIANTS panic("Failed to queue TLS data packet"); #else printf("%s: Failed to queue TLS data packet\n", __func__); INP_WUNLOCK(inp); m_freem(m); return (0); #endif } tp = intotcpcb(inp); tp->t_rcvtime = ticks; #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, be32toh(cpl->seq)); #endif INP_WUNLOCK(inp); return (0); } static int do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); struct tlsrx_hdr_pkt *tls_hdr_pkt; unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); struct inpcb *inp = toep->inp; struct tcpcb *tp; struct socket *so; struct sockbuf *sb; struct mbuf *tls_data; int len, pdu_length, pdu_overhead, sb_length; KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); /* strip off CPL header */ m_adj(m, sizeof(*cpl)); len = m->m_pkthdr.len; atomic_add_long(&toep->vi->pi->rx_tls_records, 1); KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), ("%s: payload length mismatch", __func__)); INP_WLOCK(inp); if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", __func__, tid, len, inp->inp_flags); INP_WUNLOCK(inp); m_freem(m); return (0); } pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); tp = intotcpcb(inp); #ifdef VERBOSE_TRACES CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); #endif tp->rcv_nxt += pdu_length; if (tp->rcv_wnd < pdu_length) { toep->tls.rcv_over += pdu_length - tp->rcv_wnd; tp->rcv_wnd = 0; } else tp->rcv_wnd -= pdu_length; /* XXX: Not sure what to do about urgent data. */ /* * The payload of this CPL is the TLS header followed by * additional fields. */ KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), ("%s: payload too small", __func__)); tls_hdr_pkt = mtod(m, void *); /* * Only the TLS header is sent to OpenSSL, so report errors by * altering the record type. */ if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) tls_hdr_pkt->type = CONTENT_TYPE_ERROR; /* Trim this CPL's mbuf to only include the TLS header. */ KASSERT(m->m_len == len && m->m_next == NULL, ("%s: CPL spans multiple mbufs", __func__)); m->m_len = TLS_HEADER_LENGTH; m->m_pkthdr.len = TLS_HEADER_LENGTH; tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); if (tls_data != NULL) { KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, ("%s: sequence mismatch", __func__)); /* * Update the TLS header length to be the length of * the payload data. */ tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); m->m_next = tls_data; m->m_pkthdr.len += tls_data->m_len; } so = inp_inpcbtosocket(inp); sb = &so->so_rcv; SOCKBUF_LOCK(sb); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { struct epoch_tracker et; CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, pdu_length); m_freem(m); SOCKBUF_UNLOCK(sb); INP_WUNLOCK(inp); CURVNET_SET(toep->vnet); INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); } /* * Not all of the bytes on the wire are included in the socket * buffer (e.g. the MAC of the TLS record). However, those * bytes are included in the TCP sequence space. To handle * this, compute the delta for this TLS record in * 'pdu_overhead' and treat those bytes as having already been * "read" by the application for the purposes of expanding the * window. The meat of the TLS record passed to the * application ('sb_length') will still not be counted as * "read" until userland actually reads the bytes. * * XXX: Some of the calculations below are probably still not * really correct. */ sb_length = m->m_pkthdr.len; pdu_overhead = pdu_length - sb_length; toep->rx_credits += pdu_overhead; tp->rcv_wnd += pdu_overhead; tp->rcv_adv += pdu_overhead; /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && sb_length > (sbspace(sb) / 8 * 7)) { unsigned int hiwat = sb->sb_hiwat; unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, V_tcp_autorcvbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; else toep->rx_credits += newsize - hiwat; } KASSERT(toep->sb_cc >= sbused(sb), ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); toep->rx_credits += toep->sb_cc - sbused(sb); sbappendstream_locked(sb, m, 0); toep->sb_cc = sbused(sb); #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %u PDU overhead %d rx_credits %u rcv_wnd %u", __func__, tid, pdu_overhead, toep->rx_credits, tp->rcv_wnd); #endif if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { int credits; credits = send_rx_credits(sc, toep, toep->rx_credits); toep->rx_credits -= credits; tp->rcv_wnd += credits; tp->rcv_adv += credits; } sorwakeup_locked(so); SOCKBUF_UNLOCK_ASSERT(sb); INP_WUNLOCK(inp); CURVNET_RESTORE(); return (0); } void t4_tls_mod_load(void) { mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); } void t4_tls_mod_unload(void) { t4_register_cpl_handler(CPL_TLS_DATA, NULL); t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); mtx_destroy(&tls_handshake_lock); } #endif /* TCP_OFFLOAD */ Index: head/sys/dev/cxgbe/tom/t4_tls.h =================================================================== --- head/sys/dev/cxgbe/tom/t4_tls.h (revision 340485) +++ head/sys/dev/cxgbe/tom/t4_tls.h (revision 340486) @@ -1,593 +1,583 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017-2018 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin , Atul Gupta * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __T4_TLS_H__ #define __T4_TLS_H__ #define TLS1_VERSION 0x0301 #define TLS1_1_VERSION 0x0302 #define TLS1_2_VERSION 0x0303 #define TLS_MAX_VERSION TLS1_2_VERSION #define DTLS1_VERSION 0xFEFF #define DTLS1_2_VERSION 0xFEFD #define DTLS_MAX_VERSION DTLS1_2_VERSION #define DTLS1_VERSION_MAJOR 0xFE /* Custom socket options for TLS+TOE. */ #define MAX_MAC_KSZ 64 /*512 bits */ #define MAX_CIPHER_KSZ 32 /* 256 bits */ #define CIPHER_BLOCK_SZ 16 #define SALT_SIZE 4 /* Can accomodate 16, 11-15 are reserved */ enum { CHSSL_SHA_NOP, CHSSL_SHA1, CHSSL_SHA224, CHSSL_SHA256, CHSSL_GHASH, CHSSL_SHA512_224, CHSSL_SHA512_256, CHSSL_SHA512_384, CHSSL_SHA512_512, CHSSL_CBCMAC, CHSSL_CMAC, }; /* Can accomodate 16, 8-15 are reserved */ enum { CHSSL_CIPH_NOP, CHSSL_AES_CBC, CHSSL_AES_GCM, CHSSL_AES_CTR, CHSSL_AES_GEN, CHSSL_IPSEC_ESP, CHSSL_AES_XTS, CHSSL_AES_CCM, }; /* Key Context Programming Operation type */ #define KEY_WRITE_RX 0x1 #define KEY_WRITE_TX 0x2 #define KEY_DELETE_RX 0x4 #define KEY_DELETE_TX 0x8 #define S_KEY_CLR_LOC 4 #define M_KEY_CLR_LOC 0xf #define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) #define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) #define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) #define S_KEY_GET_LOC 0 #define M_KEY_GET_LOC 0xf #define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) #define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) struct tls_ofld_state { unsigned char enc_mode; unsigned char mac_mode; unsigned char key_loc; unsigned char ofld_mode; unsigned char auth_mode; unsigned char resv[3]; }; struct tls_tx_ctxt { unsigned char salt[SALT_SIZE]; unsigned char key[MAX_CIPHER_KSZ]; unsigned char ipad[MAX_MAC_KSZ]; unsigned char opad[MAX_MAC_KSZ]; }; struct tls_rx_ctxt { unsigned char salt[SALT_SIZE]; unsigned char key[MAX_CIPHER_KSZ]; unsigned char ipad[MAX_MAC_KSZ]; unsigned char opad[MAX_MAC_KSZ]; }; struct tls_key_context { struct tls_tx_ctxt tx; struct tls_rx_ctxt rx; unsigned char l_p_key; unsigned char hmac_ctrl; unsigned char mac_first; unsigned char iv_size; unsigned char iv_ctrl; unsigned char iv_algo; unsigned char tx_seq_no; unsigned char rx_seq_no; struct tls_ofld_state state; unsigned int tx_key_info_size; unsigned int rx_key_info_size; unsigned int frag_size; unsigned int mac_secret_size; unsigned int cipher_secret_size; int proto_ver; unsigned int sock_fd; unsigned short dtls_epoch; unsigned short rsv; }; /* Set with 'struct tls_key_context'. */ #define TCP_TLSOM_SET_TLS_CONTEXT (TCP_VENDOR) /* Get returns int of enabled (1) / disabled (0). */ #define TCP_TLSOM_GET_TLS_TOM (TCP_VENDOR + 1) enum { TLS_TOM_NONE = 0, TLS_TOM_TXONLY, TLS_TOM_BOTH }; /* Set with no value. */ #define TCP_TLSOM_CLR_TLS_TOM (TCP_VENDOR + 2) /* Set with no value. */ #define TCP_TLSOM_CLR_QUIES (TCP_VENDOR + 3) #ifdef _KERNEL /* Timeouts for handshake timer in seconds. */ #define TLS_SRV_HELLO_DONE 9 #define TLS_SRV_HELLO_RD_TM 5 #define TLS_SRV_HELLO_BKOFF_TM 15 #define CONTENT_TYPE_CCS 20 #define CONTENT_TYPE_ALERT 21 #define CONTENT_TYPE_HANDSHAKE 22 #define CONTENT_TYPE_APP_DATA 23 #define CONTENT_TYPE_HEARTBEAT 24 #define CONTENT_TYPE_KEY_CONTEXT 32 #define CONTENT_TYPE_ERROR 127 #define GCM_TAG_SIZE 16 #define AEAD_EXPLICIT_DATA_SIZE 8 #define TLS_HEADER_LENGTH 5 #define TP_TX_PG_SZ 65536 #define FC_TP_PLEN_MAX 17408 #define IPAD_SIZE 64 #define OPAD_SIZE 64 #define KEY_SIZE 32 #define CIPHER_BLOCK_SIZE 16 #define HDR_KCTX_SIZE (IPAD_SIZE + OPAD_SIZE + KEY_SIZE) #define KEY_IN_DDR_SIZE 16 #define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_tx_ctxt), 32) /* MAC KEY SIZE */ #define SHA_NOP 0 #define SHA_GHASH 16 #define SHA_224 28 #define SHA_256 32 #define SHA_384 48 #define SHA_512 64 #define SHA1 20 /* CIPHER KEY SIZE */ #define AES_NOP 0 #define AES_128 16 #define AES_192 24 #define AES_256 32 enum { TLS_1_2_VERSION, TLS_1_1_VERSION, DTLS_1_2_VERSION, TLS_VERSION_MAX, }; enum { CH_EVP_CIPH_STREAM_CIPHER, CH_EVP_CIPH_CBC_MODE, CH_EVP_CIPH_GCM_MODE, CH_EVP_CIPH_CTR_MODE, }; enum { TLS_SFO_WR_CONTEXTLOC_DSGL, TLS_SFO_WR_CONTEXTLOC_IMMEDIATE, TLS_SFO_WR_CONTEXTLOC_DDR, }; enum { CPL_TX_TLS_SFO_TYPE_CCS, CPL_TX_TLS_SFO_TYPE_ALERT, CPL_TX_TLS_SFO_TYPE_HANDSHAKE, CPL_TX_TLS_SFO_TYPE_DATA, CPL_TX_TLS_SFO_TYPE_HEARTBEAT, /* XXX: Shouldn't this be "CUSTOM"? */ }; enum { CH_CK_SIZE_128, CH_CK_SIZE_192, CH_CK_SIZE_256, CH_CK_SIZE_NOP, }; enum { CH_MK_SIZE_128, CH_MK_SIZE_160, CH_MK_SIZE_192, CH_MK_SIZE_256, CH_MK_SIZE_512, CH_MK_SIZE_NOP, }; -#define SCMD_ENCDECCTRL_ENCRYPT 0 -#define SCMD_ENCDECCTRL_DECRYPT 1 - -#define SCMD_CIPH_MODE_NOP 0 -#define SCMD_CIPH_MODE_AES_CBC 1 -#define SCMD_CIPH_MODE_AES_GCM 2 -#define SCMD_CIPH_MODE_AES_CTR 3 -#define SCMD_CIPH_MODE_AES_GEN 4 -#define SCMD_CIPH_MODE_AES_CCM 7 - struct tls_scmd { __be32 seqno_numivs; __be32 ivgen_hdrlen; }; struct tls_ofld_info { struct tls_key_context k_ctx; int key_location; int mac_length; int rx_key_addr; int tx_key_addr; uint64_t tx_seq_no; unsigned short fcplenmax; unsigned short adjusted_plen; unsigned short expn_per_ulp; unsigned short pdus_per_ulp; struct tls_scmd scmd0; u_int sb_off; struct callout handshake_timer; u_int rcv_over; }; struct tls_key_req { __be32 wr_hi; __be32 wr_mid; __be32 ftid; __u8 reneg_to_write_rx; __u8 protocol; __be16 mfs; /* master command */ __be32 cmd; __be32 len16; /* command length */ __be32 dlen; /* data length in 32-byte units */ __be32 kaddr; /* sub-command */ __be32 sc_more; __be32 sc_len; }__packed; struct tls_keyctx { union key_ctx { struct tx_keyctx_hdr { __u8 ctxlen; __u8 r2; __be16 dualck_to_txvalid; __u8 txsalt[4]; __be64 r5; } txhdr; struct rx_keyctx_hdr { __u8 flitcnt_hmacctrl; __u8 protover_ciphmode; __u8 authmode_to_rxvalid; __u8 ivpresent_to_rxmk_size; __u8 rxsalt[4]; __be64 ivinsert_to_authinsrt; } rxhdr; } u; struct keys { __u8 edkey[32]; __u8 ipad[64]; __u8 opad[64]; } keys; }; #define S_TLS_KEYCTX_TX_WR_DUALCK 12 #define M_TLS_KEYCTX_TX_WR_DUALCK 0x1 #define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK) #define G_TLS_KEYCTX_TX_WR_DUALCK(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK) #define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U) #define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 #define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) #define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) #define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 #define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) #define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ M_TLS_KEYCTX_TX_WR_SALT_PRESENT) #define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 #define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf #define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) #define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ M_TLS_KEYCTX_TX_WR_TXCK_SIZE) #define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 #define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf #define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) #define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ M_TLS_KEYCTX_TX_WR_TXMK_SIZE) #define S_TLS_KEYCTX_TX_WR_TXVALID 0 #define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 #define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) #define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) #define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) #define S_TLS_KEYCTX_TX_WR_FLITCNT 3 #define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f #define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT) #define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT) #define S_TLS_KEYCTX_TX_WR_HMACCTRL 0 #define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7 #define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL) #define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL) #define S_TLS_KEYCTX_TX_WR_PROTOVER 4 #define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf #define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \ ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER) #define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER) #define S_TLS_KEYCTX_TX_WR_CIPHMODE 0 #define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf #define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE) #define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE) #define S_TLS_KEYCTX_TX_WR_AUTHMODE 4 #define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf #define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE) #define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE) #define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3 #define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1 #define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) #define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \ M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) #define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \ V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U) #define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1 #define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3 #define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) #define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \ M_TLS_KEYCTX_TX_WR_SEQNUMCTRL) #define S_TLS_KEYCTX_TX_WR_RXVALID 0 #define M_TLS_KEYCTX_TX_WR_RXVALID 0x1 #define V_TLS_KEYCTX_TX_WR_RXVALID(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXVALID) #define G_TLS_KEYCTX_TX_WR_RXVALID(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID) #define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U) #define S_TLS_KEYCTX_TX_WR_IVPRESENT 7 #define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT) #define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \ M_TLS_KEYCTX_TX_WR_IVPRESENT) #define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U) #define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6 #define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1 #define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) #define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \ M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) #define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \ V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U) #define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3 #define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7 #define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE) #define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \ M_TLS_KEYCTX_TX_WR_RXCK_SIZE) #define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0 #define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7 #define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE) #define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \ M_TLS_KEYCTX_TX_WR_RXMK_SIZE) #define S_TLS_KEYCTX_TX_WR_IVINSERT 55 #define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL #define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT) #define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT) #define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47 #define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL #define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST) #define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \ M_TLS_KEYCTX_TX_WR_AADSTRTOFST) #define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39 #define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL #define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST) #define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_AADSTOPOFST) #define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30 #define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL #define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) #define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \ M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) #define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23 #define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f #define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) #define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) #define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14 #define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff #define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) #define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \ M_TLS_KEYCTX_TX_WR_AUTHSRTOFST) #define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7 #define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f #define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) #define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \ M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) #define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0 #define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f #define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT) #define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \ M_TLS_KEYCTX_TX_WR_AUTHINSRT) struct tls_hdr { __u8 type; __be16 version; __be16 length; } __packed; struct tlsrx_hdr_pkt { __u8 type; __be16 version; __be16 length; __be64 tls_seq; __be16 reserved1; __u8 res_to_mac_error; } __packed; /* res_to_mac_error fields */ #define S_TLSRX_HDR_PKT_INTERNAL_ERROR 4 #define M_TLSRX_HDR_PKT_INTERNAL_ERROR 0x1 #define V_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \ ((x) << S_TLSRX_HDR_PKT_INTERNAL_ERROR) #define G_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \ (((x) >> S_TLSRX_HDR_PKT_INTERNAL_ERROR) & M_TLSRX_HDR_PKT_INTERNAL_ERROR) #define F_TLSRX_HDR_PKT_INTERNAL_ERROR V_TLSRX_HDR_PKT_INTERNAL_ERROR(1U) #define S_TLSRX_HDR_PKT_SPP_ERROR 3 #define M_TLSRX_HDR_PKT_SPP_ERROR 0x1 #define V_TLSRX_HDR_PKT_SPP_ERROR(x) ((x) << S_TLSRX_HDR_PKT_SPP_ERROR) #define G_TLSRX_HDR_PKT_SPP_ERROR(x) \ (((x) >> S_TLSRX_HDR_PKT_SPP_ERROR) & M_TLSRX_HDR_PKT_SPP_ERROR) #define F_TLSRX_HDR_PKT_SPP_ERROR V_TLSRX_HDR_PKT_SPP_ERROR(1U) #define S_TLSRX_HDR_PKT_CCDX_ERROR 2 #define M_TLSRX_HDR_PKT_CCDX_ERROR 0x1 #define V_TLSRX_HDR_PKT_CCDX_ERROR(x) ((x) << S_TLSRX_HDR_PKT_CCDX_ERROR) #define G_TLSRX_HDR_PKT_CCDX_ERROR(x) \ (((x) >> S_TLSRX_HDR_PKT_CCDX_ERROR) & M_TLSRX_HDR_PKT_CCDX_ERROR) #define F_TLSRX_HDR_PKT_CCDX_ERROR V_TLSRX_HDR_PKT_CCDX_ERROR(1U) #define S_TLSRX_HDR_PKT_PAD_ERROR 1 #define M_TLSRX_HDR_PKT_PAD_ERROR 0x1 #define V_TLSRX_HDR_PKT_PAD_ERROR(x) ((x) << S_TLSRX_HDR_PKT_PAD_ERROR) #define G_TLSRX_HDR_PKT_PAD_ERROR(x) \ (((x) >> S_TLSRX_HDR_PKT_PAD_ERROR) & M_TLSRX_HDR_PKT_PAD_ERROR) #define F_TLSRX_HDR_PKT_PAD_ERROR V_TLSRX_HDR_PKT_PAD_ERROR(1U) #define S_TLSRX_HDR_PKT_MAC_ERROR 0 #define M_TLSRX_HDR_PKT_MAC_ERROR 0x1 #define V_TLSRX_HDR_PKT_MAC_ERROR(x) ((x) << S_TLSRX_HDR_PKT_MAC_ERROR) #define G_TLSRX_HDR_PKT_MAC_ERROR(x) \ (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR) & M_TLSRX_HDR_PKT_MAC_ERROR) #define F_TLSRX_HDR_PKT_MAC_ERROR V_TLSRX_HDR_PKT_MAC_ERROR(1U) #define M_TLSRX_HDR_PKT_ERROR 0x1F #endif /* _KERNEL */ #endif /* !__T4_TLS_H__ */