Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/crypto/t4_kern_tls.c
- This file was added.
/*- | |||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | |||||
* | |||||
* Copyright (c) 2018-2019 Chelsio Communications, Inc. | |||||
* All rights reserved. | |||||
* Written by: John Baldwin <jhb@FreeBSD.org> | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | |||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | |||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||||
* SUCH DAMAGE. | |||||
*/ | |||||
#include "opt_inet.h" | |||||
#include "opt_inet6.h" | |||||
#include "opt_kern_tls.h" | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD$"); | |||||
#include <sys/param.h> | |||||
#include <sys/ktr.h> | |||||
#include <sys/ktls.h> | |||||
#include <sys/sglist.h> | |||||
#include <sys/socket.h> | |||||
#include <sys/socketvar.h> | |||||
#include <sys/sockbuf.h> | |||||
#include <netinet/in.h> | |||||
#include <netinet/in_pcb.h> | |||||
#include <netinet/ip.h> | |||||
#include <netinet/ip6.h> | |||||
#include <netinet/tcp_var.h> | |||||
#include <opencrypto/cryptodev.h> | |||||
#include <opencrypto/xform.h> | |||||
#include "common/common.h" | |||||
#include "common/t4_regs.h" | |||||
#include "common/t4_regs_values.h" | |||||
#include "common/t4_tcb.h" | |||||
#include "t4_l2t.h" | |||||
#include "t4_clip.h" | |||||
#include "t4_mp_ring.h" | |||||
#include "crypto/t4_crypto.h" | |||||
#if defined(INET) || defined(INET6) | |||||
#define SALT_SIZE 4 | |||||
#define GCM_TAG_SIZE 16 | |||||
#define TLS_HEADER_LENGTH 5 | |||||
#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_keyctx), 32) | |||||
struct tls_scmd { | |||||
__be32 seqno_numivs; | |||||
__be32 ivgen_hdrlen; | |||||
}; | |||||
struct tls_key_req { | |||||
/* FW_ULPTX_WR */ | |||||
__be32 wr_hi; | |||||
__be32 wr_mid; | |||||
__be32 ftid; | |||||
__u8 reneg_to_write_rx; | |||||
__u8 protocol; | |||||
__be16 mfs; | |||||
/* master command */ | |||||
__be32 cmd; | |||||
__be32 len16; /* command length */ | |||||
__be32 dlen; /* data length in 32-byte units */ | |||||
__be32 kaddr; | |||||
/* sub-command */ | |||||
__be32 sc_more; | |||||
__be32 sc_len; | |||||
}__packed; | |||||
struct tls_keyctx { | |||||
struct tx_keyctx_hdr { | |||||
__u8 ctxlen; | |||||
__u8 r2; | |||||
__be16 dualck_to_txvalid; | |||||
__u8 txsalt[4]; | |||||
__be64 r5; | |||||
} txhdr; | |||||
struct keys { | |||||
__u8 edkey[32]; | |||||
__u8 ipad[64]; | |||||
__u8 opad[64]; | |||||
} keys; | |||||
}; | |||||
#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 | |||||
#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 | |||||
#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ | |||||
((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) | |||||
#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ | |||||
(((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ | |||||
M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) | |||||
#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ | |||||
V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) | |||||
#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 | |||||
#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 | |||||
#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ | |||||
((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) | |||||
#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ | |||||
(((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ | |||||
M_TLS_KEYCTX_TX_WR_SALT_PRESENT) | |||||
#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ | |||||
V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) | |||||
#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 | |||||
#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf | |||||
#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ | |||||
((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) | |||||
#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ | |||||
(((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ | |||||
M_TLS_KEYCTX_TX_WR_TXCK_SIZE) | |||||
#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 | |||||
#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf | |||||
#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ | |||||
((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) | |||||
#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ | |||||
(((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ | |||||
M_TLS_KEYCTX_TX_WR_TXMK_SIZE) | |||||
#define S_TLS_KEYCTX_TX_WR_TXVALID 0 | |||||
#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 | |||||
#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ | |||||
((x) << S_TLS_KEYCTX_TX_WR_TXVALID) | |||||
#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ | |||||
(((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) | |||||
#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) | |||||
/* Key Context Programming Operation type */ | |||||
#define KEY_WRITE_RX 0x1 | |||||
#define KEY_WRITE_TX 0x2 | |||||
#define KEY_DELETE_RX 0x4 | |||||
#define KEY_DELETE_TX 0x8 | |||||
struct tlspcb { | |||||
struct cxgbe_snd_tag com; | |||||
struct vi_info *vi; /* virtual interface */ | |||||
struct adapter *sc; | |||||
struct l2t_entry *l2te; /* L2 table entry used by this connection */ | |||||
int tid; /* Connection identifier */ | |||||
int tx_key_addr; | |||||
bool inline_key; | |||||
bool using_timestamps; | |||||
unsigned char enc_mode; | |||||
struct tls_scmd scmd0; | |||||
struct tls_scmd scmd0_short; | |||||
unsigned int tx_key_info_size; | |||||
uint32_t prev_seq; | |||||
uint32_t prev_ack; | |||||
uint32_t prev_tsecr; | |||||
uint16_t prev_win; | |||||
uint16_t prev_mss; | |||||
/* Only used outside of setup and teardown when using inline keys. */ | |||||
struct tls_keyctx keyctx; | |||||
/* Fields only used during setup and teardown. */ | |||||
struct inpcb *inp; /* backpointer to host stack's PCB */ | |||||
struct sge_txq *txq; | |||||
struct sge_wrq *ctrlq; | |||||
struct clip_entry *ce; /* CLIP table entry used by this tid */ | |||||
unsigned char auth_mode; | |||||
unsigned char hmac_ctrl; | |||||
unsigned char mac_first; | |||||
unsigned char iv_size; | |||||
unsigned int frag_size; | |||||
unsigned int cipher_secret_size; | |||||
int proto_ver; | |||||
bool open_pending; | |||||
}; | |||||
static int ktls_setup_keys(struct tlspcb *tlsp, | |||||
const struct ktls_session *tls, struct sge_txq *txq); | |||||
static inline struct tlspcb * | |||||
mst_to_tls(struct m_snd_tag *t) | |||||
{ | |||||
return ((struct tlspcb *)mst_to_cst(t)); | |||||
} | |||||
/* XXX: There are similar versions of these two in tom/t4_tls.c. */ | |||||
static int | |||||
get_new_keyid(struct tlspcb *tlsp) | |||||
{ | |||||
vmem_addr_t addr; | |||||
if (vmem_alloc(tlsp->sc->key_map, TLS_KEY_CONTEXT_SZ, | |||||
M_NOWAIT | M_FIRSTFIT, &addr) != 0) | |||||
return (-1); | |||||
return (addr); | |||||
} | |||||
static void | |||||
free_keyid(struct tlspcb *tlsp, int keyid) | |||||
{ | |||||
CTR3(KTR_CXGBE, "%s: tid %d key addr %#x", __func__, tlsp->tid, keyid); | |||||
vmem_free(tlsp->sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); | |||||
} | |||||
static struct tlspcb * | |||||
alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags) | |||||
{ | |||||
struct port_info *pi = vi->pi; | |||||
struct adapter *sc = pi->adapter; | |||||
struct tlspcb *tlsp; | |||||
tlsp = malloc(sizeof(*tlsp), M_CXGBE, M_ZERO | flags); | |||||
if (tlsp == NULL) | |||||
return (NULL); | |||||
cxgbe_snd_tag_init(&tlsp->com, ifp, IF_SND_TAG_TYPE_TLS); | |||||
tlsp->vi = vi; | |||||
tlsp->sc = sc; | |||||
tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id]; | |||||
tlsp->tid = -1; | |||||
tlsp->tx_key_addr = -1; | |||||
return (tlsp); | |||||
} | |||||
static void | |||||
init_ktls_key_params(struct tlspcb *tlsp, const struct ktls_session *tls) | |||||
{ | |||||
int mac_key_size; | |||||
if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) | |||||
tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_1; | |||||
else | |||||
tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_2; | |||||
tlsp->cipher_secret_size = tls->params.cipher_key_len; | |||||
tlsp->tx_key_info_size = sizeof(struct tx_keyctx_hdr) + | |||||
tlsp->cipher_secret_size; | |||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | |||||
tlsp->auth_mode = SCMD_AUTH_MODE_GHASH; | |||||
tlsp->enc_mode = SCMD_CIPH_MODE_AES_GCM; | |||||
tlsp->iv_size = 4; | |||||
tlsp->mac_first = 0; | |||||
tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NOP; | |||||
tlsp->tx_key_info_size += GMAC_BLOCK_LEN; | |||||
} else { | |||||
switch (tls->params.auth_algorithm) { | |||||
case CRYPTO_SHA1_HMAC: | |||||
mac_key_size = roundup2(SHA1_HASH_LEN, 16); | |||||
tlsp->auth_mode = SCMD_AUTH_MODE_SHA1; | |||||
break; | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
mac_key_size = SHA2_256_HASH_LEN; | |||||
tlsp->auth_mode = SCMD_AUTH_MODE_SHA256; | |||||
break; | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
mac_key_size = SHA2_512_HASH_LEN; | |||||
tlsp->auth_mode = SCMD_AUTH_MODE_SHA512_384; | |||||
break; | |||||
} | |||||
tlsp->enc_mode = SCMD_CIPH_MODE_AES_CBC; | |||||
tlsp->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ | |||||
tlsp->mac_first = 1; | |||||
tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; | |||||
tlsp->tx_key_info_size += mac_key_size * 2; | |||||
} | |||||
tlsp->frag_size = tls->params.max_frame_len; | |||||
} | |||||
static int | |||||
ktls_act_open_cpl_size(bool isipv6) | |||||
{ | |||||
if (isipv6) | |||||
return (sizeof(struct cpl_t6_act_open_req6)); | |||||
else | |||||
return (sizeof(struct cpl_t6_act_open_req)); | |||||
} | |||||
static void | |||||
mk_ktls_act_open_req(struct adapter *sc, struct vi_info *vi, struct inpcb *inp, | |||||
struct tlspcb *tlsp, int atid, void *dst) | |||||
{ | |||||
struct tcpcb *tp = intotcpcb(inp); | |||||
struct cpl_t6_act_open_req *cpl6; | |||||
struct cpl_act_open_req *cpl; | |||||
uint64_t options; | |||||
int qid_atid; | |||||
cpl6 = dst; | |||||
cpl = (struct cpl_act_open_req *)cpl6; | |||||
INIT_TP_WR(cpl6, 0); | |||||
qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | | |||||
V_TID_COOKIE(CPL_COOKIE_KERN_TLS); | |||||
OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, | |||||
qid_atid)); | |||||
inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port, | |||||
&cpl->peer_ip, &cpl->peer_port); | |||||
options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE); | |||||
options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); | |||||
options |= F_NON_OFFLOAD; | |||||
cpl->opt0 = htobe64(options); | |||||
options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]); | |||||
if (tp->t_flags & TF_REQ_TSTMP) | |||||
options |= F_TSTAMPS_EN; | |||||
cpl->opt2 = htobe32(options); | |||||
} | |||||
static void | |||||
mk_ktls_act_open_req6(struct adapter *sc, struct vi_info *vi, | |||||
struct inpcb *inp, struct tlspcb *tlsp, int atid, void *dst) | |||||
{ | |||||
struct tcpcb *tp = intotcpcb(inp); | |||||
struct cpl_t6_act_open_req6 *cpl6; | |||||
struct cpl_act_open_req6 *cpl; | |||||
uint64_t options; | |||||
int qid_atid; | |||||
cpl6 = dst; | |||||
cpl = (struct cpl_act_open_req6 *)cpl6; | |||||
INIT_TP_WR(cpl6, 0); | |||||
qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | | |||||
V_TID_COOKIE(CPL_COOKIE_KERN_TLS); | |||||
OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, | |||||
qid_atid)); | |||||
cpl->local_port = inp->inp_lport; | |||||
cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0]; | |||||
cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8]; | |||||
cpl->peer_port = inp->inp_fport; | |||||
cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0]; | |||||
cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8]; | |||||
options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE); | |||||
options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); | |||||
options |= F_NON_OFFLOAD; | |||||
cpl->opt0 = htobe64(options); | |||||
options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]); | |||||
if (tp->t_flags & TF_REQ_TSTMP) | |||||
options |= F_TSTAMPS_EN; | |||||
cpl->opt2 = htobe32(options); | |||||
} | |||||
static int | |||||
send_ktls_act_open_req(struct adapter *sc, struct vi_info *vi, | |||||
struct inpcb *inp, struct tlspcb *tlsp, int atid) | |||||
{ | |||||
struct wrqe *wr; | |||||
bool isipv6; | |||||
isipv6 = (inp->inp_vflag & INP_IPV6) != 0; | |||||
if (isipv6) { | |||||
tlsp->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL); | |||||
if (tlsp->ce == NULL) | |||||
return (ENOENT); | |||||
} | |||||
wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq); | |||||
if (wr == NULL) { | |||||
CTR2(KTR_CXGBE, "%s: atid %d failed to alloc WR", __func__, | |||||
atid); | |||||
return (ENOMEM); | |||||
} | |||||
if (isipv6) | |||||
mk_ktls_act_open_req6(sc, vi, inp, tlsp, atid, wrtod(wr)); | |||||
else | |||||
mk_ktls_act_open_req(sc, vi, inp, tlsp, atid, wrtod(wr)); | |||||
tlsp->open_pending = true; | |||||
t4_wrq_tx(sc, wr); | |||||
return (0); | |||||
} | |||||
static int | |||||
ktls_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss, | |||||
struct mbuf *m) | |||||
{ | |||||
struct adapter *sc = iq->adapter; | |||||
const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); | |||||
u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); | |||||
u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); | |||||
struct tlspcb *tlsp = lookup_atid(sc, atid); | |||||
struct inpcb *inp = tlsp->inp; | |||||
CTR3(KTR_CXGBE, "%s: atid %d status %d", __func__, atid, status); | |||||
free_atid(sc, atid); | |||||
if (status == 0) | |||||
tlsp->tid = GET_TID(cpl); | |||||
INP_WLOCK(inp); | |||||
tlsp->open_pending = false; | |||||
wakeup(tlsp); | |||||
INP_WUNLOCK(inp); | |||||
return (0); | |||||
} | |||||
/* SET_TCB_FIELD sent as a ULP command looks like this */ | |||||
#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ | |||||
sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) | |||||
_Static_assert((LEN__SET_TCB_FIELD_ULP + sizeof(struct ulptx_idata)) % 16 == 0, | |||||
"CPL_SET_TCB_FIELD ULP command not 16-byte aligned"); | |||||
static void | |||||
write_set_tcb_field_ulp(struct tlspcb *tlsp, void *dst, struct sge_txq *txq, | |||||
uint16_t word, uint64_t mask, uint64_t val) | |||||
{ | |||||
struct ulp_txpkt *txpkt; | |||||
struct ulptx_idata *idata; | |||||
struct cpl_set_tcb_field_core *cpl; | |||||
/* ULP_TXPKT */ | |||||
txpkt = dst; | |||||
txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | | |||||
V_ULP_TXPKT_DATAMODIFY(0) | | |||||
V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | | |||||
V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); | |||||
txpkt->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); | |||||
/* ULPTX_IDATA sub-command */ | |||||
idata = (struct ulptx_idata *)(txpkt + 1); | |||||
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); | |||||
idata->len = htobe32(sizeof(*cpl)); | |||||
/* CPL_SET_TCB_FIELD */ | |||||
cpl = (struct cpl_set_tcb_field_core *)(idata + 1); | |||||
OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid)); | |||||
cpl->reply_ctrl = htobe16(F_NO_REPLY); | |||||
cpl->word_cookie = htobe16(V_WORD(word)); | |||||
cpl->mask = htobe64(mask); | |||||
cpl->val = htobe64(val); | |||||
/* ULPTX_NOOP */ | |||||
idata = (struct ulptx_idata *)(cpl + 1); | |||||
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); | |||||
idata->len = htobe32(0); | |||||
} | |||||
static int | |||||
ktls_set_tcb_fields(struct tlspcb *tlsp, struct tcpcb *tp, struct sge_txq *txq) | |||||
{ | |||||
struct fw_ulptx_wr *wr; | |||||
struct mbuf *m; | |||||
char *dst; | |||||
void *items[1]; | |||||
int error, len; | |||||
len = sizeof(*wr) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
if (tp->t_flags & TF_REQ_TSTMP) | |||||
len += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
m = alloc_wr_mbuf(len, M_NOWAIT); | |||||
if (m == NULL) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__, | |||||
tlsp->tid); | |||||
return (ENOMEM); | |||||
} | |||||
m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com.com); | |||||
m->m_pkthdr.csum_flags |= CSUM_SND_TAG; | |||||
/* FW_ULPTX_WR */ | |||||
wr = mtod(m, void *); | |||||
wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); | |||||
wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | | |||||
V_FW_WR_LEN16(len / 16)); | |||||
wr->cookie = 0; | |||||
dst = (char *)(wr + 1); | |||||
/* Clear TF_NON_OFFLOAD and set TF_CORE_BYPASS */ | |||||
write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_T_FLAGS, | |||||
V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1) | V_TF_NON_OFFLOAD(1)), | |||||
V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1))); | |||||
dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
/* Clear the SND_UNA_RAW, SND_NXT_RAW, and SND_MAX_RAW offsets. */ | |||||
write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_UNA_RAW, | |||||
V_TCB_SND_NXT_RAW(M_TCB_SND_NXT_RAW) | | |||||
V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW), | |||||
V_TCB_SND_NXT_RAW(0) | V_TCB_SND_UNA_RAW(0)); | |||||
dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_MAX_RAW, | |||||
V_TCB_SND_MAX_RAW(M_TCB_SND_MAX_RAW), V_TCB_SND_MAX_RAW(0)); | |||||
dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
if (tp->t_flags & TF_REQ_TSTMP) { | |||||
write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_TIMESTAMP_OFFSET, | |||||
V_TCB_TIMESTAMP_OFFSET(M_TCB_TIMESTAMP_OFFSET), | |||||
V_TCB_TIMESTAMP_OFFSET(tp->ts_offset >> 28)); | |||||
dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
} | |||||
KASSERT(dst - (char *)wr == len, ("%s: length mismatch", __func__)); | |||||
items[0] = m; | |||||
error = mp_ring_enqueue(txq->r, items, 1, 1); | |||||
if (error) | |||||
m_free(m); | |||||
return (error); | |||||
} | |||||
int | |||||
cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, | |||||
struct m_snd_tag **pt) | |||||
{ | |||||
const struct ktls_session *tls; | |||||
struct tlspcb *tlsp; | |||||
struct adapter *sc; | |||||
struct vi_info *vi; | |||||
struct inpcb *inp; | |||||
struct tcpcb *tp; | |||||
struct sge_txq *txq; | |||||
int atid, error, keyid; | |||||
tls = params->tls.tls; | |||||
/* Only TLS 1.1 and TLS 1.2 are currently supported. */ | |||||
if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || | |||||
tls->params.tls_vminor < TLS_MINOR_VER_ONE || | |||||
tls->params.tls_vminor > TLS_MINOR_VER_TWO) | |||||
return (EPROTONOSUPPORT); | |||||
/* Sanity check values in *tls. */ | |||||
switch (tls->params.cipher_algorithm) { | |||||
case CRYPTO_AES_CBC: | |||||
/* XXX: Explicitly ignore any provided IV. */ | |||||
switch (tls->params.cipher_key_len) { | |||||
case 128 / 8: | |||||
case 192 / 8: | |||||
case 256 / 8: | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
switch (tls->params.auth_algorithm) { | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
break; | |||||
default: | |||||
return (EPROTONOSUPPORT); | |||||
} | |||||
break; | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
if (tls->params.iv_len != SALT_SIZE) | |||||
return (EINVAL); | |||||
switch (tls->params.cipher_key_len) { | |||||
case 128 / 8: | |||||
case 192 / 8: | |||||
case 256 / 8: | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
break; | |||||
default: | |||||
return (EPROTONOSUPPORT); | |||||
} | |||||
vi = ifp->if_softc; | |||||
sc = vi->pi->adapter; | |||||
tlsp = alloc_tlspcb(ifp, vi, M_WAITOK); | |||||
atid = alloc_atid(sc, tlsp); | |||||
if (atid < 0) { | |||||
error = ENOMEM; | |||||
goto failed; | |||||
} | |||||
if (sc->tlst.inline_keys) | |||||
keyid = -1; | |||||
else | |||||
keyid = get_new_keyid(tlsp); | |||||
if (keyid < 0) { | |||||
CTR2(KTR_CXGBE, "%s: atid %d using immediate key ctx", __func__, | |||||
atid); | |||||
tlsp->inline_key = true; | |||||
} else { | |||||
tlsp->tx_key_addr = keyid; | |||||
CTR3(KTR_CXGBE, "%s: atid %d allocated TX key addr %#x", | |||||
__func__, | |||||
atid, tlsp->tx_key_addr); | |||||
} | |||||
inp = params->tls.inp; | |||||
INP_RLOCK(inp); | |||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | |||||
INP_RUNLOCK(inp); | |||||
error = ECONNRESET; | |||||
goto failed; | |||||
} | |||||
tlsp->inp = inp; | |||||
tp = inp->inp_ppcb; | |||||
if (tp->t_flags & TF_REQ_TSTMP) { | |||||
tlsp->using_timestamps = true; | |||||
if ((tp->ts_offset & 0xfffffff) != 0) { | |||||
INP_RUNLOCK(inp); | |||||
error = EINVAL; | |||||
goto failed; | |||||
} | |||||
} else | |||||
tlsp->using_timestamps = false; | |||||
error = send_ktls_act_open_req(sc, vi, inp, tlsp, atid); | |||||
if (error) { | |||||
INP_RUNLOCK(inp); | |||||
goto failed; | |||||
} | |||||
/* Wait for reply to active open. */ | |||||
CTR2(KTR_CXGBE, "%s: atid %d sent CPL_ACT_OPEN_REQ", __func__, | |||||
atid); | |||||
while (tlsp->open_pending) { | |||||
/* | |||||
* XXX: PCATCH? We would then have to discard the PCB | |||||
* when the completion CPL arrived. | |||||
*/ | |||||
error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0); | |||||
} | |||||
atid = -1; | |||||
if (tlsp->tid < 0) { | |||||
INP_RUNLOCK(inp); | |||||
error = ENOMEM; | |||||
goto failed; | |||||
} | |||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | |||||
INP_RUNLOCK(inp); | |||||
error = ECONNRESET; | |||||
goto failed; | |||||
} | |||||
txq = &sc->sge.txq[vi->first_txq]; | |||||
if (inp->inp_flowtype != M_HASHTYPE_NONE) | |||||
txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) + | |||||
vi->rsrv_noflowq); | |||||
tlsp->txq = txq; | |||||
error = ktls_set_tcb_fields(tlsp, tp, txq); | |||||
INP_RUNLOCK(inp); | |||||
if (error) | |||||
goto failed; | |||||
init_ktls_key_params(tlsp, tls); | |||||
error = ktls_setup_keys(tlsp, tls, txq); | |||||
if (error) | |||||
goto failed; | |||||
/* The SCMD fields used when encrypting a full TLS record. */ | |||||
tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) | | |||||
V_SCMD_PROTO_VERSION(tlsp->proto_ver) | | |||||
V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | | |||||
V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) | | |||||
V_SCMD_CIPH_MODE(tlsp->enc_mode) | | |||||
V_SCMD_AUTH_MODE(tlsp->auth_mode) | | |||||
V_SCMD_HMAC_CTRL(tlsp->hmac_ctrl) | | |||||
V_SCMD_IV_SIZE(tlsp->iv_size) | V_SCMD_NUM_IVS(1)); | |||||
tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | | |||||
V_SCMD_TLS_FRAG_ENABLE(0); | |||||
if (tlsp->inline_key) | |||||
tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); | |||||
tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen); | |||||
/* | |||||
* The SCMD fields used when encrypting a partial TLS record | |||||
* (no trailer and possibly a truncated payload). | |||||
*/ | |||||
tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | | |||||
V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | | |||||
V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | | |||||
V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) | | |||||
V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | | |||||
V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | | |||||
V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0); | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) | |||||
tlsp->scmd0_short.seqno_numivs |= | |||||
V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CTR); | |||||
else | |||||
tlsp->scmd0_short.seqno_numivs |= | |||||
V_SCMD_CIPH_MODE(tlsp->enc_mode); | |||||
tlsp->scmd0_short.seqno_numivs = | |||||
htobe32(tlsp->scmd0_short.seqno_numivs); | |||||
tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | | |||||
V_SCMD_TLS_FRAG_ENABLE(0) | | |||||
V_SCMD_AADIVDROP(1); | |||||
if (tlsp->inline_key) | |||||
tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); | |||||
TXQ_LOCK(txq); | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) | |||||
txq->kern_tls_gcm++; | |||||
else | |||||
txq->kern_tls_cbc++; | |||||
TXQ_UNLOCK(txq); | |||||
*pt = &tlsp->com.com; | |||||
return (0); | |||||
failed: | |||||
if (atid >= 0) | |||||
free_atid(sc, atid); | |||||
m_snd_tag_rele(&tlsp->com.com); | |||||
return (error); | |||||
} | |||||
/* XXX: Should share this with ccr(4) eventually. */ | |||||
static void | |||||
init_ktls_gmac_hash(const char *key, int klen, char *ghash) | |||||
{ | |||||
static char zeroes[GMAC_BLOCK_LEN]; | |||||
uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)]; | |||||
int rounds; | |||||
rounds = rijndaelKeySetupEnc(keysched, key, klen); | |||||
rijndaelEncrypt(keysched, rounds, zeroes, ghash); | |||||
} | |||||
/* XXX: Should share this with ccr(4) eventually. */ | |||||
static void | |||||
ktls_copy_partial_hash(void *dst, int cri_alg, union authctx *auth_ctx) | |||||
{ | |||||
uint32_t *u32; | |||||
uint64_t *u64; | |||||
u_int i; | |||||
u32 = (uint32_t *)dst; | |||||
u64 = (uint64_t *)dst; | |||||
switch (cri_alg) { | |||||
case CRYPTO_SHA1_HMAC: | |||||
for (i = 0; i < SHA1_HASH_LEN / 4; i++) | |||||
u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]); | |||||
break; | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
for (i = 0; i < SHA2_256_HASH_LEN / 4; i++) | |||||
u32[i] = htobe32(auth_ctx->sha256ctx.state[i]); | |||||
break; | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
for (i = 0; i < SHA2_512_HASH_LEN / 8; i++) | |||||
u64[i] = htobe64(auth_ctx->sha384ctx.state[i]); | |||||
break; | |||||
} | |||||
} | |||||
static void | |||||
init_ktls_hmac_digest(struct auth_hash *axf, u_int partial_digest_len, | |||||
char *key, int klen, char *dst) | |||||
{ | |||||
union authctx auth_ctx; | |||||
char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN]; | |||||
u_int i; | |||||
/* | |||||
* If the key is larger than the block size, use the digest of | |||||
* the key as the key instead. | |||||
*/ | |||||
klen /= 8; | |||||
if (klen > axf->blocksize) { | |||||
axf->Init(&auth_ctx); | |||||
axf->Update(&auth_ctx, key, klen); | |||||
axf->Final(ipad, &auth_ctx); | |||||
klen = axf->hashsize; | |||||
} else | |||||
memcpy(ipad, key, klen); | |||||
memset(ipad + klen, 0, axf->blocksize - klen); | |||||
memcpy(opad, ipad, axf->blocksize); | |||||
for (i = 0; i < axf->blocksize; i++) { | |||||
ipad[i] ^= HMAC_IPAD_VAL; | |||||
opad[i] ^= HMAC_OPAD_VAL; | |||||
} | |||||
/* | |||||
* Hash the raw ipad and opad and store the partial results in | |||||
* the key context. | |||||
*/ | |||||
axf->Init(&auth_ctx); | |||||
axf->Update(&auth_ctx, ipad, axf->blocksize); | |||||
ktls_copy_partial_hash(dst, axf->type, &auth_ctx); | |||||
dst += roundup2(partial_digest_len, 16); | |||||
axf->Init(&auth_ctx); | |||||
axf->Update(&auth_ctx, opad, axf->blocksize); | |||||
ktls_copy_partial_hash(dst, axf->type, &auth_ctx); | |||||
} | |||||
static int | |||||
ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls, | |||||
struct sge_txq *txq) | |||||
{ | |||||
struct auth_hash *axf; | |||||
int error, keyid, kwrlen, kctxlen, len; | |||||
struct tls_key_req *kwr; | |||||
struct tls_keyctx *kctx; | |||||
void *items[1], *key; | |||||
struct tx_keyctx_hdr *khdr; | |||||
unsigned int ck_size, mk_size, partial_digest_len; | |||||
struct mbuf *m; | |||||
/* | |||||
* Store the salt and keys in the key context. For | |||||
* connections with an inline key, this key context is passed | |||||
* as immediate data in each work request. For connections | |||||
* storing the key in DDR, a work request is used to store a | |||||
* copy of the key context in DDR. | |||||
*/ | |||||
kctx = &tlsp->keyctx; | |||||
khdr = &kctx->txhdr; | |||||
switch (tlsp->cipher_secret_size) { | |||||
case 128 / 8: | |||||
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; | |||||
break; | |||||
case 192 / 8: | |||||
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; | |||||
break; | |||||
case 256 / 8: | |||||
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; | |||||
break; | |||||
default: | |||||
panic("bad key size"); | |||||
} | |||||
axf = NULL; | |||||
partial_digest_len = 0; | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) | |||||
mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; | |||||
else { | |||||
switch (tlsp->auth_mode) { | |||||
case SCMD_AUTH_MODE_SHA1: | |||||
axf = &auth_hash_hmac_sha1; | |||||
mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; | |||||
partial_digest_len = SHA1_HASH_LEN; | |||||
break; | |||||
case SCMD_AUTH_MODE_SHA256: | |||||
axf = &auth_hash_hmac_sha2_256; | |||||
mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; | |||||
partial_digest_len = SHA2_256_HASH_LEN; | |||||
break; | |||||
case SCMD_AUTH_MODE_SHA512_384: | |||||
axf = &auth_hash_hmac_sha2_384; | |||||
mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; | |||||
partial_digest_len = SHA2_512_HASH_LEN; | |||||
break; | |||||
default: | |||||
panic("bad auth mode"); | |||||
} | |||||
} | |||||
khdr->ctxlen = (tlsp->tx_key_info_size >> 4); | |||||
khdr->dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | | |||||
V_TLS_KEYCTX_TX_WR_TXCK_SIZE(ck_size) | | |||||
V_TLS_KEYCTX_TX_WR_TXMK_SIZE(mk_size) | | |||||
V_TLS_KEYCTX_TX_WR_TXVALID(1); | |||||
if (tlsp->enc_mode != SCMD_CIPH_MODE_AES_GCM) | |||||
khdr->dualck_to_txvalid |= V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); | |||||
khdr->dualck_to_txvalid = htobe16(khdr->dualck_to_txvalid); | |||||
key = kctx->keys.edkey; | |||||
memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len); | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { | |||||
memcpy(khdr->txsalt, tls->params.iv, SALT_SIZE); | |||||
init_ktls_gmac_hash(tls->params.cipher_key, | |||||
tls->params.cipher_key_len * 8, | |||||
(char *)key + tls->params.cipher_key_len); | |||||
} else { | |||||
init_ktls_hmac_digest(axf, partial_digest_len, | |||||
tls->params.auth_key, tls->params.auth_key_len * 8, | |||||
(char *)key + tls->params.cipher_key_len); | |||||
} | |||||
if (tlsp->inline_key) | |||||
return (0); | |||||
keyid = tlsp->tx_key_addr; | |||||
/* Populate key work request. */ | |||||
kwrlen = sizeof(*kwr); | |||||
kctxlen = roundup2(sizeof(*kctx), 32); | |||||
len = kwrlen + kctxlen; | |||||
m = alloc_wr_mbuf(len, M_NOWAIT); | |||||
if (m == NULL) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__, | |||||
tlsp->tid); | |||||
return (ENOMEM); | |||||
} | |||||
m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com.com); | |||||
m->m_pkthdr.csum_flags |= CSUM_SND_TAG; | |||||
kwr = mtod(m, void *); | |||||
memset(kwr, 0, len); | |||||
kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | | |||||
F_FW_WR_ATOMIC); | |||||
kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16))); | |||||
kwr->protocol = tlsp->proto_ver; | |||||
kwr->mfs = htons(tlsp->frag_size); | |||||
kwr->reneg_to_write_rx = KEY_WRITE_TX; | |||||
/* master command */ | |||||
kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | | |||||
V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); | |||||
kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); | |||||
kwr->len16 = htobe32((tlsp->tid << 8) | | |||||
DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); | |||||
kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); | |||||
/* sub command */ | |||||
kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); | |||||
kwr->sc_len = htobe32(kctxlen); | |||||
kctx = (struct tls_keyctx *)(kwr + 1); | |||||
memcpy(kctx, &tlsp->keyctx, sizeof(*kctx)); | |||||
/* | |||||
* Place the key work request in the transmit queue. It | |||||
* should be sent to the NIC before any TLS packets using this | |||||
* session. | |||||
*/ | |||||
items[0] = m; | |||||
error = mp_ring_enqueue(txq->r, items, 1, 1); | |||||
if (error) | |||||
m_free(m); | |||||
else | |||||
CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid); | |||||
return (error); | |||||
} | |||||
static u_int | |||||
ktls_base_wr_size(struct tlspcb *tlsp) | |||||
{ | |||||
u_int wr_len; | |||||
wr_len = sizeof(struct fw_ulptx_wr); // 16 | |||||
wr_len += sizeof(struct ulp_txpkt); // 8 | |||||
wr_len += sizeof(struct ulptx_idata); // 8 | |||||
wr_len += sizeof(struct cpl_tx_sec_pdu);// 32 | |||||
if (tlsp->inline_key) | |||||
wr_len += tlsp->tx_key_info_size; | |||||
else { | |||||
wr_len += sizeof(struct ulptx_sc_memrd);// 8 | |||||
wr_len += sizeof(struct ulptx_idata); // 8 | |||||
} | |||||
wr_len += sizeof(struct cpl_tx_data); // 16 | |||||
return (wr_len); | |||||
} | |||||
/* How many bytes of TCP payload to send for a given TLS record. */ | |||||
static u_int | |||||
ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
struct tls_record_layer *hdr; | |||||
u_int plen, mlen; | |||||
MBUF_EXT_PGS_ASSERT(m_tls); | |||||
ext_pgs = m_tls->m_ext.ext_pgs; | |||||
hdr = (void *)ext_pgs->hdr; | |||||
plen = ntohs(hdr->tls_length); | |||||
/* | |||||
* What range of the TLS record is the mbuf requesting to be | |||||
* sent. | |||||
*/ | |||||
mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; | |||||
/* Always send complete records. */ | |||||
if (mlen == TLS_HEADER_LENGTH + plen) | |||||
return (mlen); | |||||
/* | |||||
* If the host stack has asked to send part of the trailer, | |||||
* trim the length to avoid sending any of the trailer. There | |||||
* is no way to send a partial trailer currently. | |||||
*/ | |||||
if (mlen > TLS_HEADER_LENGTH + plen - ext_pgs->trail_len) | |||||
mlen = TLS_HEADER_LENGTH + plen - ext_pgs->trail_len; | |||||
/* | |||||
* For AES-CBC adjust the ciphertext length for the block | |||||
* size. | |||||
*/ | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && | |||||
mlen > TLS_HEADER_LENGTH) { | |||||
mlen = TLS_HEADER_LENGTH + rounddown(mlen - TLS_HEADER_LENGTH, | |||||
AES_BLOCK_LEN); | |||||
} | |||||
#ifdef VERBOSE_TRACES | |||||
CTR4(KTR_CXGBE, "%s: tid %d short TLS record (%u vs %u)", | |||||
__func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen); | |||||
#endif | |||||
return (mlen); | |||||
} | |||||
/* | |||||
* For a "short" TLS record, determine the offset into the TLS record | |||||
* payload to send. This offset does not include the TLS header, but | |||||
* a non-zero offset implies that a header will not be sent. | |||||
*/ | |||||
static u_int | |||||
ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
struct tls_record_layer *hdr; | |||||
u_int offset, plen; | |||||
#ifdef INVARIANTS | |||||
u_int mlen; | |||||
#endif | |||||
MBUF_EXT_PGS_ASSERT(m_tls); | |||||
ext_pgs = m_tls->m_ext.ext_pgs; | |||||
hdr = (void *)ext_pgs->hdr; | |||||
plen = ntohs(hdr->tls_length); | |||||
#ifdef INVARIANTS | |||||
mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; | |||||
MPASS(mlen < TLS_HEADER_LENGTH + plen); | |||||
#endif | |||||
if (mtod(m_tls, vm_offset_t) <= ext_pgs->hdr_len) | |||||
return (0); | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { | |||||
/* | |||||
* Always send something. This function is only called | |||||
* if we aren't sending the tag at all, but if the | |||||
* request starts in the tag then we are in an odd | |||||
* state where would effectively send nothing. Cap | |||||
* the offset at the last byte of the record payload | |||||
* to send the last cipher block. | |||||
*/ | |||||
offset = min(mtod(m_tls, vm_offset_t) - ext_pgs->hdr_len, | |||||
(plen - TLS_HEADER_LENGTH - ext_pgs->trail_len) - 1); | |||||
return (rounddown(offset, AES_BLOCK_LEN)); | |||||
} | |||||
return (0); | |||||
} | |||||
static u_int | |||||
ktls_sgl_size(u_int nsegs) | |||||
{ | |||||
u_int wr_len; | |||||
/* First segment is part of ulptx_sgl. */ | |||||
nsegs--; | |||||
wr_len = sizeof(struct ulptx_sgl); | |||||
wr_len += 8 * ((3 * nsegs) / 2 + (nsegs & 1)); | |||||
return (wr_len); | |||||
} | |||||
static int | |||||
ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls, | |||||
int *nsegsp) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
struct tls_record_layer *hdr; | |||||
u_int imm_len, offset, plen, wr_len, tlen; | |||||
MBUF_EXT_PGS_ASSERT(m_tls); | |||||
ext_pgs = m_tls->m_ext.ext_pgs; | |||||
/* | |||||
* Determine the size of the TLS record payload to send | |||||
* excluding header and trailer. | |||||
*/ | |||||
tlen = ktls_tcp_payload_length(tlsp, m_tls); | |||||
if (tlen <= ext_pgs->hdr_len) { | |||||
/* | |||||
* For requests that only want to send the TLS header, | |||||
* send a tunnelled packet as immediate data. | |||||
*/ | |||||
wr_len = sizeof(struct fw_eth_tx_pkt_wr) + | |||||
sizeof(struct cpl_tx_pkt_core) + | |||||
roundup2(m->m_len + m_tls->m_len, 16); | |||||
if (wr_len > SGE_MAX_WR_LEN) { | |||||
CTR3(KTR_CXGBE, | |||||
"%s: tid %d TLS header-only packet too long (len %d)", | |||||
__func__, tlsp->tid, m->m_len + m_tls->m_len); | |||||
} | |||||
/* This should always be the last TLS record in a chain. */ | |||||
MPASS(m_tls->m_next == NULL); | |||||
/* | |||||
* XXX: Set a bogus 'nsegs' value to avoid tripping an | |||||
* assertion in mbuf_nsegs() in t4_sge.c. | |||||
*/ | |||||
*nsegsp = 1; | |||||
return (wr_len); | |||||
} | |||||
hdr = (void *)ext_pgs->hdr; | |||||
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len; | |||||
if (tlen < plen) { | |||||
plen = tlen; | |||||
offset = ktls_payload_offset(tlsp, m_tls); | |||||
} else | |||||
offset = 0; | |||||
/* Calculate the size of the work request. */ | |||||
wr_len = ktls_base_wr_size(tlsp); | |||||
/* | |||||
* Full records and short records with an offset of 0 include | |||||
* the TLS header as immediate data. Short records include a | |||||
* raw AES IV as immediate data. | |||||
*/ | |||||
imm_len = 0; | |||||
if (offset == 0) | |||||
imm_len += ext_pgs->hdr_len; | |||||
if (plen == tlen) | |||||
imm_len += AES_BLOCK_LEN; | |||||
wr_len += roundup2(imm_len, 16); | |||||
/* TLS record payload via DSGL. */ | |||||
*nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset, | |||||
plen - (ext_pgs->hdr_len + offset)); | |||||
wr_len += ktls_sgl_size(*nsegsp); | |||||
wr_len = roundup2(wr_len, 16); | |||||
return (wr_len); | |||||
} | |||||
/* | |||||
* See if we have any TCP options requiring a dedicated options-only | |||||
* packet. | |||||
*/ | |||||
static int | |||||
ktls_has_tcp_options(struct tcphdr *tcp) | |||||
{ | |||||
u_char *cp; | |||||
int cnt, opt, optlen; | |||||
cp = (u_char *)(tcp + 1); | |||||
cnt = tcp->th_off * 4 - sizeof(struct tcphdr); | |||||
for (; cnt > 0; cnt -= optlen, cp += optlen) { | |||||
opt = cp[0]; | |||||
if (opt == TCPOPT_EOL) | |||||
break; | |||||
if (opt == TCPOPT_NOP) | |||||
optlen = 1; | |||||
else { | |||||
if (cnt < 2) | |||||
break; | |||||
optlen = cp[1]; | |||||
if (optlen < 2 || optlen > cnt) | |||||
break; | |||||
} | |||||
switch (opt) { | |||||
case TCPOPT_NOP: | |||||
case TCPOPT_TIMESTAMP: | |||||
break; | |||||
default: | |||||
return (1); | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
/* | |||||
* Find the TCP timestamp option. | |||||
*/ | |||||
static void * | |||||
ktls_find_tcp_timestamps(struct tcphdr *tcp) | |||||
{ | |||||
u_char *cp; | |||||
int cnt, opt, optlen; | |||||
cp = (u_char *)(tcp + 1); | |||||
cnt = tcp->th_off * 4 - sizeof(struct tcphdr); | |||||
for (; cnt > 0; cnt -= optlen, cp += optlen) { | |||||
opt = cp[0]; | |||||
if (opt == TCPOPT_EOL) | |||||
break; | |||||
if (opt == TCPOPT_NOP) | |||||
optlen = 1; | |||||
else { | |||||
if (cnt < 2) | |||||
break; | |||||
optlen = cp[1]; | |||||
if (optlen < 2 || optlen > cnt) | |||||
break; | |||||
} | |||||
if (opt == TCPOPT_TIMESTAMP && optlen == TCPOLEN_TIMESTAMP) | |||||
return (cp + 2); | |||||
} | |||||
return (NULL); | |||||
} | |||||
int | |||||
t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p) | |||||
{ | |||||
struct tlspcb *tlsp; | |||||
struct ether_header *eh; | |||||
struct ip *ip; | |||||
struct ip6_hdr *ip6; | |||||
struct tcphdr *tcp; | |||||
struct mbuf *m_tls; | |||||
int nsegs; | |||||
u_int wr_len, tot_len; | |||||
/* | |||||
* Locate headers in initial mbuf. | |||||
* | |||||
* XXX: This assumes all of the headers are in the initial mbuf. | |||||
* Could perhaps use m_advance() like parse_pkt() if that turns | |||||
* out to not be true. | |||||
*/ | |||||
M_ASSERTPKTHDR(m); | |||||
MPASS(m->m_pkthdr.snd_tag != NULL); | |||||
tlsp = mst_to_tls(m->m_pkthdr.snd_tag); | |||||
if (m->m_len <= sizeof(*eh) + sizeof(*ip)) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short", __func__, | |||||
tlsp->tid); | |||||
return (EINVAL); | |||||
} | |||||
eh = mtod(m, struct ether_header *); | |||||
if (ntohs(eh->ether_type) != ETHERTYPE_IP && | |||||
ntohs(eh->ether_type) != ETHERTYPE_IPV6) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d mbuf not ETHERTYPE_IP{,V6}", | |||||
__func__, tlsp->tid); | |||||
return (EINVAL); | |||||
} | |||||
m->m_pkthdr.l2hlen = sizeof(*eh); | |||||
/* XXX: Reject unsupported IP options? */ | |||||
if (ntohs(eh->ether_type) == ETHERTYPE_IP) { | |||||
ip = (struct ip *)(eh + 1); | |||||
if (ip->ip_p != IPPROTO_TCP) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP", | |||||
__func__, tlsp->tid); | |||||
return (EINVAL); | |||||
} | |||||
m->m_pkthdr.l3hlen = ip->ip_hl * 4; | |||||
} else { | |||||
ip6 = (struct ip6_hdr *)(eh + 1); | |||||
if (ip6->ip6_nxt != IPPROTO_TCP) { | |||||
CTR3(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP (%u)", | |||||
__func__, tlsp->tid, ip6->ip6_nxt); | |||||
return (EINVAL); | |||||
} | |||||
m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); | |||||
} | |||||
if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + | |||||
sizeof(*tcp)) { | |||||
CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short (2)", | |||||
__func__, tlsp->tid); | |||||
return (EINVAL); | |||||
} | |||||
tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen); | |||||
m->m_pkthdr.l4hlen = tcp->th_off * 4; | |||||
/* Bail if there is TCP payload before the TLS record. */ | |||||
if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + | |||||
m->m_pkthdr.l4hlen) { | |||||
CTR6(KTR_CXGBE, | |||||
"%s: tid %d header mbuf bad length (%d + %d + %d != %d)", | |||||
__func__, tlsp->tid, m->m_pkthdr.l2hlen, | |||||
m->m_pkthdr.l3hlen, m->m_pkthdr.l4hlen, m->m_len); | |||||
return (EINVAL); | |||||
} | |||||
/* Assume all headers are in 'm' for now. */ | |||||
MPASS(m->m_next != NULL); | |||||
MPASS(m->m_next->m_flags & M_NOMAP); | |||||
tot_len = 0; | |||||
/* | |||||
* Each of the remaining mbufs in the chain should reference a | |||||
* TLS record. | |||||
*/ | |||||
*nsegsp = 0; | |||||
for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { | |||||
MPASS(m_tls->m_flags & M_NOMAP); | |||||
wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR4(KTR_CXGBE, "%s: tid %d wr_len %d nsegs %d", __func__, | |||||
tlsp->tid, wr_len, nsegs); | |||||
#endif | |||||
if (wr_len > SGE_MAX_WR_LEN || nsegs > TX_SGL_SEGS) | |||||
return (EFBIG); | |||||
tot_len += roundup2(wr_len, EQ_ESIZE); | |||||
/* | |||||
* Store 'nsegs' for the first TLS record in the | |||||
* header mbuf's metadata. | |||||
*/ | |||||
if (*nsegsp == 0) | |||||
*nsegsp = nsegs; | |||||
} | |||||
MPASS(tot_len != 0); | |||||
/* | |||||
* See if we have any TCP options or a FIN requiring a | |||||
* dedicated packet. | |||||
*/ | |||||
if ((tcp->th_flags & TH_FIN) != 0 || ktls_has_tcp_options(tcp)) { | |||||
wr_len = sizeof(struct fw_eth_tx_pkt_wr) + | |||||
sizeof(struct cpl_tx_pkt_core) + roundup2(m->m_len, 16); | |||||
if (wr_len > SGE_MAX_WR_LEN) { | |||||
CTR3(KTR_CXGBE, | |||||
"%s: tid %d options-only packet too long (len %d)", | |||||
__func__, tlsp->tid, m->m_len); | |||||
return (EINVAL); | |||||
} | |||||
tot_len += roundup2(wr_len, EQ_ESIZE); | |||||
} | |||||
/* Include room for a TP work request to program an L2T entry. */ | |||||
tot_len += EQ_ESIZE; | |||||
/* | |||||
* Include room for a ULPTX work request including up to 5 | |||||
* CPL_SET_TCB_FIELD commands before the first TLS work | |||||
* request. | |||||
*/ | |||||
wr_len = sizeof(struct fw_ulptx_wr) + | |||||
5 * roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
/* | |||||
* If timestamps are present, reserve 1 more command for | |||||
* setting the echoed timestamp. | |||||
*/ | |||||
if (tlsp->using_timestamps) | |||||
wr_len += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
tot_len += roundup2(wr_len, EQ_ESIZE); | |||||
*len16p = tot_len / 16; | |||||
#ifdef VERBOSE_TRACES | |||||
CTR4(KTR_CXGBE, "%s: tid %d len16 %d nsegs %d", __func__, | |||||
tlsp->tid, *len16p, *nsegsp); | |||||
#endif | |||||
return (0); | |||||
} | |||||
/* | |||||
* If the SGL ends on an address that is not 16 byte aligned, this function will | |||||
* add a 0 filled flit at the end. | |||||
*/ | |||||
static void | |||||
write_gl_to_buf(struct sglist *gl, caddr_t to) | |||||
{ | |||||
struct sglist_seg *seg; | |||||
__be64 *flitp; | |||||
struct ulptx_sgl *usgl; | |||||
int i, nflits, nsegs; | |||||
KASSERT(((uintptr_t)to & 0xf) == 0, | |||||
("%s: SGL must start at a 16 byte boundary: %p", __func__, to)); | |||||
nsegs = gl->sg_nseg; | |||||
MPASS(nsegs > 0); | |||||
nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; | |||||
flitp = (__be64 *)to; | |||||
seg = &gl->sg_segs[0]; | |||||
usgl = (void *)flitp; | |||||
usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | | |||||
V_ULPTX_NSGE(nsegs)); | |||||
usgl->len0 = htobe32(seg->ss_len); | |||||
usgl->addr0 = htobe64(seg->ss_paddr); | |||||
seg++; | |||||
for (i = 0; i < nsegs - 1; i++, seg++) { | |||||
usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); | |||||
usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); | |||||
} | |||||
if (i & 1) | |||||
usgl->sge[i / 2].len[1] = htobe32(0); | |||||
flitp += nflits; | |||||
if (nflits & 1) { | |||||
MPASS(((uintptr_t)flitp) & 0xf); | |||||
*flitp++ = 0; | |||||
} | |||||
MPASS((((uintptr_t)flitp) & 0xf) == 0); | |||||
} | |||||
static inline void | |||||
copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) | |||||
{ | |||||
MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); | |||||
MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); | |||||
if (__predict_true((uintptr_t)(*to) + len <= | |||||
(uintptr_t)&eq->desc[eq->sidx])) { | |||||
bcopy(from, *to, len); | |||||
(*to) += len; | |||||
if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx]) | |||||
(*to) = (caddr_t)eq->desc; | |||||
} else { | |||||
int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); | |||||
bcopy(from, *to, portion); | |||||
from += portion; | |||||
portion = len - portion; /* remaining */ | |||||
bcopy(from, (void *)eq->desc, portion); | |||||
(*to) = (caddr_t)eq->desc + portion; | |||||
} | |||||
} | |||||
static int | |||||
ktls_write_tcp_options(struct sge_txq *txq, void *dst, struct mbuf *m, | |||||
u_int available, u_int pidx) | |||||
{ | |||||
struct tx_sdesc *txsd; | |||||
struct fw_eth_tx_pkt_wr *wr; | |||||
struct cpl_tx_pkt_core *cpl; | |||||
uint32_t ctrl; | |||||
uint64_t ctrl1; | |||||
int len16, ndesc, pktlen; | |||||
struct ether_header *eh; | |||||
struct ip *ip, newip; | |||||
struct ip6_hdr *ip6, newip6; | |||||
struct tcphdr *tcp, newtcp; | |||||
caddr_t out; | |||||
TXQ_LOCK_ASSERT_OWNED(txq); | |||||
M_ASSERTPKTHDR(m); | |||||
wr = dst; | |||||
pktlen = m->m_len; | |||||
ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; | |||||
len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); | |||||
ndesc = howmany(len16, EQ_ESIZE / 16); | |||||
MPASS(ndesc <= available); | |||||
/* Firmware work request header */ | |||||
wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | | |||||
V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); | |||||
ctrl = V_FW_WR_LEN16(len16); | |||||
wr->equiq_to_len16 = htobe32(ctrl); | |||||
wr->r3 = 0; | |||||
cpl = (void *)(wr + 1); | |||||
/* Checksum offload */ | |||||
ctrl1 = 0; | |||||
txq->txcsum++; | |||||
/* CPL header */ | |||||
cpl->ctrl0 = txq->cpl_ctrl0; | |||||
cpl->pack = 0; | |||||
cpl->len = htobe16(pktlen); | |||||
cpl->ctrl1 = htobe64(ctrl1); | |||||
out = (void *)(cpl + 1); | |||||
/* Copy over Ethernet header. */ | |||||
eh = mtod(m, struct ether_header *); | |||||
copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); | |||||
/* Fixup length in IP header and copy out. */ | |||||
if (ntohs(eh->ether_type) == ETHERTYPE_IP) { | |||||
ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip = *ip; | |||||
newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); | |||||
if (m->m_pkthdr.l3hlen > sizeof(*ip)) | |||||
copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, | |||||
m->m_pkthdr.l3hlen - sizeof(*ip)); | |||||
} else { | |||||
ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip6 = *ip6; | |||||
newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); | |||||
MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); | |||||
} | |||||
/* Clear PUSH and FIN in the TCP header if present. */ | |||||
tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); | |||||
newtcp = *tcp; | |||||
newtcp.th_flags &= ~(TH_PUSH | TH_FIN); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); | |||||
/* Copy rest of packet. */ | |||||
copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, pktlen - | |||||
(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); | |||||
txq->imm_wrs++; | |||||
txq->txpkt_wrs++; | |||||
txq->kern_tls_options++; | |||||
txsd = &txq->sdesc[pidx]; | |||||
txsd->m = NULL; | |||||
txsd->desc_used = ndesc; | |||||
return (ndesc); | |||||
} | |||||
static int | |||||
ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m, | |||||
struct mbuf *m_tls, u_int available, tcp_seq tcp_seqno, u_int pidx) | |||||
{ | |||||
struct tx_sdesc *txsd; | |||||
struct fw_eth_tx_pkt_wr *wr; | |||||
struct cpl_tx_pkt_core *cpl; | |||||
uint32_t ctrl; | |||||
uint64_t ctrl1; | |||||
int len16, ndesc, pktlen; | |||||
struct ether_header *eh; | |||||
struct ip *ip, newip; | |||||
struct ip6_hdr *ip6, newip6; | |||||
struct tcphdr *tcp, newtcp; | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
caddr_t out; | |||||
TXQ_LOCK_ASSERT_OWNED(txq); | |||||
M_ASSERTPKTHDR(m); | |||||
/* Locate the template TLS header. */ | |||||
MBUF_EXT_PGS_ASSERT(m_tls); | |||||
ext_pgs = m_tls->m_ext.ext_pgs; | |||||
/* This should always be the last TLS record in a chain. */ | |||||
MPASS(m_tls->m_next == NULL); | |||||
wr = dst; | |||||
pktlen = m->m_len + m_tls->m_len; | |||||
ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; | |||||
len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); | |||||
ndesc = howmany(len16, EQ_ESIZE / 16); | |||||
MPASS(ndesc <= available); | |||||
/* Firmware work request header */ | |||||
wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | | |||||
V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); | |||||
ctrl = V_FW_WR_LEN16(len16); | |||||
wr->equiq_to_len16 = htobe32(ctrl); | |||||
wr->r3 = 0; | |||||
cpl = (void *)(wr + 1); | |||||
/* Checksum offload */ | |||||
ctrl1 = 0; | |||||
txq->txcsum++; | |||||
/* CPL header */ | |||||
cpl->ctrl0 = txq->cpl_ctrl0; | |||||
cpl->pack = 0; | |||||
cpl->len = htobe16(pktlen); | |||||
cpl->ctrl1 = htobe64(ctrl1); | |||||
out = (void *)(cpl + 1); | |||||
/* Copy over Ethernet header. */ | |||||
eh = mtod(m, struct ether_header *); | |||||
copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); | |||||
/* Fixup length in IP header and copy out. */ | |||||
if (ntohs(eh->ether_type) == ETHERTYPE_IP) { | |||||
ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip = *ip; | |||||
newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); | |||||
if (m->m_pkthdr.l3hlen > sizeof(*ip)) | |||||
copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, | |||||
m->m_pkthdr.l3hlen - sizeof(*ip)); | |||||
} else { | |||||
ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip6 = *ip6; | |||||
newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); | |||||
MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); | |||||
} | |||||
/* Set sequence number in TCP header. */ | |||||
tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); | |||||
newtcp = *tcp; | |||||
newtcp.th_seq = htonl(tcp_seqno + mtod(m_tls, vm_offset_t)); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); | |||||
/* Copy rest of TCP header. */ | |||||
copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - | |||||
(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); | |||||
/* Copy the subset of the TLS header requested. */ | |||||
copy_to_txd(&txq->eq, (char *)ext_pgs->hdr + mtod(m_tls, vm_offset_t), | |||||
&out, m_tls->m_len); | |||||
txq->imm_wrs++; | |||||
txq->txpkt_wrs++; | |||||
txq->kern_tls_header++; | |||||
txsd = &txq->sdesc[pidx]; | |||||
txsd->m = m; | |||||
txsd->desc_used = ndesc; | |||||
return (ndesc); | |||||
} | |||||
_Static_assert(sizeof(struct cpl_set_tcb_field) <= EQ_ESIZE, | |||||
"CPL_SET_TCB_FIELD must be smaller than a single TX descriptor"); | |||||
_Static_assert(W_TCB_SND_UNA_RAW == W_TCB_SND_NXT_RAW, | |||||
"SND_NXT_RAW and SND_UNA_RAW are in different words"); | |||||
static int | |||||
ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, | |||||
void *dst, struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls, | |||||
u_int nsegs, u_int available, tcp_seq tcp_seqno, uint32_t *tsopt, | |||||
u_int pidx, bool set_l2t_idx) | |||||
{ | |||||
struct sge_eq *eq = &txq->eq; | |||||
struct tx_sdesc *txsd; | |||||
struct fw_ulptx_wr *wr; | |||||
struct ulp_txpkt *txpkt; | |||||
struct ulptx_sc_memrd *memrd; | |||||
struct ulptx_idata *idata; | |||||
struct cpl_tx_sec_pdu *sec_pdu; | |||||
struct cpl_tx_data *tx_data; | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
struct tls_record_layer *hdr; | |||||
char *iv, *out; | |||||
u_int aad_start, aad_stop; | |||||
u_int auth_start, auth_stop, auth_insert; | |||||
u_int cipher_start, cipher_stop, iv_offset; | |||||
u_int imm_len, mss, ndesc, offset, plen, tlen, twr_len, wr_len; | |||||
u_int fields, tx_max_offset, tx_max; | |||||
bool first_wr, last_wr, using_scratch; | |||||
ndesc = 0; | |||||
MPASS(tlsp->txq == txq); | |||||
first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 && | |||||
tlsp->prev_win == 0); | |||||
/* | |||||
* Use the per-txq scratch pad if near the end of the ring to | |||||
* simplify handling of wrap-around. This uses a simple but | |||||
* not quite perfect test of using the scratch buffer if we | |||||
* can't fit a maximal work request in without wrapping. | |||||
*/ | |||||
using_scratch = (eq->sidx - pidx < SGE_MAX_WR_LEN / EQ_ESIZE); | |||||
/* Locate the TLS header. */ | |||||
MBUF_EXT_PGS_ASSERT(m_tls); | |||||
ext_pgs = m_tls->m_ext.ext_pgs; | |||||
hdr = (void *)ext_pgs->hdr; | |||||
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len; | |||||
/* Determine how much of the TLS record to send. */ | |||||
tlen = ktls_tcp_payload_length(tlsp, m_tls); | |||||
if (tlen <= ext_pgs->hdr_len) { | |||||
/* | |||||
* For requests that only want to send the TLS header, | |||||
* send a tunnelled packet as immediate data. | |||||
*/ | |||||
#ifdef VERBOSE_TRACES | |||||
CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u", | |||||
__func__, tlsp->tid, (u_int)ext_pgs->seqno); | |||||
#endif | |||||
return (ktls_write_tunnel_packet(txq, dst, m, m_tls, available, | |||||
tcp_seqno, pidx)); | |||||
} | |||||
if (tlen < plen) { | |||||
plen = tlen; | |||||
offset = ktls_payload_offset(tlsp, m_tls); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR4(KTR_CXGBE, "%s: tid %d short TLS record %u with offset %u", | |||||
__func__, tlsp->tid, (u_int)ext_pgs->seqno, offset); | |||||
#endif | |||||
if (m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) != 0) { | |||||
txq->kern_tls_fin_short++; | |||||
#ifdef INVARIANTS | |||||
panic("%s: FIN on short TLS record", __func__); | |||||
#endif | |||||
} | |||||
} else | |||||
offset = 0; | |||||
/* | |||||
* This is the last work request for a given TLS mbuf chain if | |||||
* it is the last mbuf in the chain and FIN is not set. If | |||||
* FIN is set, then ktls_write_tcp_fin() will write out the | |||||
* last work request. | |||||
*/ | |||||
last_wr = m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) == 0; | |||||
/* | |||||
* The host stack may ask us to not send part of the start of | |||||
* a TLS record. (For example, the stack might have | |||||
* previously sent a "short" TLS record and might later send | |||||
* down an mbuf that requests to send the remainder of the TLS | |||||
* record.) The crypto engine must process a TLS record from | |||||
* the beginning if computing a GCM tag or HMAC, so we always | |||||
* send the TLS record from the beginning as input to the | |||||
* crypto engine and via CPL_TX_DATA to TP. However, TP will | |||||
* drop individual packets after they have been chopped up | |||||
* into MSS-sized chunks if the entire sequence range of those | |||||
* packets is less than SND_UNA. SND_UNA is computed as | |||||
* TX_MAX - SND_UNA_RAW. Thus, use the offset stored in | |||||
* m_data to set TX_MAX to the first byte in the TCP sequence | |||||
* space the host actually wants us to send and set | |||||
* SND_UNA_RAW to 0. | |||||
* | |||||
* If the host sends us back to back requests that span the | |||||
* trailer of a single TLS record (first request ends "in" the | |||||
* trailer and second request starts at the next byte but | |||||
* still "in" the trailer), the initial bytes of the trailer | |||||
* that the first request drops will not be retransmitted. If | |||||
* the host uses the same requests when retransmitting the | |||||
* connection will hang. To handle this, always transmit the | |||||
* full trailer for a request that begins "in" the trailer | |||||
* (the second request in the example above). This should | |||||
* also help to avoid retransmits for the common case. | |||||
* | |||||
* A similar condition exists when using CBC for back to back | |||||
* requests that span a single AES block. The first request | |||||
* will be truncated to end at the end of the previous AES | |||||
* block. To handle this, always begin transmission at the | |||||
* start of the current AES block. | |||||
*/ | |||||
tx_max_offset = mtod(m_tls, vm_offset_t); | |||||
if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - | |||||
ext_pgs->trail_len) { | |||||
/* Always send the full trailer. */ | |||||
tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - | |||||
ext_pgs->trail_len; | |||||
} | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && | |||||
tx_max_offset > TLS_HEADER_LENGTH) { | |||||
/* Always send all of the first AES block. */ | |||||
tx_max_offset = TLS_HEADER_LENGTH + | |||||
rounddown(tx_max_offset - TLS_HEADER_LENGTH, | |||||
AES_BLOCK_LEN); | |||||
} | |||||
tx_max = tcp_seqno + tx_max_offset; | |||||
/* | |||||
* Update TCB fields. Reserve space for the FW_ULPTX_WR header | |||||
* but don't populate it until we know how many field updates | |||||
* are required. | |||||
*/ | |||||
if (using_scratch) | |||||
wr = (void *)txq->ss; | |||||
else | |||||
wr = dst; | |||||
out = (void *)(wr + 1); | |||||
fields = 0; | |||||
if (set_l2t_idx) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to set L2T_IX for subsequent TLS WR")); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR3(KTR_CXGBE, "%s: tid %d set L2T_IX to %d", __func__, | |||||
tlsp->tid, tlsp->l2te->idx); | |||||
#endif | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_L2T_IX, | |||||
V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx)); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
} | |||||
if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to set T_RTSEQ_RECENT for subsequent TLS WR")); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR2(KTR_CXGBE, "%s: tid %d wrote updated T_RTSEQ_RECENT", | |||||
__func__, tlsp->tid); | |||||
#endif | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_T_RTSEQ_RECENT, | |||||
V_TCB_T_RTSEQ_RECENT(M_TCB_T_RTSEQ_RECENT), | |||||
V_TCB_T_RTSEQ_RECENT(ntohl(tsopt[1]))); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
tlsp->prev_tsecr = ntohl(tsopt[1]); | |||||
} | |||||
if (first_wr || tlsp->prev_seq != tx_max) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to set TX_MAX for subsequent TLS WR")); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR4(KTR_CXGBE, | |||||
"%s: tid %d setting TX_MAX to %u (tcp_seqno %u)", | |||||
__func__, tlsp->tid, tx_max, tcp_seqno); | |||||
#endif | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_TX_MAX, | |||||
V_TCB_TX_MAX(M_TCB_TX_MAX), V_TCB_TX_MAX(tx_max)); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
} | |||||
/* | |||||
* If there is data to drop at the beginning of this TLS | |||||
* record or if this is a retransmit, | |||||
* reset SND_UNA_RAW to 0 so that SND_UNA == TX_MAX. | |||||
*/ | |||||
if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to clear SND_UNA_RAW for subsequent TLS WR")); | |||||
#ifdef VERBOSE_TRACES | |||||
CTR2(KTR_CXGBE, "%s: tid %d clearing SND_UNA_RAW", __func__, | |||||
tlsp->tid); | |||||
#endif | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_SND_UNA_RAW, | |||||
V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW), | |||||
V_TCB_SND_UNA_RAW(0)); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
} | |||||
/* | |||||
* Store the expected sequence number of the next byte after | |||||
* this record. | |||||
*/ | |||||
tlsp->prev_seq = tcp_seqno + tlen; | |||||
if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to set RCV_NXT for subsequent TLS WR")); | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_NXT, | |||||
V_TCB_RCV_NXT(M_TCB_RCV_NXT), | |||||
V_TCB_RCV_NXT(ntohl(tcp->th_ack))); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
tlsp->prev_ack = ntohl(tcp->th_ack); | |||||
} | |||||
if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) { | |||||
KASSERT(nsegs != 0, | |||||
("trying to set RCV_WND for subsequent TLS WR")); | |||||
write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_WND, | |||||
V_TCB_RCV_WND(M_TCB_RCV_WND), | |||||
V_TCB_RCV_WND(ntohs(tcp->th_win))); | |||||
out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
fields++; | |||||
tlsp->prev_win = ntohs(tcp->th_win); | |||||
} | |||||
/* Recalculate 'nsegs' if cached value is not available. */ | |||||
if (nsegs == 0) | |||||
nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + | |||||
offset, plen - (ext_pgs->hdr_len + offset)); | |||||
/* Calculate the size of the TLS work request. */ | |||||
twr_len = ktls_base_wr_size(tlsp); | |||||
imm_len = 0; | |||||
if (offset == 0) | |||||
imm_len += ext_pgs->hdr_len; | |||||
if (plen == tlen) | |||||
imm_len += AES_BLOCK_LEN; | |||||
twr_len += roundup2(imm_len, 16); | |||||
twr_len += ktls_sgl_size(nsegs); | |||||
/* | |||||
* If any field updates were required, determine if they can | |||||
* be included in the TLS work request. If not, use the | |||||
* FW_ULPTX_WR work request header at 'wr' as a dedicated work | |||||
* request for the field updates and start a new work request | |||||
* for the TLS work request afterward. | |||||
*/ | |||||
if (fields != 0) { | |||||
wr_len = fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16); | |||||
if (twr_len + wr_len <= SGE_MAX_WR_LEN && | |||||
tlsp->sc->tlst.combo_wrs) { | |||||
wr_len += twr_len; | |||||
txpkt = (void *)out; | |||||
} else { | |||||
wr_len += sizeof(*wr); | |||||
wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); | |||||
wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | | |||||
V_FW_WR_LEN16(wr_len / 16)); | |||||
wr->cookie = 0; | |||||
/* | |||||
* If we were using scratch space, copy the | |||||
* field updates work request to the ring. | |||||
*/ | |||||
if (using_scratch) { | |||||
out = dst; | |||||
copy_to_txd(eq, txq->ss, &out, wr_len); | |||||
} | |||||
ndesc = howmany(wr_len, EQ_ESIZE); | |||||
MPASS(ndesc <= available); | |||||
txq->raw_wrs++; | |||||
txsd = &txq->sdesc[pidx]; | |||||
txsd->m = NULL; | |||||
txsd->desc_used = ndesc; | |||||
IDXINCR(pidx, ndesc, eq->sidx); | |||||
dst = &eq->desc[pidx]; | |||||
/* | |||||
* Determine if we should use scratch space | |||||
* for the TLS work request based on the | |||||
* available space after advancing pidx for | |||||
* the field updates work request. | |||||
*/ | |||||
wr_len = twr_len; | |||||
using_scratch = (eq->sidx - pidx < | |||||
howmany(wr_len, EQ_ESIZE)); | |||||
if (using_scratch) | |||||
wr = (void *)txq->ss; | |||||
else | |||||
wr = dst; | |||||
txpkt = (void *)(wr + 1); | |||||
} | |||||
} else { | |||||
wr_len = twr_len; | |||||
txpkt = (void *)out; | |||||
} | |||||
wr_len = roundup2(wr_len, 16); | |||||
MPASS(ndesc + howmany(wr_len, EQ_ESIZE) <= available); | |||||
/* FW_ULPTX_WR */ | |||||
wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); | |||||
wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | | |||||
V_FW_WR_LEN16(wr_len / 16)); | |||||
wr->cookie = 0; | |||||
/* ULP_TXPKT */ | |||||
txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | | |||||
V_ULP_TXPKT_DATAMODIFY(0) | | |||||
V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | | |||||
V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); | |||||
txpkt->len = htobe32(howmany(twr_len - sizeof(*wr), 16)); | |||||
/* ULPTX_IDATA sub-command */ | |||||
idata = (void *)(txpkt + 1); | |||||
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | | |||||
V_ULP_TX_SC_MORE(1)); | |||||
idata->len = sizeof(struct cpl_tx_sec_pdu); | |||||
/* | |||||
* The key context, CPL_TX_DATA, and immediate data are part | |||||
* of this ULPTX_IDATA when using an inline key. When reading | |||||
* the key from memory, the CPL_TX_DATA and immediate data are | |||||
* part of a separate ULPTX_IDATA. | |||||
*/ | |||||
if (tlsp->inline_key) | |||||
idata->len += tlsp->tx_key_info_size + | |||||
sizeof(struct cpl_tx_data) + imm_len; | |||||
idata->len = htobe32(idata->len); | |||||
/* CPL_TX_SEC_PDU */ | |||||
sec_pdu = (void *)(idata + 1); | |||||
/* | |||||
* For short records, AAD is counted as header data in SCMD0, | |||||
* the IV is next followed by a cipher region for the payload. | |||||
*/ | |||||
if (plen == tlen) { | |||||
aad_start = 0; | |||||
aad_stop = 0; | |||||
iv_offset = 1; | |||||
auth_start = 0; | |||||
auth_stop = 0; | |||||
auth_insert = 0; | |||||
cipher_start = AES_BLOCK_LEN + 1; | |||||
cipher_stop = 0; | |||||
sec_pdu->pldlen = htobe32(16 + plen - | |||||
(ext_pgs->hdr_len + offset)); | |||||
/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ | |||||
sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs; | |||||
sec_pdu->ivgen_hdrlen = htobe32( | |||||
tlsp->scmd0_short.ivgen_hdrlen | | |||||
V_SCMD_HDR_LEN(offset == 0 ? ext_pgs->hdr_len : 0)); | |||||
txq->kern_tls_short++; | |||||
} else { | |||||
/* | |||||
* AAD is TLS header. IV is after AAD. The cipher region | |||||
* starts after the IV. See comments in ccr_authenc() and | |||||
* ccr_gmac() in t4_crypto.c regarding cipher and auth | |||||
* start/stop values. | |||||
*/ | |||||
aad_start = 1; | |||||
aad_stop = TLS_HEADER_LENGTH; | |||||
iv_offset = TLS_HEADER_LENGTH + 1; | |||||
cipher_start = ext_pgs->hdr_len + 1; | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { | |||||
cipher_stop = 0; | |||||
auth_start = cipher_start; | |||||
auth_stop = 0; | |||||
auth_insert = 0; | |||||
} else { | |||||
cipher_stop = 0; | |||||
auth_start = cipher_start; | |||||
auth_stop = 0; | |||||
auth_insert = 0; | |||||
} | |||||
sec_pdu->pldlen = htobe32(plen); | |||||
/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ | |||||
sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs; | |||||
sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen; | |||||
if (mtod(m_tls, vm_offset_t) == 0) | |||||
txq->kern_tls_full++; | |||||
else | |||||
txq->kern_tls_partial++; | |||||
} | |||||
sec_pdu->op_ivinsrtofst = htobe32( | |||||
V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | | |||||
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | | |||||
V_CPL_TX_SEC_PDU_IVINSRTOFST(iv_offset)); | |||||
sec_pdu->aadstart_cipherstop_hi = htobe32( | |||||
V_CPL_TX_SEC_PDU_AADSTART(aad_start) | | |||||
V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | | |||||
V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | | |||||
V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); | |||||
sec_pdu->cipherstop_lo_authinsert = htobe32( | |||||
V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | | |||||
V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) | | |||||
V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) | | |||||
V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); | |||||
sec_pdu->scmd1 = htobe64(ext_pgs->seqno); | |||||
/* Key context */ | |||||
out = (void *)(sec_pdu + 1); | |||||
if (tlsp->inline_key) { | |||||
memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size); | |||||
out += tlsp->tx_key_info_size; | |||||
} else { | |||||
/* ULPTX_SC_MEMRD to read key context. */ | |||||
memrd = (void *)out; | |||||
memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | | |||||
V_ULP_TX_SC_MORE(1) | | |||||
V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4)); | |||||
memrd->addr = htobe32(tlsp->tx_key_addr >> 5); | |||||
/* ULPTX_IDATA for CPL_TX_DATA and TLS header. */ | |||||
idata = (void *)(memrd + 1); | |||||
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | | |||||
V_ULP_TX_SC_MORE(1)); | |||||
idata->len = htobe32(sizeof(struct cpl_tx_data) + imm_len); | |||||
out = (void *)(idata + 1); | |||||
} | |||||
/* CPL_TX_DATA */ | |||||
tx_data = (void *)out; | |||||
OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid)); | |||||
if (m->m_pkthdr.csum_flags & CSUM_TSO) { | |||||
mss = m->m_pkthdr.tso_segsz; | |||||
tlsp->prev_mss = mss; | |||||
} else if (tlsp->prev_mss != 0) | |||||
mss = tlsp->prev_mss; | |||||
else | |||||
mss = tlsp->vi->ifp->if_mtu - | |||||
(m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen); | |||||
if (offset == 0) { | |||||
tx_data->len = htobe32(V_TX_DATA_MSS(mss) | V_TX_LENGTH(tlen)); | |||||
tx_data->rsvd = htobe32(tcp_seqno); | |||||
} else { | |||||
tx_data->len = htobe32(V_TX_DATA_MSS(mss) | | |||||
V_TX_LENGTH(tlen - (ext_pgs->hdr_len + offset))); | |||||
tx_data->rsvd = htobe32(tcp_seqno + ext_pgs->hdr_len + offset); | |||||
} | |||||
tx_data->flags = htobe32(F_TX_BYPASS); | |||||
if (last_wr && tcp->th_flags & TH_PUSH) | |||||
tx_data->flags |= htobe32(F_TX_PUSH | F_TX_SHOVE); | |||||
/* Populate the TLS header */ | |||||
out = (void *)(tx_data + 1); | |||||
if (offset == 0) { | |||||
memcpy(out, ext_pgs->hdr, ext_pgs->hdr_len); | |||||
out += ext_pgs->hdr_len; | |||||
} | |||||
/* AES IV for a short record. */ | |||||
if (plen == tlen) { | |||||
iv = out; | |||||
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { | |||||
memcpy(iv, tlsp->keyctx.txhdr.txsalt, SALT_SIZE); | |||||
memcpy(iv + 4, hdr + 1, 8); | |||||
*(uint32_t *)(iv + 12) = htobe32(2 + | |||||
offset / AES_BLOCK_LEN); | |||||
} else | |||||
memcpy(iv, hdr + 1, AES_BLOCK_LEN); | |||||
out += AES_BLOCK_LEN; | |||||
} | |||||
if (imm_len % 16 != 0) { | |||||
/* Zero pad to an 8-byte boundary. */ | |||||
memset(out, 0, 8 - (imm_len % 8)); | |||||
out += 8 - (imm_len % 8); | |||||
/* | |||||
* Insert a ULP_TX_SC_NOOP if needed so the SGL is | |||||
* 16-byte aligned. | |||||
*/ | |||||
if (imm_len % 16 <= 8) { | |||||
idata = (void *)out; | |||||
idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); | |||||
idata->len = htobe32(0); | |||||
out = (void *)(idata + 1); | |||||
} | |||||
} | |||||
/* SGL for record payload */ | |||||
sglist_reset(txq->gl); | |||||
if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset, | |||||
plen - (ext_pgs->hdr_len + offset)) != 0) { | |||||
#ifdef INVARIANTS | |||||
panic("%s: failed to append sglist", __func__); | |||||
#endif | |||||
} | |||||
write_gl_to_buf(txq->gl, out); | |||||
if (using_scratch) { | |||||
out = dst; | |||||
copy_to_txd(eq, txq->ss, &out, wr_len); | |||||
} | |||||
ndesc += howmany(wr_len, EQ_ESIZE); | |||||
MPASS(ndesc <= available); | |||||
txq->tls_wrs++; | |||||
txq->kern_tls_records++; | |||||
txq->kern_tls_octets += tlen - mtod(m_tls, vm_offset_t); | |||||
if (mtod(m_tls, vm_offset_t) != 0) { | |||||
if (offset == 0) | |||||
txq->kern_tls_waste += mtod(m_tls, vm_offset_t); | |||||
else | |||||
txq->kern_tls_waste += mtod(m_tls, vm_offset_t) - | |||||
(ext_pgs->hdr_len + offset); | |||||
} | |||||
txsd = &txq->sdesc[pidx]; | |||||
if (last_wr) | |||||
txsd->m = m; | |||||
else | |||||
txsd->m = NULL; | |||||
txsd->desc_used = howmany(wr_len, EQ_ESIZE); | |||||
return (ndesc); | |||||
} | |||||
static int | |||||
ktls_write_tcp_fin(struct sge_txq *txq, void *dst, struct mbuf *m, | |||||
u_int available, tcp_seq tcp_seqno, u_int pidx) | |||||
{ | |||||
struct tx_sdesc *txsd; | |||||
struct fw_eth_tx_pkt_wr *wr; | |||||
struct cpl_tx_pkt_core *cpl; | |||||
uint32_t ctrl; | |||||
uint64_t ctrl1; | |||||
int len16, ndesc, pktlen; | |||||
struct ether_header *eh; | |||||
struct ip *ip, newip; | |||||
struct ip6_hdr *ip6, newip6; | |||||
struct tcphdr *tcp, newtcp; | |||||
caddr_t out; | |||||
TXQ_LOCK_ASSERT_OWNED(txq); | |||||
M_ASSERTPKTHDR(m); | |||||
wr = dst; | |||||
pktlen = m->m_len; | |||||
ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; | |||||
len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); | |||||
ndesc = howmany(len16, EQ_ESIZE / 16); | |||||
MPASS(ndesc <= available); | |||||
/* Firmware work request header */ | |||||
wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | | |||||
V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); | |||||
ctrl = V_FW_WR_LEN16(len16); | |||||
wr->equiq_to_len16 = htobe32(ctrl); | |||||
wr->r3 = 0; | |||||
cpl = (void *)(wr + 1); | |||||
/* Checksum offload */ | |||||
ctrl1 = 0; | |||||
txq->txcsum++; | |||||
/* CPL header */ | |||||
cpl->ctrl0 = txq->cpl_ctrl0; | |||||
cpl->pack = 0; | |||||
cpl->len = htobe16(pktlen); | |||||
cpl->ctrl1 = htobe64(ctrl1); | |||||
out = (void *)(cpl + 1); | |||||
/* Copy over Ethernet header. */ | |||||
eh = mtod(m, struct ether_header *); | |||||
copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); | |||||
/* Fixup length in IP header and copy out. */ | |||||
if (ntohs(eh->ether_type) == ETHERTYPE_IP) { | |||||
ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip = *ip; | |||||
newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); | |||||
if (m->m_pkthdr.l3hlen > sizeof(*ip)) | |||||
copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, | |||||
m->m_pkthdr.l3hlen - sizeof(*ip)); | |||||
} else { | |||||
ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); | |||||
newip6 = *ip6; | |||||
newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); | |||||
MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); | |||||
} | |||||
/* Set sequence number in TCP header. */ | |||||
tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); | |||||
newtcp = *tcp; | |||||
newtcp.th_seq = htonl(tcp_seqno); | |||||
copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); | |||||
/* Copy rest of packet. */ | |||||
copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - | |||||
(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); | |||||
txq->imm_wrs++; | |||||
txq->txpkt_wrs++; | |||||
txq->kern_tls_fin++; | |||||
txsd = &txq->sdesc[pidx]; | |||||
txsd->m = m; | |||||
txsd->desc_used = ndesc; | |||||
return (ndesc); | |||||
} | |||||
int | |||||
t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, | |||||
u_int available) | |||||
{ | |||||
struct sge_eq *eq = &txq->eq; | |||||
struct tx_sdesc *txsd; | |||||
struct tlspcb *tlsp; | |||||
struct tcphdr *tcp; | |||||
struct mbuf *m_tls; | |||||
struct ether_header *eh; | |||||
tcp_seq tcp_seqno; | |||||
u_int ndesc, pidx, totdesc; | |||||
uint16_t vlan_tag; | |||||
bool has_fin, set_l2t_idx; | |||||
void *tsopt; | |||||
M_ASSERTPKTHDR(m); | |||||
MPASS(m->m_pkthdr.snd_tag != NULL); | |||||
tlsp = mst_to_tls(m->m_pkthdr.snd_tag); | |||||
totdesc = 0; | |||||
eh = mtod(m, struct ether_header *); | |||||
tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen + | |||||
m->m_pkthdr.l3hlen); | |||||
pidx = eq->pidx; | |||||
has_fin = (tcp->th_flags & TH_FIN) != 0; | |||||
/* | |||||
* If this TLS record has a FIN, then we will send any | |||||
* requested options as part of the FIN packet. | |||||
*/ | |||||
if (!has_fin && ktls_has_tcp_options(tcp)) { | |||||
ndesc = ktls_write_tcp_options(txq, dst, m, available, pidx); | |||||
totdesc += ndesc; | |||||
IDXINCR(pidx, ndesc, eq->sidx); | |||||
dst = &eq->desc[pidx]; | |||||
#ifdef VERBOSE_TRACES | |||||
CTR2(KTR_CXGBE, "%s: tid %d wrote TCP options packet", __func__, | |||||
tlsp->tid); | |||||
#endif | |||||
} | |||||
/* | |||||
* Allocate a new L2T entry if necessary. This may write out | |||||
* a work request to the txq. | |||||
*/ | |||||
if (m->m_flags & M_VLANTAG) | |||||
vlan_tag = m->m_pkthdr.ether_vtag; | |||||
else | |||||
vlan_tag = 0xfff; | |||||
set_l2t_idx = false; | |||||
if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag || | |||||
memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) { | |||||
set_l2t_idx = true; | |||||
if (tlsp->l2te) | |||||
t4_l2t_release(tlsp->l2te); | |||||
tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc, | |||||
vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost); | |||||
if (tlsp->l2te == NULL) | |||||
CXGBE_UNIMPLEMENTED("failed to allocate TLS L2TE"); | |||||
if (ndesc != 0) { | |||||
MPASS(ndesc <= available - totdesc); | |||||
txq->raw_wrs++; | |||||
txsd = &txq->sdesc[pidx]; | |||||
txsd->m = NULL; | |||||
txsd->desc_used = ndesc; | |||||
totdesc += ndesc; | |||||
IDXINCR(pidx, ndesc, eq->sidx); | |||||
dst = &eq->desc[pidx]; | |||||
} | |||||
} | |||||
/* | |||||
* Iterate over each TLS record constructing a work request | |||||
* for that record. | |||||
*/ | |||||
for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { | |||||
MPASS(m_tls->m_flags & M_NOMAP); | |||||
/* | |||||
* Determine the initial TCP sequence number for this | |||||
* record. | |||||
*/ | |||||
tsopt = NULL; | |||||
if (m_tls == m->m_next) { | |||||
tcp_seqno = ntohl(tcp->th_seq) - | |||||
mtod(m_tls, vm_offset_t); | |||||
if (tlsp->using_timestamps) | |||||
tsopt = ktls_find_tcp_timestamps(tcp); | |||||
} else { | |||||
MPASS(mtod(m_tls, vm_offset_t) == 0); | |||||
tcp_seqno = tlsp->prev_seq; | |||||
} | |||||
ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls, | |||||
nsegs, available - totdesc, tcp_seqno, tsopt, pidx, | |||||
set_l2t_idx); | |||||
totdesc += ndesc; | |||||
IDXINCR(pidx, ndesc, eq->sidx); | |||||
dst = &eq->desc[pidx]; | |||||
/* | |||||
* The value of nsegs from the header mbuf's metadata | |||||
* is only valid for the first TLS record. | |||||
*/ | |||||
nsegs = 0; | |||||
/* Only need to set the L2T index once. */ | |||||
set_l2t_idx = false; | |||||
} | |||||
if (has_fin) { | |||||
/* | |||||
* If the TCP header for this chain has FIN sent, then | |||||
* explicitly send a packet that has FIN set. This | |||||
* will also have PUSH set if requested. This assumes | |||||
* we sent at least one TLS record work request and | |||||
* uses the TCP sequence number after that reqeust as | |||||
* the sequence number for the FIN packet. | |||||
*/ | |||||
ndesc = ktls_write_tcp_fin(txq, dst, m, available, | |||||
tlsp->prev_seq, pidx); | |||||
totdesc += ndesc; | |||||
} | |||||
MPASS(totdesc <= available); | |||||
return (totdesc); | |||||
} | |||||
void | |||||
cxgbe_tls_tag_free(struct m_snd_tag *mst) | |||||
{ | |||||
struct adapter *sc; | |||||
struct tlspcb *tlsp; | |||||
tlsp = mst_to_tls(mst); | |||||
sc = tlsp->sc; | |||||
CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid); | |||||
if (tlsp->l2te) | |||||
t4_l2t_release(tlsp->l2te); | |||||
if (tlsp->tid >= 0) | |||||
release_tid(sc, tlsp->tid, tlsp->ctrlq); | |||||
if (tlsp->ce) | |||||
t4_release_lip(sc, tlsp->ce); | |||||
if (tlsp->tx_key_addr >= 0) | |||||
free_keyid(tlsp, tlsp->tx_key_addr); | |||||
explicit_bzero(&tlsp->keyctx, sizeof(&tlsp->keyctx)); | |||||
free(tlsp, M_CXGBE); | |||||
} | |||||
void | |||||
t6_ktls_modload(void) | |||||
{ | |||||
t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, ktls_act_open_rpl, | |||||
CPL_COOKIE_KERN_TLS); | |||||
} | |||||
void | |||||
t6_ktls_modunload(void) | |||||
{ | |||||
t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, | |||||
CPL_COOKIE_KERN_TLS); | |||||
} | |||||
#else | |||||
int | |||||
cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, | |||||
struct m_snd_tag **pt) | |||||
{ | |||||
return (ENXIO); | |||||
} | |||||
int | |||||
t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p) | |||||
{ | |||||
return (EINVAL); | |||||
} | |||||
int | |||||
t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, | |||||
u_int available) | |||||
{ | |||||
panic("can't happen"); | |||||
} | |||||
void | |||||
cxgbe_tls_tag_free(struct m_snd_tag *mst) | |||||
{ | |||||
panic("can't happen"); | |||||
} | |||||
void | |||||
t6_ktls_modload(void) | |||||
{ | |||||
} | |||||
void | |||||
t6_ktls_modunload(void) | |||||
{ | |||||
} | |||||
#endif |