Page MenuHomeFreeBSD
Files F161342562

D14529.diff
No OneTemporary
Actions

D14529.diff
View Options

Index: head/sys/dev/cxgbe/adapter.h
===================================================================
--- head/sys/dev/cxgbe/adapter.h
+++ head/sys/dev/cxgbe/adapter.h
@@ -297,6 +297,10 @@
struct port_stats stats;
u_int tnl_cong_drops;
u_int tx_parse_error;
+ u_long tx_tls_records;
+ u_long tx_tls_octets;
+ u_long rx_tls_records;
+ u_long rx_tls_octets;
struct callout tick;
};
Index: head/sys/dev/cxgbe/firmware/t6fw_cfg.txt
===================================================================
--- head/sys/dev/cxgbe/firmware/t6fw_cfg.txt
+++ head/sys/dev/cxgbe/firmware/t6fw_cfg.txt
@@ -163,10 +163,12 @@
nserver = 512
nhpfilter = 0
nhash = 16384
- protocol = ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu, iscsi_t10dif, crypto_lookaside
+ protocol = ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu, iscsi_t10dif, tlskeys, crypto_lookaside
tp_l2t = 4096
tp_ddp = 2
tp_ddp_iscsi = 2
+ tp_tls_key = 3
+ tp_tls_mxrxsize = 17408 # 16384 + 1024, governs max rx data, pm max xfer len, rx coalesce sizes
tp_stag = 2
tp_pbl = 5
tp_rq = 7
@@ -273,7 +275,7 @@
[fini]
version = 0x1
- checksum = 0x7191019f
+ checksum = 0x9e8952d2
#
# $FreeBSD$
#
Index: head/sys/dev/cxgbe/offload.h
===================================================================
--- head/sys/dev/cxgbe/offload.h
+++ head/sys/dev/cxgbe/offload.h
@@ -151,6 +151,9 @@
int sndbuf;
int ddp;
int rx_coalesce;
+ int tls;
+ int *tls_rx_ports;
+ int num_tls_rx_ports;
int tx_align;
int tx_zcopy;
};
Index: head/sys/dev/cxgbe/t4_main.c
===================================================================
--- head/sys/dev/cxgbe/t4_main.c
+++ head/sys/dev/cxgbe/t4_main.c
@@ -591,6 +591,7 @@
static int sysctl_tc_params(SYSCTL_HANDLER_ARGS);
#endif
#ifdef TCP_OFFLOAD
+static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
@@ -1390,6 +1391,7 @@
free(sc->sge.iqmap, M_CXGBE);
free(sc->sge.eqmap, M_CXGBE);
free(sc->tids.ftid_tab, M_CXGBE);
+ free(sc->tt.tls_rx_ports, M_CXGBE);
t4_destroy_dma_tag(sc);
if (mtx_initialized(&sc->sc_lock)) {
sx_xlock(&t4_list_lock);
@@ -5433,6 +5435,14 @@
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
+ sc->tt.tls = 0;
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW,
+ &sc->tt.tls, 0, "Inline TLS allowed");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports",
+ CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tls_rx_ports,
+ "I", "TCP ports that use inline TLS+TOE RX");
+
sc->tt.tx_align = 1;
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
@@ -5836,6 +5846,19 @@
"# of buffer-group 3 truncated packets");
#undef SYSCTL_ADD_T4_PORTSTAT
+
+ SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records",
+ CTLFLAG_RD, &pi->tx_tls_records,
+ "# of TLS records transmitted");
+ SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets",
+ CTLFLAG_RD, &pi->tx_tls_octets,
+ "# of payload octets in transmitted TLS records");
+ SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records",
+ CTLFLAG_RD, &pi->rx_tls_records,
+ "# of TLS records received");
+ SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets",
+ CTLFLAG_RD, &pi->rx_tls_octets,
+ "# of payload octets in received TLS records");
}
static int
@@ -8257,6 +8280,68 @@
#endif
#ifdef TCP_OFFLOAD
+static int
+sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ int *old_ports, *new_ports;
+ int i, new_count, rc;
+
+ if (req->newptr == NULL && req->oldptr == NULL)
+ return (SYSCTL_OUT(req, NULL, imax(sc->tt.num_tls_rx_ports, 1) *
+ sizeof(sc->tt.tls_rx_ports[0])));
+
+ rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tlsrx");
+ if (rc)
+ return (rc);
+
+ if (sc->tt.num_tls_rx_ports == 0) {
+ i = -1;
+ rc = SYSCTL_OUT(req, &i, sizeof(i));
+ } else
+ rc = SYSCTL_OUT(req, sc->tt.tls_rx_ports,
+ sc->tt.num_tls_rx_ports * sizeof(sc->tt.tls_rx_ports[0]));
+ if (rc == 0 && req->newptr != NULL) {
+ new_count = req->newlen / sizeof(new_ports[0]);
+ new_ports = malloc(new_count * sizeof(new_ports[0]), M_CXGBE,
+ M_WAITOK);
+ rc = SYSCTL_IN(req, new_ports, new_count *
+ sizeof(new_ports[0]));
+ if (rc)
+ goto err;
+
+ /* Allow setting to a single '-1' to clear the list. */
+ if (new_count == 1 && new_ports[0] == -1) {
+ ADAPTER_LOCK(sc);
+ old_ports = sc->tt.tls_rx_ports;
+ sc->tt.tls_rx_ports = NULL;
+ sc->tt.num_tls_rx_ports = 0;
+ ADAPTER_UNLOCK(sc);
+ free(old_ports, M_CXGBE);
+ } else {
+ for (i = 0; i < new_count; i++) {
+ if (new_ports[i] < 1 ||
+ new_ports[i] > IPPORT_MAX) {
+ rc = EINVAL;
+ goto err;
+ }
+ }
+
+ ADAPTER_LOCK(sc);
+ old_ports = sc->tt.tls_rx_ports;
+ sc->tt.tls_rx_ports = new_ports;
+ sc->tt.num_tls_rx_ports = new_count;
+ ADAPTER_UNLOCK(sc);
+ free(old_ports, M_CXGBE);
+ new_ports = NULL;
+ }
+ err:
+ free(new_ports, M_CXGBE);
+ }
+ end_synchronized_op(sc, 0);
+ return (rc);
+}
+
static void
unit_conv(char *buf, size_t len, u_int val, u_int factor)
{
Index: head/sys/dev/cxgbe/tom/t4_connect.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_connect.c
+++ head/sys/dev/cxgbe/tom/t4_connect.c
@@ -142,6 +142,10 @@
}
make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
+
+ if (toep->ulp_mode == ULP_MODE_TLS)
+ tls_establish(toep);
+
done:
INP_WUNLOCK(inp);
CURVNET_RESTORE();
@@ -268,6 +272,11 @@
if (toep->ulp_mode == ULP_MODE_TCPDDP)
opt2 |= F_RX_FC_VALID | F_RX_FC_DDP;
#endif
+ if (toep->ulp_mode == ULP_MODE_TLS) {
+ opt2 |= F_RX_FC_VALID;
+ opt2 &= ~V_RX_COALESCE(M_RX_COALESCE);
+ opt2 |= F_RX_FC_DISABLE;
+ }
return (htobe32(opt2));
}
@@ -378,10 +387,7 @@
DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
toep->vnet = so->so_vnet;
- if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0)
- set_tcpddp_ulp_mode(toep);
- else
- toep->ulp_mode = ULP_MODE_NONE;
+ set_ulp_mode(toep, select_ulp_mode(so, sc));
SOCKBUF_LOCK(&so->so_rcv);
/* opt0 rcv_bufsiz initially, assumes its normal meaning later */
toep->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ);
Index: head/sys/dev/cxgbe/tom/t4_cpl_io.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ head/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -73,9 +73,6 @@
#include "tom/t4_tom_l2t.h"
#include "tom/t4_tom.h"
-#define IS_AIOTX_MBUF(m) \
- ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX)
-
static void t4_aiotx_cancel(struct kaiocb *job);
static void t4_aiotx_queue_toep(struct toepcb *toep);
@@ -106,7 +103,7 @@
{
struct wrqe *wr;
struct fw_flowc_wr *flowc;
- unsigned int nparams = ftxp ? 8 : 6, flowclen;
+ unsigned int nparams, flowclen, paramidx;
struct vi_info *vi = toep->vi;
struct port_info *pi = vi->pi;
struct adapter *sc = pi->adapter;
@@ -116,6 +113,15 @@
KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
("%s: flowc for tid %u sent already", __func__, toep->tid));
+ if (ftxp != NULL)
+ nparams = 8;
+ else
+ nparams = 6;
+ if (toep->ulp_mode == ULP_MODE_TLS)
+ nparams++;
+ if (toep->tls.fcplenmax != 0)
+ nparams++;
+
flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
@@ -131,39 +137,45 @@
flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
V_FW_WR_FLOWID(toep->tid));
- flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
- flowc->mnemval[0].val = htobe32(pfvf);
- flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
- flowc->mnemval[1].val = htobe32(pi->tx_chan);
- flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
- flowc->mnemval[2].val = htobe32(pi->tx_chan);
- flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
- flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
+#define FLOWC_PARAM(__m, __v) \
+ do { \
+ flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
+ flowc->mnemval[paramidx].val = htobe32(__v); \
+ paramidx++; \
+ } while (0)
+
+ paramidx = 0;
+
+ FLOWC_PARAM(PFNVFN, pfvf);
+ FLOWC_PARAM(CH, pi->tx_chan);
+ FLOWC_PARAM(PORT, pi->tx_chan);
+ FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
if (ftxp) {
uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
- flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
- flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
- flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
- flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
- flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
- flowc->mnemval[6].val = htobe32(sndbuf);
- flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
- flowc->mnemval[7].val = htobe32(ftxp->mss);
+ FLOWC_PARAM(SNDNXT, ftxp->snd_nxt);
+ FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt);
+ FLOWC_PARAM(SNDBUF, sndbuf);
+ FLOWC_PARAM(MSS, ftxp->mss);
CTR6(KTR_CXGBE,
"%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
__func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt,
ftxp->rcv_nxt);
} else {
- flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
- flowc->mnemval[4].val = htobe32(512);
- flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
- flowc->mnemval[5].val = htobe32(512);
+ FLOWC_PARAM(SNDBUF, 512);
+ FLOWC_PARAM(MSS, 512);
CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
}
+ if (toep->ulp_mode == ULP_MODE_TLS)
+ FLOWC_PARAM(ULP_MODE, toep->ulp_mode);
+ if (toep->tls.fcplenmax != 0)
+ FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
+#undef FLOWC_PARAM
+ KASSERT(paramidx == nparams, ("nparams mismatch"));
+
txsd->tx_credits = howmany(flowclen, 16);
txsd->plen = 0;
KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
@@ -421,7 +433,7 @@
soisconnected(so);
}
-static int
+int
send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
{
struct wrqe *wr;
@@ -443,6 +455,23 @@
}
void
+send_rx_modulate(struct adapter *sc, struct toepcb *toep)
+{
+ struct wrqe *wr;
+ struct cpl_rx_data_ack *req;
+
+ wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
+ if (wr == NULL)
+ return;
+ req = wrtod(wr);
+
+ INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
+ req->credit_dack = htobe32(F_RX_MODULATE_RX);
+
+ t4_wrq_tx(sc, wr);
+}
+
+void
t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
{
struct adapter *sc = tod->tod_softc;
@@ -459,8 +488,18 @@
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
- toep->rx_credits += toep->sb_cc - sbused(sb);
+ credits = toep->sb_cc - sbused(sb);
toep->sb_cc = sbused(sb);
+ if (toep->ulp_mode == ULP_MODE_TLS) {
+ if (toep->tls.rcv_over >= credits) {
+ toep->tls.rcv_over -= credits;
+ credits = 0;
+ } else {
+ credits -= toep->tls.rcv_over;
+ toep->tls.rcv_over = 0;
+ }
+ }
+ toep->rx_credits += credits;
if (toep->rx_credits > 0 &&
(tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
@@ -471,7 +510,8 @@
toep->rx_credits -= credits;
tp->rcv_wnd += credits;
tp->rcv_adv += credits;
- }
+ } else if (toep->flags & TPF_FORCE_CREDITS)
+ send_rx_modulate(sc, toep);
}
void
@@ -489,8 +529,8 @@
/*
* Close a connection by sending a CPL_CLOSE_CON_REQ message.
*/
-static int
-close_conn(struct adapter *sc, struct toepcb *toep)
+int
+t4_close_conn(struct adapter *sc, struct toepcb *toep)
{
struct wrqe *wr;
struct cpl_close_con_req *req;
@@ -691,6 +731,7 @@
KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
toep->ulp_mode == ULP_MODE_TCPDDP ||
+ toep->ulp_mode == ULP_MODE_TLS ||
toep->ulp_mode == ULP_MODE_RDMA,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
@@ -905,7 +946,7 @@
/* Send a FIN if requested, but only if there's no more data to send */
if (m == NULL && toep->flags & TPF_SEND_FIN)
- close_conn(sc, toep);
+ t4_close_conn(sc, toep);
}
static inline void
@@ -1097,7 +1138,7 @@
/* Send a FIN if requested, but only if there are no more PDUs to send */
if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
- close_conn(sc, toep);
+ t4_close_conn(sc, toep);
}
int
@@ -1116,6 +1157,8 @@
if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, 0);
+ else if (tls_tx_key(toep))
+ t4_push_tls_records(sc, toep, 0);
else
t4_push_frames(sc, toep, 0);
@@ -1140,6 +1183,8 @@
if (tp->t_state >= TCPS_ESTABLISHED) {
if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, 0);
+ else if (tls_tx_key(toep))
+ t4_push_tls_records(sc, toep, 0);
else
t4_push_frames(sc, toep, 0);
}
@@ -1772,6 +1817,10 @@
credits -= txsd->tx_credits;
toep->tx_credits += txsd->tx_credits;
plen += txsd->plen;
+ if (txsd->iv_buffer) {
+ free(txsd->iv_buffer, M_CXGBE);
+ txsd->iv_buffer = NULL;
+ }
txsd++;
toep->txsd_avail++;
KASSERT(toep->txsd_avail <= toep->txsd_total,
@@ -1797,6 +1846,8 @@
CURVNET_SET(toep->vnet);
if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, plen);
+ else if (tls_tx_key(toep))
+ t4_push_tls_records(sc, toep, plen);
else
t4_push_frames(sc, toep, plen);
CURVNET_RESTORE();
@@ -1826,6 +1877,12 @@
tid, plen);
#endif
sbdrop_locked(sb, plen);
+ if (tls_tx_key(toep)) {
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ MPASS(tls_ofld->sb_off >= plen);
+ tls_ofld->sb_off -= plen;
+ }
if (!TAILQ_EMPTY(&toep->aiotx_jobq))
t4_aiotx_queue_toep(toep);
sowwakeup_locked(so); /* unlocks so_snd */
@@ -2298,6 +2355,9 @@
return (EOPNOTSUPP);
if (!sc->tt.tx_zcopy)
+ return (EOPNOTSUPP);
+
+ if (is_tls_offload(toep) || tls_tx_key(toep))
return (EOPNOTSUPP);
SOCKBUF_LOCK(&so->so_snd);
Index: head/sys/dev/cxgbe/tom/t4_listen.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_listen.c
+++ head/sys/dev/cxgbe/tom/t4_listen.c
@@ -1056,6 +1056,11 @@
if (ulp_mode == ULP_MODE_TCPDDP)
opt2 |= F_RX_FC_VALID | F_RX_FC_DDP;
#endif
+ if (ulp_mode == ULP_MODE_TLS) {
+ opt2 |= F_RX_FC_VALID;
+ opt2 &= ~V_RX_COALESCE(M_RX_COALESCE);
+ opt2 |= F_RX_FC_DISABLE;
+ }
return htobe32(opt2);
}
@@ -1347,11 +1352,15 @@
INIT_TP_WR_MIT_CPL(rpl5, CPL_PASS_ACCEPT_RPL, tid);
}
- if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0) {
- ulp_mode = ULP_MODE_TCPDDP;
+ ulp_mode = select_ulp_mode(so, sc);
+ switch (ulp_mode) {
+ case ULP_MODE_TCPDDP:
synqe->flags |= TPF_SYNQE_TCPDDP;
- } else
- ulp_mode = ULP_MODE_NONE;
+ break;
+ case ULP_MODE_TLS:
+ synqe->flags |= TPF_SYNQE_TLS;
+ break;
+ }
rpl->opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits, ulp_mode);
rpl->opt2 = calc_opt2p(sc, pi, rxqid, &cpl->tcpopt, &th, ulp_mode);
@@ -1407,8 +1416,8 @@
REJECT_PASS_ACCEPT();
}
- CTR5(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK",
- __func__, stid, tid, lctx, synqe);
+ CTR6(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK mode %d",
+ __func__, stid, tid, lctx, synqe, ulp_mode);
INP_WLOCK(inp);
synqe->flags |= TPF_SYNQE_HAS_L2TE;
@@ -1557,9 +1566,11 @@
toep->tid = tid;
toep->l2te = &sc->l2t->l2tab[synqe->l2e_idx];
if (synqe->flags & TPF_SYNQE_TCPDDP)
- set_tcpddp_ulp_mode(toep);
+ set_ulp_mode(toep, ULP_MODE_TCPDDP);
+ else if (synqe->flags & TPF_SYNQE_TLS)
+ set_ulp_mode(toep, ULP_MODE_TLS);
else
- toep->ulp_mode = ULP_MODE_NONE;
+ set_ulp_mode(toep, ULP_MODE_NONE);
/* opt0 rcv_bufsiz initially, assumes its normal meaning later */
toep->rx_credits = synqe->rcv_bufsize;
Index: head/sys/dev/cxgbe/tom/t4_tls.h
===================================================================
--- head/sys/dev/cxgbe/tom/t4_tls.h
+++ head/sys/dev/cxgbe/tom/t4_tls.h
@@ -0,0 +1,593 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2017-2018 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: John Baldwin <jhb@FreeBSD.org>, Atul Gupta
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef __T4_TLS_H__
+#define __T4_TLS_H__
+
+#define TLS1_VERSION 0x0301
+#define TLS1_1_VERSION 0x0302
+#define TLS1_2_VERSION 0x0303
+#define TLS_MAX_VERSION TLS1_2_VERSION
+
+#define DTLS1_VERSION 0xFEFF
+#define DTLS1_2_VERSION 0xFEFD
+#define DTLS_MAX_VERSION DTLS1_2_VERSION
+#define DTLS1_VERSION_MAJOR 0xFE
+
+/* Custom socket options for TLS+TOE. */
+
+#define MAX_MAC_KSZ 64 /*512 bits */
+#define MAX_CIPHER_KSZ 32 /* 256 bits */
+#define CIPHER_BLOCK_SZ 16
+#define SALT_SIZE 4
+
+/* Can accomodate 16, 11-15 are reserved */
+enum {
+ CHSSL_SHA_NOP,
+ CHSSL_SHA1,
+ CHSSL_SHA224,
+ CHSSL_SHA256,
+ CHSSL_GHASH,
+ CHSSL_SHA512_224,
+ CHSSL_SHA512_256,
+ CHSSL_SHA512_384,
+ CHSSL_SHA512_512,
+ CHSSL_CBCMAC,
+ CHSSL_CMAC,
+};
+
+/* Can accomodate 16, 8-15 are reserved */
+enum {
+ CHSSL_CIPH_NOP,
+ CHSSL_AES_CBC,
+ CHSSL_AES_GCM,
+ CHSSL_AES_CTR,
+ CHSSL_AES_GEN,
+ CHSSL_IPSEC_ESP,
+ CHSSL_AES_XTS,
+ CHSSL_AES_CCM,
+};
+
+/* Key Context Programming Operation type */
+#define KEY_WRITE_RX 0x1
+#define KEY_WRITE_TX 0x2
+#define KEY_DELETE_RX 0x4
+#define KEY_DELETE_TX 0x8
+
+#define S_KEY_CLR_LOC 4
+#define M_KEY_CLR_LOC 0xf
+#define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC)
+#define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC)
+#define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U)
+
+#define S_KEY_GET_LOC 0
+#define M_KEY_GET_LOC 0xf
+#define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC)
+#define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC)
+
+struct tls_ofld_state {
+ unsigned char enc_mode;
+ unsigned char mac_mode;
+ unsigned char key_loc;
+ unsigned char ofld_mode;
+ unsigned char auth_mode;
+ unsigned char resv[3];
+};
+
+struct tls_tx_ctxt {
+ unsigned char salt[SALT_SIZE];
+ unsigned char key[MAX_CIPHER_KSZ];
+ unsigned char ipad[MAX_MAC_KSZ];
+ unsigned char opad[MAX_MAC_KSZ];
+};
+
+struct tls_rx_ctxt {
+ unsigned char salt[SALT_SIZE];
+ unsigned char key[MAX_CIPHER_KSZ];
+ unsigned char ipad[MAX_MAC_KSZ];
+ unsigned char opad[MAX_MAC_KSZ];
+};
+
+struct tls_key_context {
+ struct tls_tx_ctxt tx;
+ struct tls_rx_ctxt rx;
+
+ unsigned char l_p_key;
+ unsigned char hmac_ctrl;
+ unsigned char mac_first;
+ unsigned char iv_size;
+ unsigned char iv_ctrl;
+ unsigned char iv_algo;
+ unsigned char tx_seq_no;
+ unsigned char rx_seq_no;
+
+ struct tls_ofld_state state;
+
+ unsigned int tx_key_info_size;
+ unsigned int rx_key_info_size;
+ unsigned int frag_size;
+ unsigned int mac_secret_size;
+ unsigned int cipher_secret_size;
+ int proto_ver;
+ unsigned int sock_fd;
+ unsigned short dtls_epoch;
+ unsigned short rsv;
+};
+
+/* Set with 'struct tls_key_context'. */
+#define TCP_TLSOM_SET_TLS_CONTEXT (TCP_VENDOR)
+
+/* Get returns int of enabled (1) / disabled (0). */
+#define TCP_TLSOM_GET_TLS_TOM (TCP_VENDOR + 1)
+
+enum {
+ TLS_TOM_NONE = 0,
+ TLS_TOM_TXONLY,
+ TLS_TOM_BOTH
+};
+
+/* Set with no value. */
+#define TCP_TLSOM_CLR_TLS_TOM (TCP_VENDOR + 2)
+
+/* Set with no value. */
+#define TCP_TLSOM_CLR_QUIES (TCP_VENDOR + 3)
+
+#ifdef _KERNEL
+/* Timeouts for handshake timer in seconds. */
+#define TLS_SRV_HELLO_DONE 9
+#define TLS_SRV_HELLO_RD_TM 5
+#define TLS_SRV_HELLO_BKOFF_TM 15
+
+#define CONTENT_TYPE_CCS 20
+#define CONTENT_TYPE_ALERT 21
+#define CONTENT_TYPE_HANDSHAKE 22
+#define CONTENT_TYPE_APP_DATA 23
+#define CONTENT_TYPE_HEARTBEAT 24
+#define CONTENT_TYPE_KEY_CONTEXT 32
+#define CONTENT_TYPE_ERROR 127
+
+#define GCM_TAG_SIZE 16
+#define AEAD_EXPLICIT_DATA_SIZE 8
+#define TLS_HEADER_LENGTH 5
+#define TP_TX_PG_SZ 65536
+#define FC_TP_PLEN_MAX 17408
+
+#define IPAD_SIZE 64
+#define OPAD_SIZE 64
+#define KEY_SIZE 32
+#define CIPHER_BLOCK_SIZE 16
+#define HDR_KCTX_SIZE (IPAD_SIZE + OPAD_SIZE + KEY_SIZE)
+
+#define KEY_IN_DDR_SIZE 16
+#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_tx_ctxt), 32)
+
+/* MAC KEY SIZE */
+#define SHA_NOP 0
+#define SHA_GHASH 16
+#define SHA_224 28
+#define SHA_256 32
+#define SHA_384 48
+#define SHA_512 64
+#define SHA1 20
+
+/* CIPHER KEY SIZE */
+#define AES_NOP 0
+#define AES_128 16
+#define AES_192 24
+#define AES_256 32
+
+enum {
+ TLS_1_2_VERSION,
+ TLS_1_1_VERSION,
+ DTLS_1_2_VERSION,
+ TLS_VERSION_MAX,
+};
+
+enum {
+ CH_EVP_CIPH_STREAM_CIPHER,
+ CH_EVP_CIPH_CBC_MODE,
+ CH_EVP_CIPH_GCM_MODE,
+ CH_EVP_CIPH_CTR_MODE,
+};
+
+enum {
+ TLS_SFO_WR_CONTEXTLOC_DSGL,
+ TLS_SFO_WR_CONTEXTLOC_IMMEDIATE,
+ TLS_SFO_WR_CONTEXTLOC_DDR,
+};
+
+enum {
+ CPL_TX_TLS_SFO_TYPE_CCS,
+ CPL_TX_TLS_SFO_TYPE_ALERT,
+ CPL_TX_TLS_SFO_TYPE_HANDSHAKE,
+ CPL_TX_TLS_SFO_TYPE_DATA,
+ CPL_TX_TLS_SFO_TYPE_HEARTBEAT, /* XXX: Shouldn't this be "CUSTOM"? */
+};
+
+enum {
+ CH_CK_SIZE_128,
+ CH_CK_SIZE_192,
+ CH_CK_SIZE_256,
+ CH_CK_SIZE_NOP,
+};
+
+enum {
+ CH_MK_SIZE_128,
+ CH_MK_SIZE_160,
+ CH_MK_SIZE_192,
+ CH_MK_SIZE_256,
+ CH_MK_SIZE_512,
+ CH_MK_SIZE_NOP,
+};
+
+#define SCMD_ENCDECCTRL_ENCRYPT 0
+#define SCMD_ENCDECCTRL_DECRYPT 1
+
+#define SCMD_CIPH_MODE_NOP 0
+#define SCMD_CIPH_MODE_AES_CBC 1
+#define SCMD_CIPH_MODE_AES_GCM 2
+#define SCMD_CIPH_MODE_AES_CTR 3
+#define SCMD_CIPH_MODE_AES_GEN 4
+#define SCMD_CIPH_MODE_AES_CCM 7
+
+struct tls_scmd {
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+};
+
+struct tls_ofld_info {
+ struct tls_key_context k_ctx;
+ int key_location;
+ int mac_length;
+ int rx_key_addr;
+ int tx_key_addr;
+ uint64_t tx_seq_no;
+ unsigned short fcplenmax;
+ unsigned short adjusted_plen;
+ unsigned short expn_per_ulp;
+ unsigned short pdus_per_ulp;
+ struct tls_scmd scmd0;
+ u_int sb_off;
+ struct callout handshake_timer;
+ u_int rcv_over;
+};
+
+struct tls_key_req {
+ __be32 wr_hi;
+ __be32 wr_mid;
+ __be32 ftid;
+ __u8 reneg_to_write_rx;
+ __u8 protocol;
+ __be16 mfs;
+ /* master command */
+ __be32 cmd;
+ __be32 len16; /* command length */
+ __be32 dlen; /* data length in 32-byte units */
+ __be32 kaddr;
+ /* sub-command */
+ __be32 sc_more;
+ __be32 sc_len;
+}__packed;
+
+struct tls_keyctx {
+ union key_ctx {
+ struct tx_keyctx_hdr {
+ __u8 ctxlen;
+ __u8 r2;
+ __be16 dualck_to_txvalid;
+ __u8 txsalt[4];
+ __be64 r5;
+ } txhdr;
+ struct rx_keyctx_hdr {
+ __u8 flitcnt_hmacctrl;
+ __u8 protover_ciphmode;
+ __u8 authmode_to_rxvalid;
+ __u8 ivpresent_to_rxmk_size;
+ __u8 rxsalt[4];
+ __be64 ivinsert_to_authinsrt;
+ } rxhdr;
+ } u;
+ struct keys {
+ __u8 edkey[32];
+ __u8 ipad[64];
+ __u8 opad[64];
+ } keys;
+};
+
+#define S_TLS_KEYCTX_TX_WR_DUALCK 12
+#define M_TLS_KEYCTX_TX_WR_DUALCK 0x1
+#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK)
+#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK)
+#define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U)
+
+#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11
+#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \
+ M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \
+ V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10
+#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \
+ M_TLS_KEYCTX_TX_WR_SALT_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \
+ V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6
+#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf
+#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \
+ M_TLS_KEYCTX_TX_WR_TXCK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2
+#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf
+#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \
+ M_TLS_KEYCTX_TX_WR_TXMK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_TXVALID 0
+#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1
+#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_TXVALID)
+#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID)
+#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U)
+
+#define S_TLS_KEYCTX_TX_WR_FLITCNT 3
+#define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f
+#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT)
+#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT)
+
+#define S_TLS_KEYCTX_TX_WR_HMACCTRL 0
+#define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7
+#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL)
+#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL)
+
+#define S_TLS_KEYCTX_TX_WR_PROTOVER 4
+#define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf
+#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER)
+#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHMODE 0
+#define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf
+#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE)
+#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHMODE 4
+#define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf
+#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE)
+#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3
+#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1
+#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
+#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \
+ M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL)
+#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \
+ V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U)
+
+#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1
+#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3
+#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
+#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \
+ M_TLS_KEYCTX_TX_WR_SEQNUMCTRL)
+
+#define S_TLS_KEYCTX_TX_WR_RXVALID 0
+#define M_TLS_KEYCTX_TX_WR_RXVALID 0x1
+#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_RXVALID)
+#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID)
+#define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U)
+
+#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7
+#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT)
+#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \
+ M_TLS_KEYCTX_TX_WR_IVPRESENT)
+#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6
+#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1
+#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
+#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \
+ M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT)
+#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \
+ V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U)
+
+#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3
+#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7
+#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \
+ M_TLS_KEYCTX_TX_WR_RXCK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0
+#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7
+#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE)
+#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \
+ M_TLS_KEYCTX_TX_WR_RXMK_SIZE)
+
+#define S_TLS_KEYCTX_TX_WR_IVINSERT 55
+#define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL
+#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT)
+#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT)
+
+#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47
+#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL
+#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST)
+#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \
+ M_TLS_KEYCTX_TX_WR_AADSTRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39
+#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL
+#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \
+ M_TLS_KEYCTX_TX_WR_AADSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30
+#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL
+#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
+#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \
+ M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23
+#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f
+#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \
+ M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14
+#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff
+#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
+#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \
+ M_TLS_KEYCTX_TX_WR_AUTHSRTOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7
+#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f
+#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
+#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \
+ M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST)
+
+#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0
+#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f
+#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
+ ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT)
+#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \
+ (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \
+ M_TLS_KEYCTX_TX_WR_AUTHINSRT)
+
+struct tls_hdr {
+ __u8 type;
+ __be16 version;
+ __be16 length;
+} __packed;
+
+struct tlsrx_hdr_pkt {
+ __u8 type;
+ __be16 version;
+ __be16 length;
+
+ __be64 tls_seq;
+ __be16 reserved1;
+ __u8 res_to_mac_error;
+} __packed;
+
+/* res_to_mac_error fields */
+#define S_TLSRX_HDR_PKT_INTERNAL_ERROR 4
+#define M_TLSRX_HDR_PKT_INTERNAL_ERROR 0x1
+#define V_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \
+ ((x) << S_TLSRX_HDR_PKT_INTERNAL_ERROR)
+#define G_TLSRX_HDR_PKT_INTERNAL_ERROR(x) \
+(((x) >> S_TLSRX_HDR_PKT_INTERNAL_ERROR) & M_TLSRX_HDR_PKT_INTERNAL_ERROR)
+#define F_TLSRX_HDR_PKT_INTERNAL_ERROR V_TLSRX_HDR_PKT_INTERNAL_ERROR(1U)
+
+#define S_TLSRX_HDR_PKT_SPP_ERROR 3
+#define M_TLSRX_HDR_PKT_SPP_ERROR 0x1
+#define V_TLSRX_HDR_PKT_SPP_ERROR(x) ((x) << S_TLSRX_HDR_PKT_SPP_ERROR)
+#define G_TLSRX_HDR_PKT_SPP_ERROR(x) \
+(((x) >> S_TLSRX_HDR_PKT_SPP_ERROR) & M_TLSRX_HDR_PKT_SPP_ERROR)
+#define F_TLSRX_HDR_PKT_SPP_ERROR V_TLSRX_HDR_PKT_SPP_ERROR(1U)
+
+#define S_TLSRX_HDR_PKT_CCDX_ERROR 2
+#define M_TLSRX_HDR_PKT_CCDX_ERROR 0x1
+#define V_TLSRX_HDR_PKT_CCDX_ERROR(x) ((x) << S_TLSRX_HDR_PKT_CCDX_ERROR)
+#define G_TLSRX_HDR_PKT_CCDX_ERROR(x) \
+(((x) >> S_TLSRX_HDR_PKT_CCDX_ERROR) & M_TLSRX_HDR_PKT_CCDX_ERROR)
+#define F_TLSRX_HDR_PKT_CCDX_ERROR V_TLSRX_HDR_PKT_CCDX_ERROR(1U)
+
+#define S_TLSRX_HDR_PKT_PAD_ERROR 1
+#define M_TLSRX_HDR_PKT_PAD_ERROR 0x1
+#define V_TLSRX_HDR_PKT_PAD_ERROR(x) ((x) << S_TLSRX_HDR_PKT_PAD_ERROR)
+#define G_TLSRX_HDR_PKT_PAD_ERROR(x) \
+(((x) >> S_TLSRX_HDR_PKT_PAD_ERROR) & M_TLSRX_HDR_PKT_PAD_ERROR)
+#define F_TLSRX_HDR_PKT_PAD_ERROR V_TLSRX_HDR_PKT_PAD_ERROR(1U)
+
+#define S_TLSRX_HDR_PKT_MAC_ERROR 0
+#define M_TLSRX_HDR_PKT_MAC_ERROR 0x1
+#define V_TLSRX_HDR_PKT_MAC_ERROR(x) ((x) << S_TLSRX_HDR_PKT_MAC_ERROR)
+#define G_TLSRX_HDR_PKT_MAC_ERROR(x) \
+(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR) & M_TLSRX_HDR_PKT_MAC_ERROR)
+#define F_TLSRX_HDR_PKT_MAC_ERROR V_TLSRX_HDR_PKT_MAC_ERROR(1U)
+
+#define M_TLSRX_HDR_PKT_ERROR 0x1F
+
+#endif /* _KERNEL */
+
+#endif /* !__T4_TLS_H__ */
Index: head/sys/dev/cxgbe/tom/t4_tls.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_tls.c
+++ head/sys/dev/cxgbe/tom/t4_tls.c
@@ -0,0 +1,1642 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2017-2018 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_inet.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/sglist.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/systm.h>
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp_var.h>
+#include <netinet/toecore.h>
+
+#ifdef TCP_OFFLOAD
+#include "common/common.h"
+#include "common/t4_tcb.h"
+#include "tom/t4_tom_l2t.h"
+#include "tom/t4_tom.h"
+
+/*
+ * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while
+ * the mbuf is in the ulp_pdu_reclaimq.
+ */
+#define tls_tcp_seq PH_loc.thirtytwo[0]
+
+/*
+ * Handshake lock used for the handshake timer. Having a global lock
+ * is perhaps not ideal, but it avoids having to use callout_drain()
+ * in tls_uninit_toep() which can't block. Also, the timer shouldn't
+ * actually fire for most connections.
+ */
+static struct mtx tls_handshake_lock;
+
+static void
+t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
+ uint64_t val)
+{
+ struct adapter *sc = td_adapter(toep->td);
+
+ t4_set_tcb_field(sc, toep->ctrlq, toep->tid, word, mask, val, 0, 0,
+ toep->ofld_rxq->iq.abs_id);
+}
+
+/* TLS and DTLS common routines */
+int
+tls_tx_key(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ return (tls_ofld->tx_key_addr >= 0);
+}
+
+int
+tls_rx_key(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ return (tls_ofld->rx_key_addr >= 0);
+}
+
+static int
+key_size(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ?
+ tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE);
+}
+
+/* Set TLS Key-Id in TCB */
+static void
+t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id)
+{
+
+ t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG,
+ V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG),
+ V_TCB_RX_TLS_KEY_TAG(key_id));
+}
+
+/* Clear TF_RX_QUIESCE to re-enable receive. */
+static void
+t4_clear_rx_quiesce(struct toepcb *toep)
+{
+
+ t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
+}
+
+static void
+tls_clr_ofld_mode(struct toepcb *toep)
+{
+
+ tls_stop_handshake_timer(toep);
+
+ /* Operate in PDU extraction mode only. */
+ t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
+ V_TCB_ULP_RAW(M_TCB_ULP_RAW),
+ V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
+ t4_clear_rx_quiesce(toep);
+}
+
+static void
+tls_clr_quiesce(struct toepcb *toep)
+{
+
+ tls_stop_handshake_timer(toep);
+ t4_clear_rx_quiesce(toep);
+}
+
+/*
+ * Calculate the TLS data expansion size
+ */
+static int
+tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only,
+ unsigned short *pdus_per_ulp)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct tls_scmd *scmd = &tls_ofld->scmd0;
+ int expn_size = 0, frag_count = 0, pad_per_pdu = 0,
+ pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0;
+ int exp_per_pdu = 0;
+ int hdr_len = TLS_HEADER_LENGTH;
+
+ do {
+ max_frag_size = tls_ofld->k_ctx.frag_size;
+ if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) ==
+ SCMD_CIPH_MODE_AES_GCM) {
+ frag_count = (data_len / max_frag_size);
+ exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
+ hdr_len;
+ expn_size = frag_count * exp_per_pdu;
+ if (full_pdus_only) {
+ *pdus_per_ulp = data_len / (exp_per_pdu +
+ max_frag_size);
+ if (*pdus_per_ulp > 32)
+ *pdus_per_ulp = 32;
+ else if(!*pdus_per_ulp)
+ *pdus_per_ulp = 1;
+ expn_size = (*pdus_per_ulp) * exp_per_pdu;
+ break;
+ }
+ if ((last_frag_size = data_len % max_frag_size) > 0) {
+ frag_count += 1;
+ expn_size += exp_per_pdu;
+ }
+ break;
+ } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) !=
+ SCMD_CIPH_MODE_NOP) {
+ /* Calculate the number of fragments we can make */
+ frag_count = (data_len / max_frag_size);
+ if (frag_count > 0) {
+ pad_per_pdu = (((howmany((max_frag_size +
+ tls_ofld->mac_length),
+ CIPHER_BLOCK_SIZE)) *
+ CIPHER_BLOCK_SIZE) -
+ (max_frag_size +
+ tls_ofld->mac_length));
+ if (!pad_per_pdu)
+ pad_per_pdu = CIPHER_BLOCK_SIZE;
+ exp_per_pdu = pad_per_pdu +
+ tls_ofld->mac_length +
+ hdr_len + CIPHER_BLOCK_SIZE;
+ expn_size = frag_count * exp_per_pdu;
+ }
+ if (full_pdus_only) {
+ *pdus_per_ulp = data_len / (exp_per_pdu +
+ max_frag_size);
+ if (*pdus_per_ulp > 32)
+ *pdus_per_ulp = 32;
+ else if (!*pdus_per_ulp)
+ *pdus_per_ulp = 1;
+ expn_size = (*pdus_per_ulp) * exp_per_pdu;
+ break;
+ }
+ /* Consider the last fragment */
+ if ((last_frag_size = data_len % max_frag_size) > 0) {
+ pad_last_pdu = (((howmany((last_frag_size +
+ tls_ofld->mac_length),
+ CIPHER_BLOCK_SIZE)) *
+ CIPHER_BLOCK_SIZE) -
+ (last_frag_size +
+ tls_ofld->mac_length));
+ if (!pad_last_pdu)
+ pad_last_pdu = CIPHER_BLOCK_SIZE;
+ expn_size += (pad_last_pdu +
+ tls_ofld->mac_length + hdr_len +
+ CIPHER_BLOCK_SIZE);
+ }
+ }
+ } while (0);
+
+ return (expn_size);
+}
+
+/* Copy Key to WR */
+static void
+tls_copy_tx_key(struct toepcb *toep, void *dst)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct ulptx_sc_memrd *sc_memrd;
+ struct ulptx_idata *sc;
+
+ if (tls_ofld->k_ctx.tx_key_info_size <= 0)
+ return;
+
+ if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) {
+ sc = dst;
+ sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
+ sc->len = htobe32(0);
+ sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
+ sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
+ V_ULP_TX_SC_MORE(1) |
+ V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4));
+ sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5);
+ } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) {
+ memcpy(dst, &tls_ofld->k_ctx.tx,
+ tls_ofld->k_ctx.tx_key_info_size);
+ }
+}
+
+/* TLS/DTLS content type for CPL SFO */
+static inline unsigned char
+tls_content_type(unsigned char content_type)
+{
+ /*
+ * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and
+ * default to "CUSTOM" for all other types including
+ * heartbeat?
+ */
+ switch (content_type) {
+ case CONTENT_TYPE_CCS:
+ return CPL_TX_TLS_SFO_TYPE_CCS;
+ case CONTENT_TYPE_ALERT:
+ return CPL_TX_TLS_SFO_TYPE_ALERT;
+ case CONTENT_TYPE_HANDSHAKE:
+ return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
+ case CONTENT_TYPE_HEARTBEAT:
+ return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
+ }
+ return CPL_TX_TLS_SFO_TYPE_DATA;
+}
+
+static unsigned char
+get_cipher_key_size(unsigned int ck_size)
+{
+ switch (ck_size) {
+ case AES_NOP: /* NOP */
+ return 15;
+ case AES_128: /* AES128 */
+ return CH_CK_SIZE_128;
+ case AES_192: /* AES192 */
+ return CH_CK_SIZE_192;
+ case AES_256: /* AES256 */
+ return CH_CK_SIZE_256;
+ default:
+ return CH_CK_SIZE_256;
+ }
+}
+
+static unsigned char
+get_mac_key_size(unsigned int mk_size)
+{
+ switch (mk_size) {
+ case SHA_NOP: /* NOP */
+ return CH_MK_SIZE_128;
+ case SHA_GHASH: /* GHASH */
+ case SHA_512: /* SHA512 */
+ return CH_MK_SIZE_512;
+ case SHA_224: /* SHA2-224 */
+ return CH_MK_SIZE_192;
+ case SHA_256: /* SHA2-256*/
+ return CH_MK_SIZE_256;
+ case SHA_384: /* SHA384 */
+ return CH_MK_SIZE_512;
+ case SHA1: /* SHA1 */
+ default:
+ return CH_MK_SIZE_160;
+ }
+}
+
+static unsigned int
+get_proto_ver(int proto_ver)
+{
+ switch (proto_ver) {
+ case TLS1_2_VERSION:
+ return TLS_1_2_VERSION;
+ case TLS1_1_VERSION:
+ return TLS_1_1_VERSION;
+ case DTLS1_2_VERSION:
+ return DTLS_1_2_VERSION;
+ default:
+ return TLS_VERSION_MAX;
+ }
+}
+
+static void
+tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx)
+{
+
+ if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
+ kwr->u.rxhdr.ivinsert_to_authinsrt =
+ htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
+ V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
+ V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
+ V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
+ V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
+ kwr->u.rxhdr.ivpresent_to_rxmk_size &=
+ ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1));
+ kwr->u.rxhdr.authmode_to_rxvalid &=
+ ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1));
+ } else {
+ kwr->u.rxhdr.ivinsert_to_authinsrt =
+ htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
+ V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
+ V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
+ V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
+ V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
+ V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
+ }
+}
+
+/* Rx key */
+static void
+prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
+{
+ unsigned int ck_size = kctx->cipher_secret_size;
+ unsigned int mk_size = kctx->mac_secret_size;
+ int proto_ver = kctx->proto_ver;
+
+ kwr->u.rxhdr.flitcnt_hmacctrl =
+ ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
+
+ kwr->u.rxhdr.protover_ciphmode =
+ V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) |
+ V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode);
+
+ kwr->u.rxhdr.authmode_to_rxvalid =
+ V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) |
+ V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) |
+ V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
+ V_TLS_KEYCTX_TX_WR_RXVALID(1);
+
+ kwr->u.rxhdr.ivpresent_to_rxmk_size =
+ V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
+ V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) |
+ V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) |
+ V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size));
+
+ tls_rxkey_flit1(kwr, kctx);
+
+ /* No key reversal for GCM */
+ if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) {
+ t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key,
+ (kctx->cipher_secret_size << 3));
+ memcpy(kwr->keys.edkey + kctx->cipher_secret_size,
+ kctx->rx.key + kctx->cipher_secret_size,
+ (IPAD_SIZE + OPAD_SIZE));
+ } else {
+ memcpy(kwr->keys.edkey, kctx->rx.key,
+ (kctx->tx_key_info_size - SALT_SIZE));
+ memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE);
+ }
+}
+
+/* Tx key */
+static void
+prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
+{
+ unsigned int ck_size = kctx->cipher_secret_size;
+ unsigned int mk_size = kctx->mac_secret_size;
+
+ kwr->u.txhdr.ctxlen =
+ (kctx->tx_key_info_size >> 4);
+ kwr->u.txhdr.dualck_to_txvalid =
+ V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) |
+ V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
+ V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) |
+ V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) |
+ V_TLS_KEYCTX_TX_WR_TXVALID(1);
+
+ memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE);
+ if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
+ memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE);
+ kwr->u.txhdr.dualck_to_txvalid &=
+ ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1));
+ }
+ kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid);
+}
+
+/* TLS Key memory management */
+int
+tls_init_kmap(struct adapter *sc, struct tom_data *td)
+{
+
+ td->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
+ sc->vres.key.size, 8, 0, M_FIRSTFIT | M_NOWAIT);
+ if (td->key_map == NULL)
+ return (ENOMEM);
+ return (0);
+}
+
+void
+tls_free_kmap(struct tom_data *td)
+{
+
+ if (td->key_map != NULL)
+ vmem_destroy(td->key_map);
+}
+
+static int
+get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx)
+{
+ struct tom_data *td = toep->td;
+ vmem_addr_t addr;
+
+ if (vmem_alloc(td->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
+ &addr) != 0)
+ return (-1);
+
+ return (addr);
+}
+
+static void
+free_keyid(struct toepcb *toep, int keyid)
+{
+ struct tom_data *td = toep->td;
+
+ vmem_free(td->key_map, keyid, TLS_KEY_CONTEXT_SZ);
+}
+
+static void
+clear_tls_keyid(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ if (tls_ofld->rx_key_addr >= 0) {
+ free_keyid(toep, tls_ofld->rx_key_addr);
+ tls_ofld->rx_key_addr = -1;
+ }
+ if (tls_ofld->tx_key_addr >= 0) {
+ free_keyid(toep, tls_ofld->tx_key_addr);
+ tls_ofld->tx_key_addr = -1;
+ }
+}
+
+static int
+get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops)
+{
+ return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr :
+ ((ops & KEY_WRITE_TX) ? tls_ofld->rx_key_addr : -1));
+}
+
+static int
+get_tp_plen_max(struct tls_ofld_info *tls_ofld)
+{
+ int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448;
+
+ return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX);
+}
+
+/* Send request to get the key-id */
+static int
+tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct adapter *sc = td_adapter(toep->td);
+ struct ofld_tx_sdesc *txsd;
+ int kwrlen, kctxlen, keyid, len;
+ struct wrqe *wr;
+ struct tls_key_req *kwr;
+ struct tls_keyctx *kctx;
+
+ kwrlen = roundup2(sizeof(*kwr), 16);
+ kctxlen = roundup2(sizeof(*kctx), 32);
+ len = kwrlen + kctxlen;
+
+ if (toep->txsd_avail == 0)
+ return (EAGAIN);
+
+ /* Dont initialize key for re-neg */
+ if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) {
+ if ((keyid = get_new_keyid(toep, k_ctx)) < 0) {
+ return (ENOSPC);
+ }
+ } else {
+ keyid = get_keyid(tls_ofld, k_ctx->l_p_key);
+ }
+
+ wr = alloc_wrqe(len, toep->ofld_txq);
+ if (wr == NULL) {
+ free_keyid(toep, keyid);
+ return (ENOMEM);
+ }
+ kwr = wrtod(wr);
+ memset(kwr, 0, kwrlen);
+
+ kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL |
+ F_FW_WR_ATOMIC);
+ kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) |
+ V_FW_WR_FLOWID(toep->tid));
+ kwr->protocol = get_proto_ver(k_ctx->proto_ver);
+ kwr->mfs = htons(k_ctx->frag_size);
+ kwr->reneg_to_write_rx = k_ctx->l_p_key;
+
+ /* master command */
+ kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
+ V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
+ kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
+ kwr->len16 = htobe32((toep->tid << 8) |
+ DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
+ kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
+
+ /* sub command */
+ kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+ kwr->sc_len = htobe32(kctxlen);
+
+ /* XXX: This assumes that kwrlen == sizeof(*kwr). */
+ kctx = (struct tls_keyctx *)(kwr + 1);
+ memset(kctx, 0, kctxlen);
+
+ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
+ tls_ofld->tx_key_addr = keyid;
+ prepare_txkey_wr(kctx, k_ctx);
+ } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
+ tls_ofld->rx_key_addr = keyid;
+ prepare_rxkey_wr(kctx, k_ctx);
+ }
+
+ txsd = &toep->txsd[toep->txsd_pidx];
+ txsd->tx_credits = DIV_ROUND_UP(len, 16);
+ txsd->plen = 0;
+ toep->tx_credits -= txsd->tx_credits;
+ if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
+ toep->txsd_pidx = 0;
+ toep->txsd_avail--;
+
+ t4_wrq_tx(sc, wr);
+
+ return (0);
+}
+
+/* Store a key received from SSL in DDR. */
+static int
+program_key_context(struct tcpcb *tp, struct toepcb *toep,
+ struct tls_key_context *uk_ctx)
+{
+ struct adapter *sc = td_adapter(toep->td);
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct tls_key_context *k_ctx;
+ int error, key_offset;
+
+ if (tp->t_state != TCPS_ESTABLISHED) {
+ /*
+ * XXX: Matches Linux driver, but not sure this is a
+ * very appropriate error.
+ */
+ return (ENOENT);
+ }
+
+ /* Stop timer on handshake completion */
+ tls_stop_handshake_timer(toep);
+
+ toep->flags &= ~TPF_FORCE_CREDITS;
+
+ CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid,
+ G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" :
+ "KEY_WRITE_TX", uk_ctx->proto_ver);
+
+ if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX &&
+ toep->ulp_mode != ULP_MODE_TLS)
+ return (EOPNOTSUPP);
+
+ /* Don't copy the 'tx' and 'rx' fields. */
+ k_ctx = &tls_ofld->k_ctx;
+ memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key,
+ sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key));
+
+ /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */
+ if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) {
+ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
+ tls_ofld->rx_key_addr = -1;
+ t4_clear_rx_quiesce(toep);
+ } else {
+ tls_ofld->tx_key_addr = -1;
+ }
+ return (0);
+ }
+
+ if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
+ k_ctx->iv_size = 4;
+ k_ctx->mac_first = 0;
+ k_ctx->hmac_ctrl = 0;
+ } else {
+ k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
+ k_ctx->mac_first = 1;
+ }
+
+ tls_ofld->scmd0.seqno_numivs =
+ (V_SCMD_SEQ_NO_CTRL(3) |
+ V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
+ V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
+ V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
+ V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
+ V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
+ V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
+ V_SCMD_IV_SIZE(k_ctx->iv_size));
+
+ tls_ofld->scmd0.ivgen_hdrlen =
+ (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
+ V_SCMD_KEY_CTX_INLINE(0) |
+ V_SCMD_TLS_FRAG_ENABLE(1));
+
+ tls_ofld->mac_length = k_ctx->mac_secret_size;
+
+ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
+ k_ctx->rx = uk_ctx->rx;
+ /* Dont initialize key for re-neg */
+ if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
+ tls_ofld->rx_key_addr = -1;
+ } else {
+ k_ctx->tx = uk_ctx->tx;
+ /* Dont initialize key for re-neg */
+ if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
+ tls_ofld->tx_key_addr = -1;
+ }
+
+ /* Flush pending data before new Tx key becomes active */
+ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
+ struct sockbuf *sb;
+
+ /* XXX: This might not drain everything. */
+ t4_push_frames(sc, toep, 0);
+ sb = &toep->inp->inp_socket->so_snd;
+ SOCKBUF_LOCK(sb);
+
+ /* XXX: This asserts that everything has been pushed. */
+ MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL);
+ sb->sb_sndptr = NULL;
+ tls_ofld->sb_off = sbavail(sb);
+ SOCKBUF_UNLOCK(sb);
+ tls_ofld->tx_seq_no = 0;
+ }
+
+ if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) ||
+ (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) {
+ error = tls_program_key_id(toep, k_ctx);
+ if (error) {
+ /* XXX: Only clear quiesce for KEY_WRITE_RX? */
+ t4_clear_rx_quiesce(toep);
+ return (error);
+ }
+ }
+
+ if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
+ /*
+ * RX key tags are an index into the key portion of MA
+ * memory stored as an offset from the base address in
+ * units of 64 bytes.
+ */
+ key_offset = tls_ofld->rx_key_addr - sc->vres.key.start;
+ t4_set_tls_keyid(toep, key_offset / 64);
+ t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
+ V_TCB_ULP_RAW(M_TCB_ULP_RAW),
+ V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
+ V_TF_TLS_CONTROL(1) |
+ V_TF_TLS_ACTIVE(1) |
+ V_TF_TLS_ENABLE(1))));
+ t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
+ V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
+ V_TCB_TLS_SEQ(0));
+ t4_clear_rx_quiesce(toep);
+ } else {
+ unsigned short pdus_per_ulp;
+
+ if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE)
+ tls_ofld->tx_key_addr = 1;
+
+ tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld);
+ tls_ofld->expn_per_ulp = tls_expansion_size(toep,
+ tls_ofld->fcplenmax, 1, &pdus_per_ulp);
+ tls_ofld->pdus_per_ulp = pdus_per_ulp;
+ tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp *
+ ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) +
+ tls_ofld->k_ctx.frag_size);
+ }
+
+ return (0);
+}
+
+/*
+ * In some cases a client connection can hang without sending the
+ * ServerHelloDone message from the NIC to the host. Send a dummy
+ * RX_DATA_ACK with RX_MODULATE to unstick the connection.
+ */
+static void
+tls_send_handshake_ack(void *arg)
+{
+ struct toepcb *toep = arg;
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct adapter *sc = td_adapter(toep->td);
+
+ /*
+ * XXX: Does not have the t4_get_tcb() checks to refine the
+ * workaround.
+ */
+ callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);
+
+ CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
+ send_rx_modulate(sc, toep);
+}
+
+static void
+tls_start_handshake_timer(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ mtx_lock(&tls_handshake_lock);
+ callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
+ tls_send_handshake_ack, toep);
+ mtx_unlock(&tls_handshake_lock);
+}
+
+void
+tls_stop_handshake_timer(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ mtx_lock(&tls_handshake_lock);
+ callout_stop(&tls_ofld->handshake_timer);
+ mtx_unlock(&tls_handshake_lock);
+}
+
+int
+t4_ctloutput_tls(struct socket *so, struct sockopt *sopt)
+{
+ struct tls_key_context uk_ctx;
+ struct inpcb *inp;
+ struct tcpcb *tp;
+ struct toepcb *toep;
+ int error, optval;
+
+ error = 0;
+ if (sopt->sopt_dir == SOPT_SET &&
+ sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) {
+ error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx),
+ sizeof(uk_ctx));
+ if (error)
+ return (error);
+ }
+
+ inp = sotoinpcb(so);
+ KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
+ INP_WLOCK(inp);
+ if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
+ INP_WUNLOCK(inp);
+ return (ECONNRESET);
+ }
+ tp = intotcpcb(inp);
+ toep = tp->t_toe;
+ switch (sopt->sopt_dir) {
+ case SOPT_SET:
+ switch (sopt->sopt_name) {
+ case TCP_TLSOM_SET_TLS_CONTEXT:
+ error = program_key_context(tp, toep, &uk_ctx);
+ INP_WUNLOCK(inp);
+ break;
+ case TCP_TLSOM_CLR_TLS_TOM:
+ if (toep->ulp_mode == ULP_MODE_TLS) {
+ CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM",
+ __func__, toep->tid);
+ tls_clr_ofld_mode(toep);
+ } else
+ error = EOPNOTSUPP;
+ INP_WUNLOCK(inp);
+ break;
+ case TCP_TLSOM_CLR_QUIES:
+ if (toep->ulp_mode == ULP_MODE_TLS) {
+ CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES",
+ __func__, toep->tid);
+ tls_clr_quiesce(toep);
+ } else
+ error = EOPNOTSUPP;
+ INP_WUNLOCK(inp);
+ break;
+ default:
+ INP_WUNLOCK(inp);
+ error = EOPNOTSUPP;
+ break;
+ }
+ break;
+ case SOPT_GET:
+ switch (sopt->sopt_name) {
+ case TCP_TLSOM_GET_TLS_TOM:
+ /*
+ * TLS TX is permitted on any TOE socket, but
+ * TLS RX requires a TLS ULP mode.
+ */
+ optval = TLS_TOM_NONE;
+ if (can_tls_offload(td_adapter(toep->td))) {
+ switch (toep->ulp_mode) {
+ case ULP_MODE_NONE:
+ case ULP_MODE_TCPDDP:
+ optval = TLS_TOM_TXONLY;
+ break;
+ case ULP_MODE_TLS:
+ optval = TLS_TOM_BOTH;
+ break;
+ }
+ }
+ CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d",
+ __func__, toep->tid, optval);
+ INP_WUNLOCK(inp);
+ error = sooptcopyout(sopt, &optval, sizeof(optval));
+ break;
+ default:
+ INP_WUNLOCK(inp);
+ error = EOPNOTSUPP;
+ break;
+ }
+ break;
+ }
+ return (error);
+}
+
+void
+tls_init_toep(struct toepcb *toep)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+
+ tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR;
+ tls_ofld->rx_key_addr = -1;
+ tls_ofld->tx_key_addr = -1;
+ if (toep->ulp_mode == ULP_MODE_TLS)
+ callout_init_mtx(&tls_ofld->handshake_timer,
+ &tls_handshake_lock, 0);
+}
+
+void
+tls_establish(struct toepcb *toep)
+{
+
+ /*
+ * Enable PDU extraction.
+ *
+ * XXX: Supposedly this should be done by the firmware when
+ * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
+ * in practice this seems to be required.
+ */
+ CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
+ t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
+ V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
+
+ toep->flags |= TPF_FORCE_CREDITS;
+
+ tls_start_handshake_timer(toep);
+}
+
+void
+tls_uninit_toep(struct toepcb *toep)
+{
+
+ if (toep->ulp_mode == ULP_MODE_TLS)
+ tls_stop_handshake_timer(toep);
+ clear_tls_keyid(toep);
+}
+
+#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
+#define MIN_OFLD_TLSTX_CREDITS(toep) \
+ (howmany(sizeof(struct fw_tlstx_data_wr) + \
+ sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \
+ CIPHER_BLOCK_SIZE + 1, 16))
+
+static inline u_int
+max_imm_tls_space(int tx_credits)
+{
+ const int n = 2; /* Use only up to 2 desc for imm. data WR */
+ int space;
+
+ KASSERT(tx_credits >= 0 &&
+ tx_credits <= MAX_OFLD_TX_CREDITS,
+ ("%s: %d credits", __func__, tx_credits));
+
+ if (tx_credits >= (n * EQ_ESIZE) / 16)
+ space = (n * EQ_ESIZE);
+ else
+ space = tx_credits * 16;
+ return (space);
+}
+
+static int
+count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp)
+{
+ int max_nsegs_1mbuf, n, nsegs;
+
+ while (skip >= m->m_len) {
+ skip -= m->m_len;
+ m = m->m_next;
+ }
+
+ nsegs = 0;
+ max_nsegs_1mbuf = 0;
+ while (len > 0) {
+ n = sglist_count(mtod(m, char *) + skip, m->m_len - skip);
+ if (n > max_nsegs_1mbuf)
+ max_nsegs_1mbuf = n;
+ nsegs += n;
+ len -= m->m_len - skip;
+ skip = 0;
+ m = m->m_next;
+ }
+ *max_nsegs_1mbufp = max_nsegs_1mbuf;
+ return (nsegs);
+}
+
+static void
+write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
+ unsigned int immdlen, unsigned int plen, unsigned int expn,
+ unsigned int pdus, uint8_t credits, int shove, int imm_ivs)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ unsigned int len = plen + expn;
+
+ txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) |
+ V_FW_TLSTX_DATA_WR_COMPL(1) |
+ V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen));
+ txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
+ V_FW_TLSTX_DATA_WR_LEN16(credits));
+ txwr->plen = htobe32(len);
+ txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) |
+ V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove));
+ txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) |
+ V_FW_TLSTX_DATA_WR_EXP(expn) |
+ V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) |
+ V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) |
+ V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4));
+ txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size);
+ txwr->adjustedplen_pkd = htobe16(
+ V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen));
+ txwr->expinplenmax_pkd = htobe16(
+ V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp));
+ txwr->pdusinplenmax_pkd = htobe16(
+ V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp));
+}
+
+static void
+write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
+ struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus)
+{
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ int data_type, seglen;
+
+ if (plen < tls_ofld->k_ctx.frag_size)
+ seglen = plen;
+ else
+ seglen = tls_ofld->k_ctx.frag_size;
+ data_type = tls_content_type(tls_hdr->type);
+ cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) |
+ V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) |
+ V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen));
+ cpl->pld_len = htobe32(plen);
+ if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT)
+ cpl->type_protover = htobe32(
+ V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type));
+ cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs |
+ V_SCMD_NUM_IVS(pdus));
+ cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen);
+ cpl->scmd1 = htobe64(tls_ofld->tx_seq_no);
+ tls_ofld->tx_seq_no += pdus;
+}
+
+/*
+ * Similar to write_tx_sgl() except that it accepts an optional
+ * trailer buffer for IVs.
+ */
+static void
+write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen,
+ void *iv_buffer, int iv_len, int nsegs, int n)
+{
+ struct mbuf *m;
+ struct ulptx_sgl *usgl = dst;
+ int i, j, rc;
+ struct sglist sg;
+ struct sglist_seg segs[n];
+
+ KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
+
+ sglist_init(&sg, n, segs);
+ usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
+ V_ULPTX_NSGE(nsegs));
+
+ for (m = start; skip >= m->m_len; m = m->m_next)
+ skip -= m->m_len;
+
+ i = -1;
+ for (m = start; plen > 0; m = m->m_next) {
+ rc = sglist_append(&sg, mtod(m, char *) + skip,
+ m->m_len - skip);
+ if (__predict_false(rc != 0))
+ panic("%s: sglist_append %d", __func__, rc);
+ plen -= m->m_len - skip;
+ skip = 0;
+
+ for (j = 0; j < sg.sg_nseg; i++, j++) {
+ if (i < 0) {
+ usgl->len0 = htobe32(segs[j].ss_len);
+ usgl->addr0 = htobe64(segs[j].ss_paddr);
+ } else {
+ usgl->sge[i / 2].len[i & 1] =
+ htobe32(segs[j].ss_len);
+ usgl->sge[i / 2].addr[i & 1] =
+ htobe64(segs[j].ss_paddr);
+ }
+#ifdef INVARIANTS
+ nsegs--;
+#endif
+ }
+ sglist_reset(&sg);
+ }
+ if (iv_buffer != NULL) {
+ rc = sglist_append(&sg, iv_buffer, iv_len);
+ if (__predict_false(rc != 0))
+ panic("%s: sglist_append %d", __func__, rc);
+
+ for (j = 0; j < sg.sg_nseg; i++, j++) {
+ if (i < 0) {
+ usgl->len0 = htobe32(segs[j].ss_len);
+ usgl->addr0 = htobe64(segs[j].ss_paddr);
+ } else {
+ usgl->sge[i / 2].len[i & 1] =
+ htobe32(segs[j].ss_len);
+ usgl->sge[i / 2].addr[i & 1] =
+ htobe64(segs[j].ss_paddr);
+ }
+#ifdef INVARIANTS
+ nsegs--;
+#endif
+ }
+ }
+ if (i & 1)
+ usgl->sge[i / 2].len[1] = htobe32(0);
+ KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p",
+ __func__, nsegs, start, iv_buffer));
+}
+
+/*
+ * Similar to t4_push_frames() but handles TLS sockets when TLS offload
+ * is enabled. Rather than transmitting bulk data, the socket buffer
+ * contains TLS records. The work request requires a full TLS record,
+ * so batch mbufs up until a full TLS record is seen. This requires
+ * reading the TLS header out of the start of each record to determine
+ * its length.
+ */
+void
+t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
+{
+ struct tls_hdr thdr;
+ struct mbuf *sndptr;
+ struct fw_tlstx_data_wr *txwr;
+ struct cpl_tx_tls_sfo *cpl;
+ struct wrqe *wr;
+ u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len;
+ u_int expn_size, iv_len, pdus, sndptroff;
+ struct tls_ofld_info *tls_ofld = &toep->tls;
+ struct inpcb *inp = toep->inp;
+ struct tcpcb *tp = intotcpcb(inp);
+ struct socket *so = inp->inp_socket;
+ struct sockbuf *sb = &so->so_snd;
+ int tls_size, tx_credits, shove, /* compl,*/ sowwakeup;
+ struct ofld_tx_sdesc *txsd;
+ bool imm_ivs, imm_payload;
+ void *iv_buffer, *iv_dst, *buf;
+
+ INP_WLOCK_ASSERT(inp);
+ KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
+ ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
+
+ KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
+ toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS,
+ ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
+ KASSERT(tls_tx_key(toep),
+ ("%s: TX key not set for toep %p", __func__, toep));
+
+#ifdef VERBOSE_TRACES
+ CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
+ __func__, toep->tid, toep->flags, tp->t_flags);
+#endif
+ if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
+ return;
+
+#ifdef RATELIMIT
+ if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
+ (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
+ inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
+ }
+#endif
+
+ /*
+ * This function doesn't resume by itself. Someone else must clear the
+ * flag and call this function.
+ */
+ if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
+ KASSERT(drop == 0,
+ ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
+ return;
+ }
+
+ txsd = &toep->txsd[toep->txsd_pidx];
+ for (;;) {
+ tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
+ space = max_imm_tls_space(tx_credits);
+ wr_len = sizeof(struct fw_tlstx_data_wr) +
+ sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
+ if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) {
+#ifdef VERBOSE_TRACES
+ CTR5(KTR_CXGBE,
+ "%s: tid %d tx_credits %d min_wr %d space %d",
+ __func__, toep->tid, tx_credits, wr_len +
+ CIPHER_BLOCK_SIZE + 1, space);
+#endif
+ return;
+ }
+
+ SOCKBUF_LOCK(sb);
+ sowwakeup = drop;
+ if (drop) {
+ sbdrop_locked(sb, drop);
+ MPASS(tls_ofld->sb_off >= drop);
+ tls_ofld->sb_off -= drop;
+ drop = 0;
+ }
+
+ /*
+ * Send a FIN if requested, but only if there's no
+ * more data to send.
+ */
+ if (sbavail(sb) == 0 && toep->flags & TPF_SEND_FIN) {
+ if (sowwakeup)
+ sowwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(sb);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ t4_close_conn(sc, toep);
+ return;
+ }
+
+ if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) {
+ /*
+ * A full TLS header is not yet queued, stop
+ * for now until more data is added to the
+ * socket buffer.
+ */
+#ifdef VERBOSE_TRACES
+ CTR4(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d",
+ __func__, toep->tid, sbavail(sb), tls_ofld->sb_off);
+#endif
+ if (sowwakeup)
+ sowwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(sb);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ return;
+ }
+
+ /* Read the header of the next TLS record. */
+ sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff);
+ MPASS(!IS_AIOTX_MBUF(sndptr));
+ m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr);
+ tls_size = htons(thdr.length);
+ plen = TLS_HEADER_LENGTH + tls_size;
+ pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size);
+ iv_len = pdus * CIPHER_BLOCK_SIZE;
+
+ if (sbavail(sb) < tls_ofld->sb_off + plen) {
+ /*
+ * The full TLS record is not yet queued, stop
+ * for now until more data is added to the
+ * socket buffer.
+ */
+#ifdef VERBOSE_TRACES
+ CTR5(KTR_CXGBE,
+ "%s: tid %d sbavail %d sb_off %d plen %d",
+ __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
+ plen);
+#endif
+ if (sowwakeup)
+ sowwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(sb);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ return;
+ }
+
+ /* Shove if there is no additional data pending. */
+ shove = (sbavail(sb) == tls_ofld->sb_off + plen) &&
+ !(tp->t_flags & TF_MORETOCOME);
+
+ if (sb->sb_flags & SB_AUTOSIZE &&
+ V_tcp_do_autosndbuf &&
+ sb->sb_hiwat < V_tcp_autosndbuf_max &&
+ sbused(sb) >= sb->sb_hiwat * 7 / 8) {
+ int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
+ V_tcp_autosndbuf_max);
+
+ if (!sbreserve_locked(sb, newsize, so, NULL))
+ sb->sb_flags &= ~SB_AUTOSIZE;
+ else
+ sowwakeup = 1; /* room available */
+ }
+ if (sowwakeup)
+ sowwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(sb);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+
+ if (__predict_false(toep->flags & TPF_FIN_SENT))
+ panic("%s: excess tx.", __func__);
+
+ /* Determine whether to use immediate vs SGL. */
+ imm_payload = false;
+ imm_ivs = false;
+ if (wr_len + iv_len <= space) {
+ imm_ivs = true;
+ wr_len += iv_len;
+ if (wr_len + tls_size <= space) {
+ wr_len += tls_size;
+ imm_payload = true;
+ }
+ }
+
+ /* Allocate space for IVs if needed. */
+ if (!imm_ivs) {
+ iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT);
+ if (iv_buffer == NULL) {
+ /*
+ * XXX: How to restart this?
+ */
+ if (sowwakeup)
+ sowwakeup_locked(so);
+ else
+ SOCKBUF_UNLOCK(sb);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ CTR3(KTR_CXGBE,
+ "%s: tid %d failed to alloc IV space len %d",
+ __func__, toep->tid, iv_len);
+ return;
+ }
+ } else
+ iv_buffer = NULL;
+
+ /* Determine size of SGL. */
+ nsegs = 0;
+ max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
+ if (!imm_payload) {
+ nsegs = count_mbuf_segs(sndptr, sndptroff +
+ TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf);
+ if (!imm_ivs) {
+ int n = sglist_count(iv_buffer, iv_len);
+ nsegs += n;
+ if (n > max_nsegs_1mbuf)
+ max_nsegs_1mbuf = n;
+ }
+
+ /* Account for SGL in work request length. */
+ wr_len += sizeof(struct ulptx_sgl) +
+ ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
+ }
+
+ wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
+ if (wr == NULL) {
+ /* XXX: how will we recover from this? */
+ toep->flags |= TPF_TX_SUSPENDED;
+ return;
+ }
+
+#ifdef VERBOSE_TRACES
+ CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d",
+ __func__, toep->tid, thdr.type, tls_size, pdus);
+#endif
+ txwr = wrtod(wr);
+ cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
+ memset(txwr, 0, roundup2(wr_len, 16));
+ credits = howmany(wr_len, 16);
+ expn_size = tls_expansion_size(toep, tls_size, 0, NULL);
+ write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0,
+ tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0);
+ write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus);
+ tls_copy_tx_key(toep, cpl + 1);
+
+ /* Generate random IVs */
+ buf = (char *)(cpl + 1) + key_size(toep);
+ if (imm_ivs) {
+ MPASS(iv_buffer == NULL);
+ iv_dst = buf;
+ buf = (char *)iv_dst + iv_len;
+ } else
+ iv_dst = iv_buffer;
+ arc4rand(iv_dst, iv_len, 0);
+
+ if (imm_payload) {
+ m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH,
+ tls_size, buf);
+ } else {
+ write_tlstx_sgl(buf, sndptr,
+ sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer,
+ iv_len, nsegs, max_nsegs_1mbuf);
+ }
+
+ KASSERT(toep->tx_credits >= credits,
+ ("%s: not enough credits", __func__));
+
+ toep->tx_credits -= credits;
+
+ tp->snd_nxt += plen;
+ tp->snd_max += plen;
+
+ SOCKBUF_LOCK(sb);
+ sbsndptr(sb, tls_ofld->sb_off, plen, &sndptroff);
+ tls_ofld->sb_off += plen;
+ SOCKBUF_UNLOCK(sb);
+
+ toep->flags |= TPF_TX_DATA_SENT;
+ if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
+ toep->flags |= TPF_TX_SUSPENDED;
+
+ KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
+ txsd->plen = plen;
+ txsd->tx_credits = credits;
+ txsd->iv_buffer = iv_buffer;
+ txsd++;
+ if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
+ toep->txsd_pidx = 0;
+ txsd = &toep->txsd[0];
+ }
+ toep->txsd_avail--;
+
+ atomic_add_long(&toep->vi->pi->tx_tls_records, 1);
+ atomic_add_long(&toep->vi->pi->tx_tls_octets, plen);
+
+ t4_l2t_send(sc, wr, toep->l2te);
+ }
+}
+
+/*
+ * For TLS data we place received mbufs received via CPL_TLS_DATA into
+ * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is
+ * received, the completed PDUs are placed into the socket receive
+ * buffer.
+ *
+ * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs.
+ */
+static int
+do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_tls_data *cpl = mtod(m, const void *);
+ unsigned int tid = GET_TID(cpl);
+ struct toepcb *toep = lookup_tid(sc, tid);
+ struct inpcb *inp = toep->inp;
+ struct tcpcb *tp;
+ int len;
+
+ /* XXX: Should this match do_rx_data instead? */
+ KASSERT(!(toep->flags & TPF_SYNQE),
+ ("%s: toep %p claims to be a synq entry", __func__, toep));
+
+ KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
+
+ /* strip off CPL header */
+ m_adj(m, sizeof(*cpl));
+ len = m->m_pkthdr.len;
+
+ atomic_add_long(&toep->vi->pi->rx_tls_octets, len);
+
+ KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
+ ("%s: payload length mismatch", __func__));
+
+ INP_WLOCK(inp);
+ if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
+ __func__, tid, len, inp->inp_flags);
+ INP_WUNLOCK(inp);
+ m_freem(m);
+ return (0);
+ }
+
+ /* Save TCP sequence number. */
+ m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq);
+
+ if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
+#ifdef INVARIANTS
+ panic("Failed to queue TLS data packet");
+#else
+ printf("%s: Failed to queue TLS data packet\n", __func__);
+ INP_WUNLOCK(inp);
+ m_freem(m);
+ return (0);
+#endif
+ }
+
+ tp = intotcpcb(inp);
+ tp->t_rcvtime = ticks;
+
+#ifdef VERBOSE_TRACES
+ CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
+ be32toh(cpl->seq));
+#endif
+
+ INP_WUNLOCK(inp);
+ return (0);
+}
+
+static int
+do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *);
+ struct tlsrx_hdr_pkt *tls_hdr_pkt;
+ unsigned int tid = GET_TID(cpl);
+ struct toepcb *toep = lookup_tid(sc, tid);
+ struct inpcb *inp = toep->inp;
+ struct tcpcb *tp;
+ struct socket *so;
+ struct sockbuf *sb;
+ struct mbuf *tls_data;
+ int len, pdu_length, pdu_overhead, sb_length;
+
+ KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
+ KASSERT(!(toep->flags & TPF_SYNQE),
+ ("%s: toep %p claims to be a synq entry", __func__, toep));
+
+ /* strip off CPL header */
+ m_adj(m, sizeof(*cpl));
+ len = m->m_pkthdr.len;
+
+ atomic_add_long(&toep->vi->pi->rx_tls_records, 1);
+
+ KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
+ ("%s: payload length mismatch", __func__));
+
+ INP_WLOCK(inp);
+ if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
+ CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
+ __func__, tid, len, inp->inp_flags);
+ INP_WUNLOCK(inp);
+ m_freem(m);
+ return (0);
+ }
+
+ pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
+
+ tp = intotcpcb(inp);
+
+#ifdef VERBOSE_TRACES
+ CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
+ __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt);
+#endif
+
+ tp->rcv_nxt += pdu_length;
+ if (tp->rcv_wnd < pdu_length) {
+ toep->tls.rcv_over += pdu_length - tp->rcv_wnd;
+ tp->rcv_wnd = 0;
+ } else
+ tp->rcv_wnd -= pdu_length;
+
+ /* XXX: Not sure what to do about urgent data. */
+
+ /*
+ * The payload of this CPL is the TLS header followed by
+ * additional fields.
+ */
+ KASSERT(m->m_len >= sizeof(*tls_hdr_pkt),
+ ("%s: payload too small", __func__));
+ tls_hdr_pkt = mtod(m, void *);
+
+ /*
+ * Only the TLS header is sent to OpenSSL, so report errors by
+ * altering the record type.
+ */
+ if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0)
+ tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
+
+ /* Trim this CPL's mbuf to only include the TLS header. */
+ KASSERT(m->m_len == len && m->m_next == NULL,
+ ("%s: CPL spans multiple mbufs", __func__));
+ m->m_len = TLS_HEADER_LENGTH;
+ m->m_pkthdr.len = TLS_HEADER_LENGTH;
+
+ tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
+ if (tls_data != NULL) {
+ KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
+ ("%s: sequence mismatch", __func__));
+
+ /*
+ * Update the TLS header length to be the length of
+ * the payload data.
+ */
+ tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
+
+ m->m_next = tls_data;
+ m->m_pkthdr.len += tls_data->m_len;
+ }
+
+ so = inp_inpcbtosocket(inp);
+ sb = &so->so_rcv;
+ SOCKBUF_LOCK(sb);
+
+ if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
+ CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
+ __func__, tid, pdu_length);
+ m_freem(m);
+ SOCKBUF_UNLOCK(sb);
+ INP_WUNLOCK(inp);
+
+ CURVNET_SET(toep->vnet);
+ INP_INFO_RLOCK(&V_tcbinfo);
+ INP_WLOCK(inp);
+ tp = tcp_drop(tp, ECONNRESET);
+ if (tp)
+ INP_WUNLOCK(inp);
+ INP_INFO_RUNLOCK(&V_tcbinfo);
+ CURVNET_RESTORE();
+
+ return (0);
+ }
+
+ /*
+ * Not all of the bytes on the wire are included in the socket
+ * buffer (e.g. the MAC of the TLS record). However, those
+ * bytes are included in the TCP sequence space. To handle
+ * this, compute the delta for this TLS record in
+ * 'pdu_overhead' and treat those bytes as having already been
+ * "read" by the application for the purposes of expanding the
+ * window. The meat of the TLS record passed to the
+ * application ('sb_length') will still not be counted as
+ * "read" until userland actually reads the bytes.
+ *
+ * XXX: Some of the calculations below are probably still not
+ * really correct.
+ */
+ sb_length = m->m_pkthdr.len;
+ pdu_overhead = pdu_length - sb_length;
+ toep->rx_credits += pdu_overhead;
+ tp->rcv_wnd += pdu_overhead;
+ tp->rcv_adv += pdu_overhead;
+
+ /* receive buffer autosize */
+ MPASS(toep->vnet == so->so_vnet);
+ CURVNET_SET(toep->vnet);
+ if (sb->sb_flags & SB_AUTOSIZE &&
+ V_tcp_do_autorcvbuf &&
+ sb->sb_hiwat < V_tcp_autorcvbuf_max &&
+ sb_length > (sbspace(sb) / 8 * 7)) {
+ unsigned int hiwat = sb->sb_hiwat;
+ unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
+ V_tcp_autorcvbuf_max);
+
+ if (!sbreserve_locked(sb, newsize, so, NULL))
+ sb->sb_flags &= ~SB_AUTOSIZE;
+ else
+ toep->rx_credits += newsize - hiwat;
+ }
+
+ KASSERT(toep->sb_cc >= sbused(sb),
+ ("%s: sb %p has more data (%d) than last time (%d).",
+ __func__, sb, sbused(sb), toep->sb_cc));
+ toep->rx_credits += toep->sb_cc - sbused(sb);
+ sbappendstream_locked(sb, m, 0);
+ toep->sb_cc = sbused(sb);
+#ifdef VERBOSE_TRACES
+ CTR5(KTR_CXGBE, "%s: tid %u PDU overhead %d rx_credits %u rcv_wnd %u",
+ __func__, tid, pdu_overhead, toep->rx_credits, tp->rcv_wnd);
+#endif
+ if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) {
+ int credits;
+
+ credits = send_rx_credits(sc, toep, toep->rx_credits);
+ toep->rx_credits -= credits;
+ tp->rcv_wnd += credits;
+ tp->rcv_adv += credits;
+ }
+
+ sorwakeup_locked(so);
+ SOCKBUF_UNLOCK_ASSERT(sb);
+
+ INP_WUNLOCK(inp);
+ CURVNET_RESTORE();
+ return (0);
+}
+
+void
+t4_tls_mod_load(void)
+{
+
+ mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF);
+ t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
+ t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
+}
+
+void
+t4_tls_mod_unload(void)
+{
+
+ t4_register_cpl_handler(CPL_TLS_DATA, NULL);
+ t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
+ mtx_destroy(&tls_handshake_lock);
+}
+#endif /* TCP_OFFLOAD */
Index: head/sys/dev/cxgbe/tom/t4_tom.h
===================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.h
+++ head/sys/dev/cxgbe/tom/t4_tom.h
@@ -33,6 +33,7 @@
#ifndef __T4_TOM_H__
#define __T4_TOM_H__
#include <sys/vmem.h>
+#include "tom/t4_tls.h"
#define LISTEN_HASH_SIZE 32
@@ -71,6 +72,8 @@
TPF_SYNQE_TCPDDP = (1 << 10), /* ulp_mode TCPDDP in toepcb */
TPF_SYNQE_EXPANDED = (1 << 11), /* toepcb ready, tid context updated */
TPF_SYNQE_HAS_L2TE = (1 << 12), /* we've replied to PASS_ACCEPT_REQ */
+ TPF_SYNQE_TLS = (1 << 13), /* ulp_mode TLS in toepcb */
+ TPF_FORCE_CREDITS = (1 << 14), /* always send credits */
};
enum {
@@ -83,9 +86,12 @@
DDP_DEAD = (1 << 6), /* toepcb is shutting down */
};
+struct sockopt;
+
struct ofld_tx_sdesc {
uint32_t plen; /* payload length */
uint8_t tx_credits; /* firmware tx credits (unit is 16B) */
+ void *iv_buffer; /* optional buffer holding IVs for TLS */
};
struct ppod_region {
@@ -125,6 +131,9 @@
#define EXT_FLAG_AIOTX EXT_FLAG_VENDOR1
+#define IS_AIOTX_MBUF(m) \
+ ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX)
+
struct ddp_buffer {
struct pageset *ps;
@@ -185,6 +194,7 @@
struct mbufq ulp_pdu_reclaimq;
struct ddp_pcb ddp;
+ struct tls_ofld_info tls;
TAILQ_HEAD(, kaiocb) aiotx_jobq;
struct task aiotx_task;
@@ -269,6 +279,8 @@
struct ppod_region pr;
+ vmem_t *key_map;
+
struct mtx clip_table_lock;
struct clip_head clip_table;
int clip_gen;
@@ -309,6 +321,18 @@
return (m->m_pkthdr.PH_per.eight[0]);
}
+static inline int
+is_tls_offload(struct toepcb *toep)
+{
+ return (toep->ulp_mode == ULP_MODE_TLS);
+}
+
+static inline int
+can_tls_offload(struct adapter *sc)
+{
+ return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS);
+}
+
/* t4_tom.c */
struct toepcb *alloc_toepcb(struct vi_info *, int, int, int);
struct toepcb *hold_toepcb(struct toepcb *);
@@ -327,7 +351,8 @@
uint64_t calc_opt0(struct socket *, struct vi_info *, struct l2t_entry *,
int, int, int, int);
uint64_t select_ntuple(struct vi_info *, struct l2t_entry *);
-void set_tcpddp_ulp_mode(struct toepcb *);
+int select_ulp_mode(struct socket *, struct adapter *);
+void set_ulp_mode(struct toepcb *, int);
int negative_advice(int);
struct clip_entry *hold_lip(struct tom_data *, struct in6_addr *,
struct clip_entry *);
@@ -362,7 +387,10 @@
void send_abort_rpl(struct adapter *, struct sge_wrq *, int , int);
void send_flowc_wr(struct toepcb *, struct flowc_tx_params *);
void send_reset(struct adapter *, struct toepcb *, uint32_t);
+int send_rx_credits(struct adapter *, struct toepcb *, int);
+void send_rx_modulate(struct adapter *, struct toepcb *);
void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t);
+int t4_close_conn(struct adapter *, struct toepcb *);
void t4_rcvd(struct toedev *, struct tcpcb *);
void t4_rcvd_locked(struct toedev *, struct tcpcb *);
int t4_tod_output(struct toedev *, struct tcpcb *);
@@ -400,5 +428,19 @@
void handle_ddp_indicate(struct toepcb *);
void handle_ddp_tcb_rpl(struct toepcb *, const struct cpl_set_tcb_rpl *);
void insert_ddp_data(struct toepcb *, uint32_t);
+
+/* t4_tls.c */
+int t4_ctloutput_tls(struct socket *, struct sockopt *);
+void t4_push_tls_records(struct adapter *, struct toepcb *, int);
+void t4_tls_mod_load(void);
+void t4_tls_mod_unload(void);
+void tls_establish(struct toepcb *);
+void tls_free_kmap(struct tom_data *);
+int tls_init_kmap(struct adapter *, struct tom_data *);
+void tls_init_toep(struct toepcb *);
+int tls_rx_key(struct toepcb *);
+void tls_stop_handshake_timer(struct toepcb *);
+int tls_tx_key(struct toepcb *);
+void tls_uninit_toep(struct toepcb *);
#endif
Index: head/sys/dev/cxgbe/tom/t4_tom.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.c
+++ head/sys/dev/cxgbe/tom/t4_tom.c
@@ -71,6 +71,7 @@
#include "common/t4_tcb.h"
#include "tom/t4_tom_l2t.h"
#include "tom/t4_tom.h"
+#include "tom/t4_tls.h"
static struct protosw toe_protosw;
static struct pr_usrreqs toe_usrreqs;
@@ -199,6 +200,7 @@
if (toep->ulp_mode == ULP_MODE_TCPDDP)
ddp_uninit_toep(toep);
+ tls_uninit_toep(toep);
free(toep, M_CXGBE);
}
@@ -619,12 +621,48 @@
return (htobe64(V_FILTER_TUPLE(ntuple)));
}
+static int
+is_tls_sock(struct socket *so, struct adapter *sc)
+{
+ struct inpcb *inp = sotoinpcb(so);
+ int i, rc;
+
+ /* XXX: Eventually add a SO_WANT_TLS socket option perhaps? */
+ rc = 0;
+ ADAPTER_LOCK(sc);
+ for (i = 0; i < sc->tt.num_tls_rx_ports; i++) {
+ if (inp->inp_lport == htons(sc->tt.tls_rx_ports[i]) ||
+ inp->inp_fport == htons(sc->tt.tls_rx_ports[i])) {
+ rc = 1;
+ break;
+ }
+ }
+ ADAPTER_UNLOCK(sc);
+ return (rc);
+}
+
+int
+select_ulp_mode(struct socket *so, struct adapter *sc)
+{
+
+ if (can_tls_offload(sc) && is_tls_sock(so, sc))
+ return (ULP_MODE_TLS);
+ else if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0)
+ return (ULP_MODE_TCPDDP);
+ else
+ return (ULP_MODE_NONE);
+}
+
void
-set_tcpddp_ulp_mode(struct toepcb *toep)
+set_ulp_mode(struct toepcb *toep, int ulp_mode)
{
- toep->ulp_mode = ULP_MODE_TCPDDP;
- ddp_init_toep(toep);
+ CTR4(KTR_CXGBE, "%s: toep %p (tid %d) ulp_mode %d",
+ __func__, toep, toep->tid, ulp_mode);
+ toep->ulp_mode = ulp_mode;
+ tls_init_toep(toep);
+ if (toep->ulp_mode == ULP_MODE_TCPDDP)
+ ddp_init_toep(toep);
}
int
@@ -959,6 +997,7 @@
KASSERT(td->lctx_count == 0,
("%s: lctx hash table is not empty.", __func__));
+ tls_free_kmap(td);
t4_free_ppod_region(&td->pr);
destroy_clip_table(sc, td);
@@ -1063,6 +1102,12 @@
/* CLIP table for IPv6 offload */
init_clip_table(sc, td);
+ if (sc->vres.key.size != 0) {
+ rc = tls_init_kmap(sc, td);
+ if (rc != 0)
+ goto done;
+ }
+
/* toedev ops */
tod = &td->tod;
init_toedev(tod);
@@ -1168,6 +1213,24 @@
}
static int
+t4_ctloutput_tom(struct socket *so, struct sockopt *sopt)
+{
+
+ if (sopt->sopt_level != IPPROTO_TCP)
+ return (tcp_ctloutput(so, sopt));
+
+ switch (sopt->sopt_name) {
+ case TCP_TLSOM_SET_TLS_CONTEXT:
+ case TCP_TLSOM_GET_TLS_TOM:
+ case TCP_TLSOM_CLR_TLS_TOM:
+ case TCP_TLSOM_CLR_QUIES:
+ return (t4_ctloutput_tls(so, sopt));
+ default:
+ return (tcp_ctloutput(so, sopt));
+ }
+}
+
+static int
t4_tom_mod_load(void)
{
struct protosw *tcp_protosw, *tcp6_protosw;
@@ -1178,6 +1241,7 @@
t4_init_cpl_io_handlers();
t4_ddp_mod_load();
+ t4_tls_mod_load();
tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM);
if (tcp_protosw == NULL)
@@ -1185,6 +1249,7 @@
bcopy(tcp_protosw, &toe_protosw, sizeof(toe_protosw));
bcopy(tcp_protosw->pr_usrreqs, &toe_usrreqs, sizeof(toe_usrreqs));
toe_usrreqs.pru_aio_queue = t4_aio_queue_tom;
+ toe_protosw.pr_ctloutput = t4_ctloutput_tom;
toe_protosw.pr_usrreqs = &toe_usrreqs;
tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM);
@@ -1193,6 +1258,7 @@
bcopy(tcp6_protosw, &toe6_protosw, sizeof(toe6_protosw));
bcopy(tcp6_protosw->pr_usrreqs, &toe6_usrreqs, sizeof(toe6_usrreqs));
toe6_usrreqs.pru_aio_queue = t4_aio_queue_tom;
+ toe6_protosw.pr_ctloutput = t4_ctloutput_tom;
toe6_protosw.pr_usrreqs = &toe6_usrreqs;
TIMEOUT_TASK_INIT(taskqueue_thread, &clip_task, 0, t4_clip_task, NULL);
@@ -1228,6 +1294,7 @@
taskqueue_cancel_timeout(taskqueue_thread, &clip_task, NULL);
}
+ t4_tls_mod_unload();
t4_ddp_mod_unload();
t4_uninit_connect_cpl_handlers();
Index: head/sys/modules/cxgbe/tom/Makefile
===================================================================
--- head/sys/modules/cxgbe/tom/Makefile
+++ head/sys/modules/cxgbe/tom/Makefile
@@ -16,6 +16,7 @@
SRCS+= t4_cpl_io.c
SRCS+= t4_ddp.c
SRCS+= t4_listen.c
+SRCS+= t4_tls.c
SRCS+= t4_tom.c
SRCS+= t4_tom_l2t.c

File Metadata

Mime Type
text/plain
Expires
Sat, Jul 4, 12:09 AM (18 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34642696
Default Alt Text
D14529.diff (88 KB)

Event Timeline