Page MenuHomeFreeBSD
Files F136948663

D21891.id63058.diff
No OneTemporary
Actions

D21891.id63058.diff
View Options

Index: head/sys/kern/kern_sendfile.c
===================================================================
--- head/sys/kern/kern_sendfile.c
+++ head/sys/kern/kern_sendfile.c
@@ -56,6 +56,7 @@
#include <sys/vnode.h>
#include <net/vnet.h>
+#include <netinet/tcp.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
@@ -295,7 +296,7 @@
mb_free_notready(sfio->m, sfio->npages);
#ifdef KERN_TLS
- } else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) {
+ } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
/*
* I/O operation is complete, but we still need to
* encrypt. We cannot do this in the interrupt thread
@@ -1028,7 +1029,7 @@
*/
free(sfio, M_TEMP);
#ifdef KERN_TLS
- if (tls != NULL && tls->sw_encrypt != NULL) {
+ if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);
soref(so);
Index: head/sys/kern/uipc_ktls.c
===================================================================
--- head/sys/kern/uipc_ktls.c
+++ head/sys/kern/uipc_ktls.c
@@ -63,6 +63,9 @@
#include <netinet/in_pcb.h>
#endif
#include <netinet/tcp_var.h>
+#ifdef TCP_OFFLOAD
+#include <netinet/tcp_offload.h>
+#endif
#include <opencrypto/xform.h>
#include <vm/uma_dbg.h>
#include <vm/vm.h>
@@ -161,6 +164,10 @@
"Software TLS session stats");
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0,
"Hardware (ifnet) TLS session stats");
+#ifdef TCP_OFFLOAD
+SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0,
+ "TOE TLS session stats");
+#endif
static counter_u64_t ktls_sw_cbc;
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc,
@@ -199,6 +206,18 @@
&ktls_ifnet_permitted, 1,
"Whether to permit hardware (ifnet) TLS sessions");
+#ifdef TCP_OFFLOAD
+static counter_u64_t ktls_toe_cbc;
+SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD,
+ &ktls_toe_cbc,
+ "Active number of TOE TLS sessions using AES-CBC");
+
+static counter_u64_t ktls_toe_gcm;
+SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD,
+ &ktls_toe_gcm,
+ "Active number of TOE TLS sessions using AES-GCM");
+#endif
+
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS");
static void ktls_cleanup(struct ktls_session *tls);
@@ -325,6 +344,10 @@
ktls_ifnet_reset = counter_u64_alloc(M_WAITOK);
ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK);
ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK);
+#ifdef TCP_OFFLOAD
+ ktls_toe_cbc = counter_u64_alloc(M_WAITOK);
+ ktls_toe_gcm = counter_u64_alloc(M_WAITOK);
+#endif
rm_init(&ktls_backends_lock, "ktls backends");
LIST_INIT(&ktls_backends);
@@ -607,7 +630,8 @@
{
counter_u64_add(ktls_offload_active, -1);
- if (tls->free != NULL) {
+ switch (tls->mode) {
+ case TCP_TLS_MODE_SW:
MPASS(tls->be != NULL);
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
@@ -618,7 +642,8 @@
break;
}
tls->free(tls);
- } else if (tls->snd_tag != NULL) {
+ break;
+ case TCP_TLS_MODE_IFNET:
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_ifnet_cbc, -1);
@@ -628,6 +653,19 @@
break;
}
m_snd_tag_rele(tls->snd_tag);
+ break;
+#ifdef TCP_OFFLOAD
+ case TCP_TLS_MODE_TOE:
+ switch (tls->params.cipher_algorithm) {
+ case CRYPTO_AES_CBC:
+ counter_u64_add(ktls_toe_cbc, -1);
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ counter_u64_add(ktls_toe_gcm, -1);
+ break;
+ }
+ break;
+#endif
}
if (tls->params.auth_key != NULL) {
explicit_bzero(tls->params.auth_key, tls->params.auth_key_len);
@@ -646,6 +684,52 @@
}
#if defined(INET) || defined(INET6)
+
+#ifdef TCP_OFFLOAD
+static int
+ktls_try_toe(struct socket *so, struct ktls_session *tls)
+{
+ struct inpcb *inp;
+ struct tcpcb *tp;
+ int error;
+
+ inp = so->so_pcb;
+ INP_WLOCK(inp);
+ if (inp->inp_flags2 & INP_FREED) {
+ INP_WUNLOCK(inp);
+ return (ECONNRESET);
+ }
+ if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
+ INP_WUNLOCK(inp);
+ return (ECONNRESET);
+ }
+ if (inp->inp_socket == NULL) {
+ INP_WUNLOCK(inp);
+ return (ECONNRESET);
+ }
+ tp = intotcpcb(inp);
+ if (tp->tod == NULL) {
+ INP_WUNLOCK(inp);
+ return (EOPNOTSUPP);
+ }
+
+ error = tcp_offload_alloc_tls_session(tp, tls);
+ INP_WUNLOCK(inp);
+ if (error == 0) {
+ tls->mode = TCP_TLS_MODE_TOE;
+ switch (tls->params.cipher_algorithm) {
+ case CRYPTO_AES_CBC:
+ counter_u64_add(ktls_toe_cbc, 1);
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ counter_u64_add(ktls_toe_gcm, 1);
+ break;
+ }
+ }
+ return (error);
+}
+#endif
+
/*
* Common code used when first enabling ifnet TLS on a connection or
* when allocating a new ifnet TLS session due to a routing change.
@@ -744,6 +828,7 @@
error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
if (error == 0) {
+ tls->mode = TCP_TLS_MODE_IFNET;
tls->snd_tag = mst;
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
@@ -787,6 +872,7 @@
rm_runlock(&ktls_backends_lock, &prio);
if (be == NULL)
return (EOPNOTSUPP);
+ tls->mode = TCP_TLS_MODE_SW;
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_CBC:
counter_u64_add(ktls_sw_cbc, 1);
@@ -834,9 +920,13 @@
if (error)
return (error);
- /* Prefer ifnet TLS over software TLS. */
- error = ktls_try_ifnet(so, tls, false);
+ /* Prefer TOE -> ifnet TLS -> software TLS. */
+#ifdef TCP_OFFLOAD
+ error = ktls_try_toe(so, tls);
if (error)
+#endif
+ error = ktls_try_ifnet(so, tls, false);
+ if (error)
error = ktls_try_sw(so, tls);
if (error) {
@@ -852,7 +942,7 @@
SOCKBUF_LOCK(&so->so_snd);
so->so_snd.sb_tls_info = tls;
- if (tls->sw_encrypt == NULL)
+ if (tls->mode != TCP_TLS_MODE_SW)
so->so_snd.sb_flags |= SB_TLS_IFNET;
SOCKBUF_UNLOCK(&so->so_snd);
sbunlock(&so->so_snd);
@@ -875,10 +965,8 @@
tls = so->so_snd.sb_tls_info;
if (tls == NULL)
mode = TCP_TLS_MODE_NONE;
- else if (tls->sw_encrypt != NULL)
- mode = TCP_TLS_MODE_SW;
else
- mode = TCP_TLS_MODE_IFNET;
+ mode = tls->mode;
SOCKBUF_UNLOCK(&so->so_snd);
return (mode);
}
@@ -893,7 +981,13 @@
struct inpcb *inp;
int error;
- MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET);
+ switch (mode) {
+ case TCP_TLS_MODE_SW:
+ case TCP_TLS_MODE_IFNET:
+ break;
+ default:
+ return (EINVAL);
+ }
inp = so->so_pcb;
INP_WLOCK_ASSERT(inp);
@@ -904,8 +998,7 @@
return (0);
}
- if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) ||
- (tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) {
+ if (tls->mode == mode) {
SOCKBUF_UNLOCK(&so->so_snd);
return (0);
}
@@ -952,7 +1045,7 @@
SOCKBUF_LOCK(&so->so_snd);
so->so_snd.sb_tls_info = tls_new;
- if (tls_new->sw_encrypt == NULL)
+ if (tls_new->mode != TCP_TLS_MODE_SW)
so->so_snd.sb_flags |= SB_TLS_IFNET;
SOCKBUF_UNLOCK(&so->so_snd);
sbunlock(&so->so_snd);
@@ -1238,7 +1331,7 @@
* When using ifnet TLS, unencrypted TLS records are
* sent down the stack to the NIC.
*/
- if (tls->sw_encrypt != NULL) {
+ if (tls->mode == TCP_TLS_MODE_SW) {
m->m_flags |= M_NOTREADY;
pgs->nrdy = pgs->npgs;
*enq_cnt += pgs->npgs;
@@ -1278,7 +1371,7 @@
pgs = m->m_ext.ext_pgs;
- KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf"));
+ KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
pgs->enc_cnt = page_count;
pgs->mbuf = m;
Index: head/sys/kern/uipc_socket.c
===================================================================
--- head/sys/kern/uipc_socket.c
+++ head/sys/kern/uipc_socket.c
@@ -1491,7 +1491,7 @@
tls_pruflag = 0;
tls = ktls_hold(so->so_snd.sb_tls_info);
if (tls != NULL) {
- if (tls->sw_encrypt != NULL)
+ if (tls->mode == TCP_TLS_MODE_SW)
tls_pruflag = PRUS_NOTREADY;
if (control != NULL) {
@@ -1659,7 +1659,7 @@
}
#ifdef KERN_TLS
- if (tls != NULL && tls->sw_encrypt != NULL) {
+ if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
/*
* Note that error is intentionally
* ignored.
Index: head/sys/netinet/tcp.h
===================================================================
--- head/sys/netinet/tcp.h
+++ head/sys/netinet/tcp.h
@@ -357,6 +357,7 @@
#define TCP_TLS_MODE_NONE 0
#define TCP_TLS_MODE_SW 1
#define TCP_TLS_MODE_IFNET 2
+#define TCP_TLS_MODE_TOE 3
/*
* TCP Control message types
Index: head/sys/netinet/tcp_offload.h
===================================================================
--- head/sys/netinet/tcp_offload.h
+++ head/sys/netinet/tcp_offload.h
@@ -46,6 +46,7 @@
void tcp_offload_rcvd(struct tcpcb *);
void tcp_offload_ctloutput(struct tcpcb *, int, int);
void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
+int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
void tcp_offload_detach(struct tcpcb *);
#endif
Index: head/sys/netinet/tcp_offload.c
===================================================================
--- head/sys/netinet/tcp_offload.c
+++ head/sys/netinet/tcp_offload.c
@@ -178,6 +178,17 @@
tod->tod_tcp_info(tod, tp, ti);
}
+int
+tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
+{
+ struct toedev *tod = tp->tod;
+
+ KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
+ INP_WLOCK_ASSERT(tp->t_inpcb);
+
+ return (tod->tod_alloc_tls_session(tod, tp, tls));
+}
+
void
tcp_offload_detach(struct tcpcb *tp)
{
Index: head/sys/netinet/tcp_usrreq.c
===================================================================
--- head/sys/netinet/tcp_usrreq.c
+++ head/sys/netinet/tcp_usrreq.c
@@ -1936,8 +1936,6 @@
error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
if (error)
return (error);
- if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET)
- return (EINVAL);
INP_WLOCK_RECHECK(inp);
error = ktls_set_tx_mode(so, ui);
Index: head/sys/netinet/toecore.h
===================================================================
--- head/sys/netinet/toecore.h
+++ head/sys/netinet/toecore.h
@@ -41,6 +41,7 @@
struct tcphdr;
struct in_conninfo;
struct tcp_info;
+struct ktls_session;
struct toedev {
TAILQ_ENTRY(toedev) link; /* glue for toedev_list */
@@ -108,6 +109,10 @@
/* Update software state */
void (*tod_tcp_info)(struct toedev *, struct tcpcb *,
struct tcp_info *);
+
+ /* Create a TLS session */
+ int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
+ struct ktls_session *);
};
typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);
Index: head/sys/netinet/toecore.c
===================================================================
--- head/sys/netinet/toecore.c
+++ head/sys/netinet/toecore.c
@@ -191,6 +191,14 @@
return;
}
+static int
+toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
+ struct ktls_session *tls __unused)
+{
+
+ return (EINVAL);
+}
+
/*
* Inform one or more TOE devices about a listening socket.
*/
@@ -281,6 +289,7 @@
tod->tod_offload_socket = toedev_offload_socket;
tod->tod_ctloutput = toedev_ctloutput;
tod->tod_tcp_info = toedev_tcp_info;
+ tod->tod_alloc_tls_session = toedev_alloc_tls_session;
}
/*
Index: head/sys/sys/ktls.h
===================================================================
--- head/sys/sys/ktls.h
+++ head/sys/sys/ktls.h
@@ -156,6 +156,7 @@
struct tls_session_params params;
u_int wq_index;
volatile u_int refcount;
+ int mode;
struct task reset_tag_task;
struct inpcb *inp;

File Metadata

Mime Type
text/plain
Expires
Fri, Nov 21, 7:31 PM (56 m, 24 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
25790865
Default Alt Text
D21891.id63058.diff (11 KB)

Event Timeline