Index: head/sys/kern/kern_sendfile.c =================================================================== --- head/sys/kern/kern_sendfile.c +++ head/sys/kern/kern_sendfile.c @@ -56,6 +56,7 @@ #include #include +#include #include #include @@ -295,7 +296,7 @@ mb_free_notready(sfio->m, sfio->npages); #ifdef KERN_TLS - } else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) { + } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) { /* * I/O operation is complete, but we still need to * encrypt. We cannot do this in the interrupt thread @@ -1028,7 +1029,7 @@ */ free(sfio, M_TEMP); #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { error = (*so->so_proto->pr_usrreqs->pru_send) (so, PRUS_NOTREADY, m, NULL, NULL, td); soref(so); Index: head/sys/kern/uipc_ktls.c =================================================================== --- head/sys/kern/uipc_ktls.c +++ head/sys/kern/uipc_ktls.c @@ -63,6 +63,9 @@ #include #endif #include +#ifdef TCP_OFFLOAD +#include +#endif #include #include #include @@ -161,6 +164,10 @@ "Software TLS session stats"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0, "Hardware (ifnet) TLS session stats"); +#ifdef TCP_OFFLOAD +SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0, + "TOE TLS session stats"); +#endif static counter_u64_t ktls_sw_cbc; SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, @@ -199,6 +206,18 @@ &ktls_ifnet_permitted, 1, "Whether to permit hardware (ifnet) TLS sessions"); +#ifdef TCP_OFFLOAD +static counter_u64_t ktls_toe_cbc; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, + &ktls_toe_cbc, + "Active number of TOE TLS sessions using AES-CBC"); + +static counter_u64_t ktls_toe_gcm; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, + &ktls_toe_gcm, + "Active number of TOE TLS sessions using AES-GCM"); +#endif + static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); static void ktls_cleanup(struct ktls_session *tls); @@ -325,6 +344,10 @@ ktls_ifnet_reset = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK); +#ifdef TCP_OFFLOAD + ktls_toe_cbc = counter_u64_alloc(M_WAITOK); + ktls_toe_gcm = counter_u64_alloc(M_WAITOK); +#endif rm_init(&ktls_backends_lock, "ktls backends"); LIST_INIT(&ktls_backends); @@ -607,7 +630,8 @@ { counter_u64_add(ktls_offload_active, -1); - if (tls->free != NULL) { + switch (tls->mode) { + case TCP_TLS_MODE_SW: MPASS(tls->be != NULL); switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -618,7 +642,8 @@ break; } tls->free(tls); - } else if (tls->snd_tag != NULL) { + break; + case TCP_TLS_MODE_IFNET: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_ifnet_cbc, -1); @@ -628,6 +653,19 @@ break; } m_snd_tag_rele(tls->snd_tag); + break; +#ifdef TCP_OFFLOAD + case TCP_TLS_MODE_TOE: + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, -1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, -1); + break; + } + break; +#endif } if (tls->params.auth_key != NULL) { explicit_bzero(tls->params.auth_key, tls->params.auth_key_len); @@ -646,6 +684,52 @@ } #if defined(INET) || defined(INET6) + +#ifdef TCP_OFFLOAD +static int +ktls_try_toe(struct socket *so, struct ktls_session *tls) +{ + struct inpcb *inp; + struct tcpcb *tp; + int error; + + inp = so->so_pcb; + INP_WLOCK(inp); + if (inp->inp_flags2 & INP_FREED) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_socket == NULL) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + tp = intotcpcb(inp); + if (tp->tod == NULL) { + INP_WUNLOCK(inp); + return (EOPNOTSUPP); + } + + error = tcp_offload_alloc_tls_session(tp, tls); + INP_WUNLOCK(inp); + if (error == 0) { + tls->mode = TCP_TLS_MODE_TOE; + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, 1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, 1); + break; + } + } + return (error); +} +#endif + /* * Common code used when first enabling ifnet TLS on a connection or * when allocating a new ifnet TLS session due to a routing change. @@ -744,6 +828,7 @@ error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); if (error == 0) { + tls->mode = TCP_TLS_MODE_IFNET; tls->snd_tag = mst; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -787,6 +872,7 @@ rm_runlock(&ktls_backends_lock, &prio); if (be == NULL) return (EOPNOTSUPP); + tls->mode = TCP_TLS_MODE_SW; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_sw_cbc, 1); @@ -834,9 +920,13 @@ if (error) return (error); - /* Prefer ifnet TLS over software TLS. */ - error = ktls_try_ifnet(so, tls, false); + /* Prefer TOE -> ifnet TLS -> software TLS. */ +#ifdef TCP_OFFLOAD + error = ktls_try_toe(so, tls); if (error) +#endif + error = ktls_try_ifnet(so, tls, false); + if (error) error = ktls_try_sw(so, tls); if (error) { @@ -852,7 +942,7 @@ SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls; - if (tls->sw_encrypt == NULL) + if (tls->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -875,10 +965,8 @@ tls = so->so_snd.sb_tls_info; if (tls == NULL) mode = TCP_TLS_MODE_NONE; - else if (tls->sw_encrypt != NULL) - mode = TCP_TLS_MODE_SW; else - mode = TCP_TLS_MODE_IFNET; + mode = tls->mode; SOCKBUF_UNLOCK(&so->so_snd); return (mode); } @@ -893,7 +981,13 @@ struct inpcb *inp; int error; - MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET); + switch (mode) { + case TCP_TLS_MODE_SW: + case TCP_TLS_MODE_IFNET: + break; + default: + return (EINVAL); + } inp = so->so_pcb; INP_WLOCK_ASSERT(inp); @@ -904,8 +998,7 @@ return (0); } - if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) || - (tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) { + if (tls->mode == mode) { SOCKBUF_UNLOCK(&so->so_snd); return (0); } @@ -952,7 +1045,7 @@ SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls_new; - if (tls_new->sw_encrypt == NULL) + if (tls_new->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -1238,7 +1331,7 @@ * When using ifnet TLS, unencrypted TLS records are * sent down the stack to the NIC. */ - if (tls->sw_encrypt != NULL) { + if (tls->mode == TCP_TLS_MODE_SW) { m->m_flags |= M_NOTREADY; pgs->nrdy = pgs->npgs; *enq_cnt += pgs->npgs; @@ -1278,7 +1371,7 @@ pgs = m->m_ext.ext_pgs; - KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf")); + KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf")); pgs->enc_cnt = page_count; pgs->mbuf = m; Index: head/sys/kern/uipc_socket.c =================================================================== --- head/sys/kern/uipc_socket.c +++ head/sys/kern/uipc_socket.c @@ -1491,7 +1491,7 @@ tls_pruflag = 0; tls = ktls_hold(so->so_snd.sb_tls_info); if (tls != NULL) { - if (tls->sw_encrypt != NULL) + if (tls->mode == TCP_TLS_MODE_SW) tls_pruflag = PRUS_NOTREADY; if (control != NULL) { @@ -1659,7 +1659,7 @@ } #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { /* * Note that error is intentionally * ignored. Index: head/sys/netinet/tcp.h =================================================================== --- head/sys/netinet/tcp.h +++ head/sys/netinet/tcp.h @@ -357,6 +357,7 @@ #define TCP_TLS_MODE_NONE 0 #define TCP_TLS_MODE_SW 1 #define TCP_TLS_MODE_IFNET 2 +#define TCP_TLS_MODE_TOE 3 /* * TCP Control message types Index: head/sys/netinet/tcp_offload.h =================================================================== --- head/sys/netinet/tcp_offload.h +++ head/sys/netinet/tcp_offload.h @@ -46,6 +46,7 @@ void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); +int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *); void tcp_offload_detach(struct tcpcb *); #endif Index: head/sys/netinet/tcp_offload.c =================================================================== --- head/sys/netinet/tcp_offload.c +++ head/sys/netinet/tcp_offload.c @@ -178,6 +178,17 @@ tod->tod_tcp_info(tod, tp, ti); } +int +tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + return (tod->tod_alloc_tls_session(tod, tp, tls)); +} + void tcp_offload_detach(struct tcpcb *tp) { Index: head/sys/netinet/tcp_usrreq.c =================================================================== --- head/sys/netinet/tcp_usrreq.c +++ head/sys/netinet/tcp_usrreq.c @@ -1936,8 +1936,6 @@ error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui)); if (error) return (error); - if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET) - return (EINVAL); INP_WLOCK_RECHECK(inp); error = ktls_set_tx_mode(so, ui); Index: head/sys/netinet/toecore.h =================================================================== --- head/sys/netinet/toecore.h +++ head/sys/netinet/toecore.h @@ -41,6 +41,7 @@ struct tcphdr; struct in_conninfo; struct tcp_info; +struct ktls_session; struct toedev { TAILQ_ENTRY(toedev) link; /* glue for toedev_list */ @@ -108,6 +109,10 @@ /* Update software state */ void (*tod_tcp_info)(struct toedev *, struct tcpcb *, struct tcp_info *); + + /* Create a TLS session */ + int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *, + struct ktls_session *); }; typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); Index: head/sys/netinet/toecore.c =================================================================== --- head/sys/netinet/toecore.c +++ head/sys/netinet/toecore.c @@ -191,6 +191,14 @@ return; } +static int +toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, + struct ktls_session *tls __unused) +{ + + return (EINVAL); +} + /* * Inform one or more TOE devices about a listening socket. */ @@ -281,6 +289,7 @@ tod->tod_offload_socket = toedev_offload_socket; tod->tod_ctloutput = toedev_ctloutput; tod->tod_tcp_info = toedev_tcp_info; + tod->tod_alloc_tls_session = toedev_alloc_tls_session; } /* Index: head/sys/sys/ktls.h =================================================================== --- head/sys/sys/ktls.h +++ head/sys/sys/ktls.h @@ -156,6 +156,7 @@ struct tls_session_params params; u_int wq_index; volatile u_int refcount; + int mode; struct task reset_tag_task; struct inpcb *inp;