Changeset View
Standalone View
sys/kern/uipc_ktls.c
Show All 24 Lines | ||||||||||||
* SUCH DAMAGE. | * SUCH DAMAGE. | |||||||||||
*/ | */ | |||||||||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | |||||||||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | |||||||||||
#include "opt_inet.h" | #include "opt_inet.h" | |||||||||||
#include "opt_inet6.h" | #include "opt_inet6.h" | |||||||||||
#include "opt_kern_tls.h" | ||||||||||||
#include "opt_ratelimit.h" | #include "opt_ratelimit.h" | |||||||||||
#include "opt_rss.h" | #include "opt_rss.h" | |||||||||||
#include <sys/param.h> | #include <sys/param.h> | |||||||||||
#include <sys/kernel.h> | #include <sys/kernel.h> | |||||||||||
#include <sys/domainset.h> | #include <sys/domainset.h> | |||||||||||
#include <sys/ktls.h> | #include <sys/ktls.h> | |||||||||||
#include <sys/lock.h> | #include <sys/lock.h> | |||||||||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | ||||||||||||
SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, | SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, | |||||||||||
&ktls_maxlen, 0, "Maximum TLS record size"); | &ktls_maxlen, 0, "Maximum TLS record size"); | |||||||||||
static int ktls_number_threads; | static int ktls_number_threads; | |||||||||||
SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, | SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, | |||||||||||
&ktls_number_threads, 0, | &ktls_number_threads, 0, | |||||||||||
"Number of TLS threads in thread-pool"); | "Number of TLS threads in thread-pool"); | |||||||||||
int ktls_ifnet_max_rexmit = 2; | ||||||||||||
markj: IMHO the name could be more descriptive, like `ktls_hw_ifnet_max_rexmit_pct`. That might be a… | ||||||||||||
Done Inline ActionsProbably want to make this unsigned. hselasky: Probably want to make this unsigned. | ||||||||||||
Not Done Inline ActionsMissing "static" here? hselasky: Missing "static" here? | ||||||||||||
Not Done Inline ActionsNo, this need to be global. tcp_account_for_send() accesses it. This is why it is in sys/ktls.h gallatin: No, this need to be global. tcp_account_for_send() accesses it. This is why it is in… | ||||||||||||
SYSCTL_INT(_kern_ipc_tls, OID_AUTO, ifnet_max_rexmit, CTLFLAG_RWTUN, | ||||||||||||
&ktls_ifnet_max_rexmit, 2, | ||||||||||||
"Max percent bytes retransmitted before ifnet TLS is disabled"); | ||||||||||||
static bool ktls_offload_enable; | static bool ktls_offload_enable; | |||||||||||
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, | SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, | |||||||||||
&ktls_offload_enable, 0, | &ktls_offload_enable, 0, | |||||||||||
"Enable support for kernel TLS offload"); | "Enable support for kernel TLS offload"); | |||||||||||
static bool ktls_cbc_enable = true; | static bool ktls_cbc_enable = true; | |||||||||||
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, | SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, | |||||||||||
&ktls_cbc_enable, 1, | &ktls_cbc_enable, 1, | |||||||||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | ||||||||||||
static COUNTER_U64_DEFINE_EARLY(ktls_switch_to_sw); | static COUNTER_U64_DEFINE_EARLY(ktls_switch_to_sw); | |||||||||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_sw, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_sw, CTLFLAG_RD, | |||||||||||
&ktls_switch_to_sw, "TLS sessions switched from ifnet to SW"); | &ktls_switch_to_sw, "TLS sessions switched from ifnet to SW"); | |||||||||||
static COUNTER_U64_DEFINE_EARLY(ktls_switch_failed); | static COUNTER_U64_DEFINE_EARLY(ktls_switch_failed); | |||||||||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_failed, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_failed, CTLFLAG_RD, | |||||||||||
&ktls_switch_failed, "TLS sessions unable to switch between SW and ifnet"); | &ktls_switch_failed, "TLS sessions unable to switch between SW and ifnet"); | |||||||||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_disable_fail); | ||||||||||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_failed, CTLFLAG_RD, | ||||||||||||
&ktls_ifnet_disable_fail, "TLS sessions unable to switch to SW from ifnet"); | ||||||||||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_disable_ok); | ||||||||||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_ok, CTLFLAG_RD, | ||||||||||||
&ktls_ifnet_disable_ok, "TLS sessions able to switch to SW from ifnet"); | ||||||||||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | |||||||||||
"Software TLS session stats"); | "Software TLS session stats"); | |||||||||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | |||||||||||
"Hardware (ifnet) TLS session stats"); | "Hardware (ifnet) TLS session stats"); | |||||||||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | |||||||||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | |||||||||||
"TOE TLS session stats"); | "TOE TLS session stats"); | |||||||||||
#endif | #endif | |||||||||||
▲ Show 20 Lines • Show All 1,986 Lines • ▼ Show 20 Lines | STAILQ_FOREACH_SAFE(m, &local_m_head, m_epg_stailq, n) { | |||||||||||
} | } | |||||||||||
} | } | |||||||||||
STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) { | STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) { | |||||||||||
ktls_decrypt(so); | ktls_decrypt(so); | |||||||||||
counter_u64_add(ktls_cnt_rx_queued, -1); | counter_u64_add(ktls_cnt_rx_queued, -1); | |||||||||||
} | } | |||||||||||
} | } | |||||||||||
} | ||||||||||||
hselaskyUnsubmitted Done Inline ActionsExtra space here. hselasky: Extra space here. | ||||||||||||
static void | ||||||||||||
Done Inline Actions
markj: | ||||||||||||
ktls_disable_ifnet_help(void *context, int pending) | ||||||||||||
{ | ||||||||||||
struct ktls_session *tls; | ||||||||||||
struct inpcb *inp; | ||||||||||||
struct tcpcb *tp; | ||||||||||||
struct socket *so; | ||||||||||||
int err; | ||||||||||||
MPASS(pending == 1); | ||||||||||||
tls = context; | ||||||||||||
inp = tls->inp; | ||||||||||||
if (inp == NULL) | ||||||||||||
return; | ||||||||||||
INP_WLOCK(inp); | ||||||||||||
if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || | ||||||||||||
(inp->inp_flags2 & INP_FREED)) { | ||||||||||||
Done Inline ActionsThe pcb is only detached when the socket is about to be freed, but the socket reference taken earlier prevents that. So I think this should assert inp_socket != NULL instead. Otherwise it looks like the sock ref is leaked, but it isn't, unless I'm missing something. markj: The pcb is only detached when the socket is about to be freed, but the socket reference taken… | ||||||||||||
Done Inline ActionsYes, I see what you mean. I've replaced the check with an assert. This also simplifies the error handling, as a separate label is no longer needed. gallatin: Yes, I see what you mean. I've replaced the check with an assert. This also simplifies the… | ||||||||||||
if (!in_pcbrele_wlocked(inp)) | ||||||||||||
INP_WUNLOCK(inp); | ||||||||||||
markjUnsubmitted Done Inline ActionsIs it possible for the sock ref to be leaked here? markj: Is it possible for the sock ref to be leaked here? | ||||||||||||
gallatinAuthorUnsubmitted Done Inline ActionsGood catch. I've re-structured things to fix that. gallatin: Good catch. I've re-structured things to fix that. | ||||||||||||
return; | ||||||||||||
} | ||||||||||||
so = inp->inp_socket; | ||||||||||||
if (so != NULL && so->so_snd.sb_tls_info != NULL) | ||||||||||||
err = ktls_set_tx_mode(so, TCP_TLS_MODE_SW); | ||||||||||||
else | ||||||||||||
err = ENXIO; | ||||||||||||
if (err == 0) { | ||||||||||||
counter_u64_add(ktls_ifnet_disable_ok, 1); | ||||||||||||
tp = intotcpcb(inp); | ||||||||||||
if (tp != NULL && | ||||||||||||
tp->t_fb->tfb_hwtls_change != NULL) | ||||||||||||
(*tp->t_fb->tfb_hwtls_change)(tp, 0); | ||||||||||||
hselaskyUnsubmitted Done Inline ActionsEmpty line here can be removed. hselasky: Empty line here can be removed. | ||||||||||||
} else { | ||||||||||||
counter_u64_add(ktls_ifnet_disable_fail, 1); | ||||||||||||
} | ||||||||||||
ktls_free(tls); | ||||||||||||
SOCK_LOCK(so); | ||||||||||||
markjUnsubmitted Done Inline ActionsAbove we handle the case so == NULL but here we dereference so unconditionally. markj: Above we handle the case `so == NULL` but here we dereference `so` unconditionally. | ||||||||||||
sorele(so); | ||||||||||||
if (!in_pcbrele_wlocked(inp)) | ||||||||||||
INP_WUNLOCK(inp); | ||||||||||||
} | ||||||||||||
/* | ||||||||||||
* Called when re-transmits are becoming a substantial portion of the | ||||||||||||
* sends on this connection. When this happens, we transition the | ||||||||||||
* connection to software TLS. | ||||||||||||
markjUnsubmitted Done Inline ActionsIt might be worth adding an explanation of exactly why it's beneficial to switch. markj: It might be worth adding an explanation of exactly why it's beneficial to switch. | ||||||||||||
*/ | ||||||||||||
void ktls_disable_ifnet(void *arg) | ||||||||||||
{ | ||||||||||||
struct tcpcb *tp; | ||||||||||||
struct inpcb *inp; | ||||||||||||
struct socket *so; | ||||||||||||
struct ktls_session *tls; | ||||||||||||
tp = arg; | ||||||||||||
inp = tp->t_inpcb; | ||||||||||||
Done Inline Actions
markj: | ||||||||||||
INP_WLOCK_ASSERT(inp); | ||||||||||||
so = inp->inp_socket; | ||||||||||||
SOCK_UNLOCK_ASSERT(so); | ||||||||||||
markjUnsubmitted Done Inline ActionsSOCK_LOCK will assert if the socket lock is already held. markj: SOCK_LOCK will assert if the socket lock is already held. | ||||||||||||
gallatinAuthorUnsubmitted Done Inline ActionsThanks; removed gallatin: Thanks; removed | ||||||||||||
SOCK_LOCK(so); | ||||||||||||
tls = so->so_snd.sb_tls_info; | ||||||||||||
if (tls->disable_ifnet_pending) { | ||||||||||||
SOCK_UNLOCK(so); | ||||||||||||
return; | ||||||||||||
} | ||||||||||||
/* | ||||||||||||
* note that disable_ifnet_pending is never cleared; disabling | ||||||||||||
* ifnet can only be done once per session, so we never want | ||||||||||||
* to do it again | ||||||||||||
*/ | ||||||||||||
(void) ktls_hold(tls); | ||||||||||||
in_pcbref(inp); | ||||||||||||
soref(so); | ||||||||||||
tls->disable_ifnet_pending = true; | ||||||||||||
tls->inp = inp; | ||||||||||||
SOCK_UNLOCK(so); | ||||||||||||
TASK_INIT(&tls->disable_ifnet_task, 0, ktls_disable_ifnet_help, tls); | ||||||||||||
taskqueue_enqueue(taskqueue_thread, &tls->disable_ifnet_task); | ||||||||||||
Done Inline Actions
markj: | ||||||||||||
} | } | |||||||||||
Not Done Inline Actions
markj: |
IMHO the name could be more descriptive, like ktls_hw_ifnet_max_rexmit_pct. That might be a bit long, but I'd at least add _pct.