Changeset View
Standalone View
sys/kern/uipc_ktls.c
Show First 20 Lines • Show All 293 Lines • ▼ Show 20 Lines | SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, chacha20, CTLFLAG_RD, | ||||
&ktls_toe_chacha20, | &ktls_toe_chacha20, | ||||
"Active number of TOE TLS sessions using Chacha20-Poly1305"); | "Active number of TOE TLS sessions using Chacha20-Poly1305"); | ||||
#endif | #endif | ||||
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); | static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); | ||||
static void ktls_cleanup(struct ktls_session *tls); | static void ktls_cleanup(struct ktls_session *tls); | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
static void ktls_reset_send_tag(void *context, int pending); | static void ktls_reset_send_receive_tag(void *context, int pending); | ||||
#endif | #endif | ||||
static void ktls_work_thread(void *ctx); | static void ktls_work_thread(void *ctx); | ||||
static void ktls_alloc_thread(void *ctx); | static void ktls_alloc_thread(void *ctx); | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
static u_int | static u_int | ||||
ktls_get_cpu(struct socket *so) | ktls_get_cpu(struct socket *so) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 187 Lines • ▼ Show 20 Lines | start: | ||||
atomic_store_rel_int(&ktls_init_state, state); | atomic_store_rel_int(&ktls_init_state, state); | ||||
sx_xunlock(&ktls_init_lock); | sx_xunlock(&ktls_init_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
static int | static int | ||||
ktls_create_session(struct socket *so, struct tls_enable *en, | ktls_create_session(struct socket *so, struct tls_enable *en, | ||||
struct ktls_session **tlsp) | struct ktls_session **tlsp, int direction) | ||||
{ | { | ||||
struct ktls_session *tls; | struct ktls_session *tls; | ||||
int error; | int error; | ||||
/* Only TLS 1.0 - 1.3 are supported. */ | /* Only TLS 1.0 - 1.3 are supported. */ | ||||
if (en->tls_vmajor != TLS_MAJOR_VER_ONE) | if (en->tls_vmajor != TLS_MAJOR_VER_ONE) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (en->tls_vminor < TLS_MINOR_VER_ZERO || | if (en->tls_vminor < TLS_MINOR_VER_ZERO || | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | #endif | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | ||||
counter_u64_add(ktls_offload_active, 1); | counter_u64_add(ktls_offload_active, 1); | ||||
refcount_init(&tls->refcount, 1); | refcount_init(&tls->refcount, 1); | ||||
TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls); | TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_receive_tag, tls); | ||||
tls->wq_index = ktls_get_cpu(so); | tls->wq_index = ktls_get_cpu(so); | ||||
tls->direction = direction; | |||||
tls->params.cipher_algorithm = en->cipher_algorithm; | tls->params.cipher_algorithm = en->cipher_algorithm; | ||||
tls->params.auth_algorithm = en->auth_algorithm; | tls->params.auth_algorithm = en->auth_algorithm; | ||||
tls->params.tls_vmajor = en->tls_vmajor; | tls->params.tls_vmajor = en->tls_vmajor; | ||||
tls->params.tls_vminor = en->tls_vminor; | tls->params.tls_vminor = en->tls_vminor; | ||||
tls->params.flags = en->flags; | tls->params.flags = en->flags; | ||||
tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen); | tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen); | ||||
▲ Show 20 Lines • Show All 116 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct ktls_session *tls_new; | struct ktls_session *tls_new; | ||||
tls_new = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | tls_new = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | ||||
counter_u64_add(ktls_offload_active, 1); | counter_u64_add(ktls_offload_active, 1); | ||||
refcount_init(&tls_new->refcount, 1); | refcount_init(&tls_new->refcount, 1); | ||||
TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new); | TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_receive_tag, tls_new); | ||||
/* Copy fields from existing session. */ | /* Copy fields from existing session. */ | ||||
tls_new->params = tls->params; | tls_new->params = tls->params; | ||||
tls_new->wq_index = tls->wq_index; | tls_new->wq_index = tls->wq_index; | ||||
tls_new->direction = tls->direction; | |||||
/* Deep copy keys. */ | /* Deep copy keys. */ | ||||
if (tls_new->params.auth_key != NULL) { | if (tls_new->params.auth_key != NULL) { | ||||
tls_new->params.auth_key = malloc(tls->params.auth_key_len, | tls_new->params.auth_key = malloc(tls->params.auth_key_len, | ||||
M_KTLS, M_WAITOK); | M_KTLS, M_WAITOK); | ||||
memcpy(tls_new->params.auth_key, tls->params.auth_key, | memcpy(tls_new->params.auth_key, tls->params.auth_key, | ||||
tls->params.auth_key_len); | tls->params.auth_key_len); | ||||
} | } | ||||
Show All 33 Lines | case CRYPTO_AES_CBC: | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_ifnet_gcm, -1); | counter_u64_add(ktls_ifnet_gcm, -1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | case CRYPTO_CHACHA20_POLY1305: | ||||
counter_u64_add(ktls_ifnet_chacha20, -1); | counter_u64_add(ktls_ifnet_chacha20, -1); | ||||
break; | break; | ||||
} | } | ||||
if (tls->snd_tag != NULL) | if (tls->snd_rcv_tag != NULL) | ||||
m_snd_tag_rele(tls->snd_tag); | m_snd_tag_rele(tls->snd_rcv_tag); | ||||
break; | break; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
case TCP_TLS_MODE_TOE: | case TCP_TLS_MODE_TOE: | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_toe_cbc, -1); | counter_u64_add(ktls_toe_cbc, -1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
▲ Show 20 Lines • Show All 165 Lines • ▼ Show 20 Lines | if (inp->inp_vflag & INP_IPV6) { | ||||
} | } | ||||
} | } | ||||
error = m_snd_tag_alloc(ifp, ¶ms, mstp); | error = m_snd_tag_alloc(ifp, ¶ms, mstp); | ||||
out: | out: | ||||
if_rele(ifp); | if_rele(ifp); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | |||||
* Common code for allocating a TLS receive tag for doing HW | |||||
* decryption of TLS data. | |||||
* | |||||
* This function allocates a new TLS receive tag on whatever interface | |||||
* the connection is currently routed over. | |||||
*/ | |||||
static int | static int | ||||
ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force) | ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls, bool force, | ||||
struct m_snd_tag **mstp) | |||||
{ | { | ||||
union if_snd_tag_alloc_params params; | |||||
struct ifnet *ifp; | |||||
struct nhop_object *nh; | |||||
struct tcpcb *tp; | |||||
int error; | |||||
INP_RLOCK(inp); | |||||
if (inp->inp_flags2 & INP_FREED) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
if (inp->inp_socket == NULL) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
tp = intotcpcb(inp); | |||||
/* | |||||
* Check administrative controls on ifnet TLS to determine if | |||||
* ifnet TLS should be denied. | |||||
* | |||||
* - Always permit 'force' requests. | |||||
* - ktls_ifnet_permitted == 0: always deny. | |||||
*/ | |||||
if (!force && ktls_ifnet_permitted == 0) { | |||||
INP_RUNLOCK(inp); | |||||
return (ENXIO); | |||||
} | |||||
/* | |||||
* XXX: Use the cached route in the inpcb to find the | |||||
* interface. This should perhaps instead use | |||||
* rtalloc1_fib(dst, 0, 0, fibnum). Since KTLS is only | |||||
* enabled after a connection has completed key negotiation in | |||||
* userland, the cached route will be present in practice. | |||||
*/ | |||||
nh = inp->inp_route.ro_nh; | |||||
if (nh == NULL) { | |||||
INP_RUNLOCK(inp); | |||||
return (ENXIO); | |||||
} | |||||
ifp = nh->nh_ifp; | |||||
melifaro: If we’re running under network epoch, this is not necessary. Nhops are guaranteed to exists… | |||||
Done Inline ActionsI'll have a look at this. Thank you for your comment! hselasky: I'll have a look at this. Thank you for your comment! | |||||
Done Inline Actionsping? :-) melifaro: ping? :-) | |||||
Done Inline ActionsYou suggest putting the following piece into an EPOCH section? nh = inp->inp_route.ro_nh; if (nh == NULL) { INP_RUNLOCK(inp); return (ENXIO); } ifp = nh->nh_ifp; What about the NULL check? Is it safe to skip then? hselasky: You suggest putting the following piece into an EPOCH section?
```
nh = inp->inp_route.ro_nh… | |||||
Not Done Inline Actions
I'd say wider than that - all code block which uses ifp, which is entire function.
IIRC it is possible that routing change happened, we invalidated the cache in the datapath, but didn't get the result, ending with NULL there. Normally the socket would die after that, but to avoid the implicit dependency on the datapath logic, I'd keep it (probably making with __predict_false). melifaro: > You suggest putting the following piece into an EPOCH section?
I'd say wider than that - all… | |||||
if_ref(ifp); | |||||
params.hdr.type = IF_SND_TAG_TYPE_TLS_RX; | |||||
params.hdr.flowid = inp->inp_flowid; | |||||
params.hdr.flowtype = inp->inp_flowtype; | |||||
params.hdr.numa_domain = inp->inp_numa_domain; | |||||
params.tls_rx.inp = inp; | |||||
params.tls_rx.tls = tls; | |||||
params.tls_rx.vlan_id = 0; | |||||
INP_RUNLOCK(inp); | |||||
if ((ifp->if_capenable & IFCAP_MEXTPG) == 0) { | |||||
error = EOPNOTSUPP; | |||||
goto out; | |||||
} | |||||
/* XXX reusing TXTLS flags */ | |||||
Done Inline ActionsThis check is racy. Is that ok? markj: This check is racy. Is that ok? | |||||
Done Inline ActionsYes, that's OK for now. We also need to implement the RXTLS version of these flags! kib@ is working on that. hselasky: Yes, that's OK for now. We also need to implement the RXTLS version of these flags!
kib@ is… | |||||
if (inp->inp_vflag & INP_IPV6) { | |||||
if ((ifp->if_capenable & IFCAP_TXTLS6) == 0) { | |||||
error = EOPNOTSUPP; | |||||
goto out; | |||||
} | |||||
} else { | |||||
if ((ifp->if_capenable & IFCAP_TXTLS4) == 0) { | |||||
error = EOPNOTSUPP; | |||||
goto out; | |||||
} | |||||
} | |||||
error = m_snd_tag_alloc(ifp, ¶ms, mstp); | |||||
out: | |||||
if_rele(ifp); | |||||
return (error); | |||||
} | |||||
static int | |||||
ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction, bool force) | |||||
{ | |||||
struct m_snd_tag *mst; | struct m_snd_tag *mst; | ||||
int error; | int error; | ||||
switch (direction) { | |||||
case KTLS_TX: | |||||
error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); | error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); | ||||
if (error == 0) { | if (__predict_false(error != 0)) | ||||
goto done; | |||||
break; | |||||
case KTLS_RX: | |||||
error = ktls_alloc_rcv_tag(so->so_pcb, tls, force, &mst); | |||||
if (__predict_false(error != 0)) | |||||
goto done; | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
tls->mode = TCP_TLS_MODE_IFNET; | tls->mode = TCP_TLS_MODE_IFNET; | ||||
tls->snd_tag = mst; | tls->snd_rcv_tag = mst; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_ifnet_cbc, 1); | counter_u64_add(ktls_ifnet_cbc, 1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_ifnet_gcm, 1); | counter_u64_add(ktls_ifnet_gcm, 1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | case CRYPTO_CHACHA20_POLY1305: | ||||
counter_u64_add(ktls_ifnet_chacha20, 1); | counter_u64_add(ktls_ifnet_chacha20, 1); | ||||
break; | break; | ||||
default: | |||||
break; | |||||
} | } | ||||
} | done: | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
ktls_use_sw(struct ktls_session *tls) | ktls_use_sw(struct ktls_session *tls) | ||||
{ | { | ||||
tls->mode = TCP_TLS_MODE_SW; | tls->mode = TCP_TLS_MODE_SW; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines | ktls_enable_rx(struct socket *so, struct tls_enable *en) | ||||
if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) | if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) | ||||
return (ENOTSUP); | return (ENOTSUP); | ||||
/* TLS 1.3 is not yet supported. */ | /* TLS 1.3 is not yet supported. */ | ||||
if (en->tls_vmajor == TLS_MAJOR_VER_ONE && | if (en->tls_vmajor == TLS_MAJOR_VER_ONE && | ||||
en->tls_vminor == TLS_MINOR_VER_THREE) | en->tls_vminor == TLS_MINOR_VER_THREE) | ||||
return (ENOTSUP); | return (ENOTSUP); | ||||
error = ktls_create_session(so, en, &tls); | error = ktls_create_session(so, en, &tls, KTLS_RX); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
error = ktls_ocf_try(so, tls, KTLS_RX); | error = ktls_ocf_try(so, tls, KTLS_RX); | ||||
if (error) { | if (error) { | ||||
ktls_cleanup(tls); | ktls_cleanup(tls); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef TCP_OFFLOAD | |||||
error = ktls_try_toe(so, tls, KTLS_RX); | |||||
if (error) | |||||
#endif | |||||
ktls_use_sw(tls); | |||||
/* Mark the socket as using TLS offload. */ | /* Mark the socket as using TLS offload. */ | ||||
SOCKBUF_LOCK(&so->so_rcv); | SOCKBUF_LOCK(&so->so_rcv); | ||||
so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq); | so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq); | ||||
so->so_rcv.sb_tls_info = tls; | so->so_rcv.sb_tls_info = tls; | ||||
so->so_rcv.sb_flags |= SB_TLS_RX; | so->so_rcv.sb_flags |= SB_TLS_RX; | ||||
/* Mark existing data as not ready until it can be decrypted. */ | /* Mark existing data as not ready until it can be decrypted. */ | ||||
if (tls->mode != TCP_TLS_MODE_TOE) { | if (tls->mode != TCP_TLS_MODE_TOE) { | ||||
sb_mark_notready(&so->so_rcv); | sb_mark_notready(&so->so_rcv); | ||||
ktls_check_rx(&so->so_rcv); | ktls_check_rx(&so->so_rcv); | ||||
} | } | ||||
SOCKBUF_UNLOCK(&so->so_rcv); | SOCKBUF_UNLOCK(&so->so_rcv); | ||||
/* Prefer TOE -> ifnet TLS -> software TLS. */ | |||||
Not Done Inline ActionsHmm, I need to think about moving this. My TOE patches did move it, but TOE as it is today isn't ready for this to move I think. jhb: Hmm, I need to think about moving this. My TOE patches did move it, but TOE as it is today… | |||||
#ifdef TCP_OFFLOAD | |||||
error = ktls_try_toe(so, tls, KTLS_RX); | |||||
if (error) | |||||
#endif | |||||
error = ktls_try_ifnet(so, tls, KTLS_RX, false); | |||||
if (error) | |||||
ktls_use_sw(tls); | |||||
counter_u64_add(ktls_offload_total, 1); | counter_u64_add(ktls_offload_total, 1); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
ktls_enable_tx(struct socket *so, struct tls_enable *en) | ktls_enable_tx(struct socket *so, struct tls_enable *en) | ||||
{ | { | ||||
Show All 24 Lines | ktls_enable_tx(struct socket *so, struct tls_enable *en) | ||||
if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) | if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) | ||||
return (ENOTSUP); | return (ENOTSUP); | ||||
/* TLS requires ext pgs */ | /* TLS requires ext pgs */ | ||||
if (mb_use_ext_pgs == 0) | if (mb_use_ext_pgs == 0) | ||||
return (ENXIO); | return (ENXIO); | ||||
error = ktls_create_session(so, en, &tls); | error = ktls_create_session(so, en, &tls, KTLS_TX); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
/* Prefer TOE -> ifnet TLS -> software TLS. */ | /* Prefer TOE -> ifnet TLS -> software TLS. */ | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
error = ktls_try_toe(so, tls, KTLS_TX); | error = ktls_try_toe(so, tls, KTLS_TX); | ||||
if (error) | if (error) | ||||
#endif | #endif | ||||
error = ktls_try_ifnet(so, tls, false); | error = ktls_try_ifnet(so, tls, KTLS_TX, false); | ||||
if (error) | if (error) | ||||
error = ktls_try_sw(so, tls, KTLS_TX); | error = ktls_try_sw(so, tls, KTLS_TX); | ||||
if (error) { | if (error) { | ||||
ktls_cleanup(tls); | ktls_cleanup(tls); | ||||
return (error); | return (error); | ||||
} | } | ||||
Show All 40 Lines | if (tls == NULL) | ||||
*modep = TCP_TLS_MODE_NONE; | *modep = TCP_TLS_MODE_NONE; | ||||
else | else | ||||
*modep = tls->mode; | *modep = tls->mode; | ||||
SOCK_RECVBUF_UNLOCK(so); | SOCK_RECVBUF_UNLOCK(so); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq) | |||||
{ | |||||
struct socket *so; | |||||
struct tcpcb *tp; | |||||
INP_RLOCK(inp); | |||||
so = inp->inp_socket; | |||||
Done Inline ActionsShouldn't this check happen under the inp lock? markj: Shouldn't this check happen under the inp lock? | |||||
if (__predict_false(so == NULL)) { | |||||
INP_RUNLOCK(inp); | |||||
return (EINVAL); | |||||
} | |||||
if (inp->inp_flags2 & INP_FREED) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
tp = intotcpcb(inp); | |||||
MPASS(tp != NULL); | |||||
SOCKBUF_LOCK(&so->so_rcv); | |||||
Not Done Inline ActionsI don't think this will really do the right thing if there are multiple TLS records in flight? Or at least, it seems to return an arbitrary future TCP sequence against an older TLS record? Ah, it seems this needs the first TLS record, and is only really useful in the case that sb_tlscc is zero. For correctness you might also want to subtract off the count of data currently being decrypted as well? We should also document what semantics this returns (first TLS record rather than last). jhb: I don't think this will really do the right thing if there are multiple TLS records in flight? | |||||
*tcpseq = tp->rcv_nxt - so->so_rcv.sb_tlscc; | |||||
*tlsseq = so->so_rcv.sb_tls_seqno; | |||||
SOCKBUF_UNLOCK(&so->so_rcv); | |||||
INP_RUNLOCK(inp); | |||||
return (0); | |||||
} | |||||
int | |||||
ktls_get_tx_mode(struct socket *so, int *modep) | ktls_get_tx_mode(struct socket *so, int *modep) | ||||
{ | { | ||||
struct ktls_session *tls; | struct ktls_session *tls; | ||||
struct inpcb *inp; | struct inpcb *inp; | ||||
if (SOLISTENING(so)) | if (SOLISTENING(so)) | ||||
return (EINVAL); | return (EINVAL); | ||||
inp = so->so_pcb; | inp = so->so_pcb; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | ktls_set_tx_mode(struct socket *so, int mode) | ||||
tls = ktls_hold(tls); | tls = ktls_hold(tls); | ||||
SOCKBUF_UNLOCK(&so->so_snd); | SOCKBUF_UNLOCK(&so->so_snd); | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
tls_new = ktls_clone_session(tls); | tls_new = ktls_clone_session(tls); | ||||
if (mode == TCP_TLS_MODE_IFNET) | if (mode == TCP_TLS_MODE_IFNET) | ||||
error = ktls_try_ifnet(so, tls_new, true); | error = ktls_try_ifnet(so, tls_new, KTLS_TX, true); | ||||
else | else | ||||
error = ktls_try_sw(so, tls_new, KTLS_TX); | error = ktls_try_sw(so, tls_new, KTLS_TX); | ||||
if (error) { | if (error) { | ||||
counter_u64_add(ktls_switch_failed, 1); | counter_u64_add(ktls_switch_failed, 1); | ||||
ktls_free(tls_new); | ktls_free(tls_new); | ||||
ktls_free(tls); | ktls_free(tls); | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
return (error); | return (error); | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | ktls_set_tx_mode(struct socket *so, int mode) | ||||
else | else | ||||
counter_u64_add(ktls_switch_to_sw, 1); | counter_u64_add(ktls_switch_to_sw, 1); | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Try to allocate a new TLS send tag. This task is scheduled when | * ktls_reset_send_receive_tag - try to allocate a new TLS send or receive tag. | ||||
* ip_output detects a route change while trying to transmit a packet | * | ||||
* holding a TLS record. If a new tag is allocated, replace the tag | * This task is scheduled when ip_output detects a route change while | ||||
* in the TLS session. Subsequent packets on the connection will use | * trying to transmit a packet holding a TLS record. If a new tag is | ||||
* the new tag. If a new tag cannot be allocated, drop the | * allocated, replace the tag in the TLS session. Subsequent packets | ||||
* connection. | * on the connection will use the new tag. If a new tag cannot be | ||||
* allocated, drop the connection. | |||||
*/ | */ | ||||
static void | static void | ||||
ktls_reset_send_tag(void *context, int pending) | ktls_reset_send_receive_tag(void *context, int pending) | ||||
{ | { | ||||
struct epoch_tracker et; | struct epoch_tracker et; | ||||
struct ktls_session *tls; | struct ktls_session *tls; | ||||
struct m_snd_tag *old, *new; | struct m_snd_tag *snd_rcv_old; | ||||
struct m_snd_tag *snd_rcv_new; | |||||
struct inpcb *inp; | struct inpcb *inp; | ||||
struct tcpcb *tp; | struct tcpcb *tp; | ||||
int error; | int error; | ||||
MPASS(pending == 1); | MPASS(pending == 1); | ||||
tls = context; | tls = context; | ||||
inp = tls->inp; | inp = tls->inp; | ||||
/* | /* | ||||
* Free the old tag first before allocating a new one. | * Free the old tag first before allocating a new one. | ||||
* ip[6]_output_send() will treat a NULL send tag the same as | * ip[6]_output_send() will treat a NULL send tag the same as | ||||
* an ifp mismatch and drop packets until a new tag is | * an ifp mismatch and drop packets until a new tag is | ||||
* allocated. | * allocated. | ||||
* | * | ||||
* Write-lock the INP when changing tls->snd_tag since | * Write-lock the INP when changing tls->snd_rcv_tag since | ||||
* ip[6]_output_send() holds a read-lock when reading the | * ip[6]_output_send() holds a read-lock when reading the | ||||
* pointer. | * pointer. | ||||
*/ | */ | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
old = tls->snd_tag; | snd_rcv_old = tls->snd_rcv_tag; | ||||
tls->snd_tag = NULL; | tls->snd_rcv_tag = NULL; | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
if (old != NULL) | |||||
m_snd_tag_rele(old); | |||||
error = ktls_alloc_snd_tag(inp, tls, true, &new); | if (snd_rcv_old != NULL) | ||||
m_snd_tag_rele(snd_rcv_old); | |||||
if (error == 0) { | switch (tls->direction) { | ||||
case KTLS_TX: | |||||
error = ktls_alloc_snd_tag(inp, tls, true, &snd_rcv_new); | |||||
break; | |||||
case KTLS_RX: | |||||
error = ktls_alloc_rcv_tag(inp, tls, true, &snd_rcv_new); | |||||
break; | |||||
default: | |||||
goto drop_connection; | |||||
Done Inline ActionsI would use __assume_notreached() or whatever the wrapper we for that is for cases that are impossible here and in other places. jhb: I would use `__assume_notreached()` or whatever the wrapper we for that is for cases that are… | |||||
} | |||||
if (error != 0) | |||||
goto drop_connection; | |||||
Not Done Inline ActionsWe might be able to infer this from the existing send tag's type instead and avoid adding the new field to the TLS session if this is the only place it is used. jhb: We might be able to infer this from the existing send tag's type instead and avoid adding the… | |||||
Done Inline ActionsI'll check what options are possible. hselasky: I'll check what options are possible. | |||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
tls->snd_tag = new; | tls->snd_rcv_tag = snd_rcv_new; | ||||
mtx_pool_lock(mtxpool_sleep, tls); | mtx_pool_lock(mtxpool_sleep, tls); | ||||
tls->reset_pending = false; | tls->reset_pending = false; | ||||
mtx_pool_unlock(mtxpool_sleep, tls); | mtx_pool_unlock(mtxpool_sleep, tls); | ||||
if (!in_pcbrele_wlocked(inp)) | if (!in_pcbrele_wlocked(inp)) | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
counter_u64_add(ktls_ifnet_reset, 1); | counter_u64_add(ktls_ifnet_reset, 1); | ||||
ktls_free(tls); | |||||
/* | /* | ||||
* XXX: Should we kick tcp_output explicitly now that | * XXX: Should we kick tcp_output explicitly now that | ||||
* the send tag is fixed or just rely on timers? | * the send tag is fixed or just rely on timers? | ||||
*/ | */ | ||||
} else { | return; | ||||
drop_connection: | |||||
NET_EPOCH_ENTER(et); | NET_EPOCH_ENTER(et); | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
if (!in_pcbrele_wlocked(inp)) { | if (!in_pcbrele_wlocked(inp)) { | ||||
if (!(inp->inp_flags & INP_TIMEWAIT) && | if (!(inp->inp_flags & INP_TIMEWAIT) && | ||||
!(inp->inp_flags & INP_DROPPED)) { | !(inp->inp_flags & INP_DROPPED)) { | ||||
tp = intotcpcb(inp); | tp = intotcpcb(inp); | ||||
CURVNET_SET(tp->t_vnet); | CURVNET_SET(tp->t_vnet); | ||||
tp = tcp_drop(tp, ECONNABORTED); | tp = tcp_drop(tp, ECONNABORTED); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
if (tp != NULL) | if (tp != NULL) | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
counter_u64_add(ktls_ifnet_reset_dropped, 1); | counter_u64_add(ktls_ifnet_reset_dropped, 1); | ||||
} else | } else | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
} | } | ||||
NET_EPOCH_EXIT(et); | NET_EPOCH_EXIT(et); | ||||
counter_u64_add(ktls_ifnet_reset_failed, 1); | counter_u64_add(ktls_ifnet_reset_failed, 1); | ||||
/* | /* | ||||
* Leave reset_pending true to avoid future tasks while | * Leave reset_pending true to avoid future tasks while | ||||
* the socket goes away. | * the socket goes away. | ||||
*/ | */ | ||||
} | |||||
ktls_free(tls); | ktls_free(tls); | ||||
} | } | ||||
int | static void | ||||
ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls) | ktls_output_eagain_tls(struct inpcb *inp, struct ktls_session *tls) | ||||
{ | { | ||||
if (inp == NULL) | |||||
return (ENOBUFS); | |||||
INP_LOCK_ASSERT(inp); | |||||
/* | /* | ||||
* See if we should schedule a task to update the send tag for | * See if we should schedule a task to update the send tag for | ||||
* this session. | * this session. | ||||
*/ | */ | ||||
mtx_pool_lock(mtxpool_sleep, tls); | mtx_pool_lock(mtxpool_sleep, tls); | ||||
if (!tls->reset_pending) { | if (!tls->reset_pending) { | ||||
(void) ktls_hold(tls); | (void) ktls_hold(tls); | ||||
in_pcbref(inp); | in_pcbref(inp); | ||||
tls->inp = inp; | tls->inp = inp; | ||||
tls->reset_pending = true; | tls->reset_pending = true; | ||||
taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task); | taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task); | ||||
} | } | ||||
mtx_pool_unlock(mtxpool_sleep, tls); | mtx_pool_unlock(mtxpool_sleep, tls); | ||||
} | |||||
int | |||||
ktls_output_eagain(struct inpcb *inp) | |||||
{ | |||||
struct socket *so; | |||||
struct ktls_session *tls; | |||||
if (__predict_false(inp == NULL)) | |||||
goto done; | |||||
INP_LOCK_ASSERT(inp); | |||||
so = inp->inp_socket; | |||||
if (__predict_false(so == NULL)) | |||||
goto done; | |||||
tls = so->so_rcv.sb_tls_info; | |||||
if (__predict_true(tls != NULL)) | |||||
ktls_output_eagain_tls(inp, tls); | |||||
tls = so->so_snd.sb_tls_info; | |||||
if (__predict_true(tls != NULL)) | |||||
ktls_output_eagain_tls(inp, tls); | |||||
done: | |||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} | } | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
int | int | ||||
ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate) | ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate) | ||||
{ | { | ||||
union if_snd_tag_modify_params params = { | union if_snd_tag_modify_params params = { | ||||
.rate_limit.max_rate = max_pacing_rate, | .rate_limit.max_rate = max_pacing_rate, | ||||
.rate_limit.flags = M_NOWAIT, | .rate_limit.flags = M_NOWAIT, | ||||
}; | }; | ||||
struct m_snd_tag *mst; | struct m_snd_tag *mst; | ||||
/* Can't get to the inp, but it should be locked. */ | /* Can't get to the inp, but it should be locked. */ | ||||
/* INP_LOCK_ASSERT(inp); */ | /* INP_LOCK_ASSERT(inp); */ | ||||
MPASS(tls->mode == TCP_TLS_MODE_IFNET); | MPASS(tls->mode == TCP_TLS_MODE_IFNET); | ||||
if (tls->snd_tag == NULL) { | if (tls->snd_rcv_tag == NULL) { | ||||
/* | /* | ||||
* Resetting send tag, ignore this change. The | * Resetting send tag, ignore this change. The | ||||
* pending reset may or may not see this updated rate | * pending reset may or may not see this updated rate | ||||
* in the tcpcb. If it doesn't, we will just lose | * in the tcpcb. If it doesn't, we will just lose | ||||
* this rate change. | * this rate change. | ||||
*/ | */ | ||||
return (0); | return (0); | ||||
} | } | ||||
Done Inline Actionsjhb@ : May I commit this change separately? It basically makes sure we don't see different values in the assert and the "mst = " read below. hselasky: jhb@ : May I commit this change separately?
It basically makes sure we don't see different… | |||||
Done Inline ActionsI think this is a nop since the caller holds the INP_WLOCK here? It's fine to merge as it is a bit more readable, but I don't think it is a functional change due to the lock. jhb: I think this is a nop since the caller holds the INP_WLOCK here? It's fine to merge as it is a… | |||||
MPASS(tls->snd_tag != NULL); | mst = tls->snd_rcv_tag; | ||||
MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT); | |||||
mst = tls->snd_tag; | MPASS(mst != NULL); | ||||
MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT); | |||||
return (mst->sw->snd_tag_modify(mst, ¶ms)); | return (mst->sw->snd_tag_modify(mst, ¶ms)); | ||||
} | } | ||||
#endif | #endif | ||||
#endif | #endif | ||||
void | void | ||||
ktls_destroy(struct ktls_session *tls) | ktls_destroy(struct ktls_session *tls) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | if (n == NULL) { | ||||
n = m_get(M_WAITOK, MT_DATA); | n = m_get(M_WAITOK, MT_DATA); | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
if (sb->sb_mtls != top) { | if (sb->sb_mtls != top) { | ||||
m_free(n); | m_free(n); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} | } | ||||
n->m_flags |= M_NOTREADY; | n->m_flags |= (m->m_flags & (M_NOTREADY | M_DECRYPTED)); | ||||
/* Store remainder in 'n'. */ | /* Store remainder in 'n'. */ | ||||
n->m_len = m->m_len - remain; | n->m_len = m->m_len - remain; | ||||
if (m->m_flags & M_EXT) { | if (m->m_flags & M_EXT) { | ||||
n->m_data = m->m_data + remain; | n->m_data = m->m_data + remain; | ||||
mb_dupcl(n, m); | mb_dupcl(n, m); | ||||
} else { | } else { | ||||
bcopy(mtod(m, caddr_t) + remain, mtod(n, caddr_t), n->m_len); | bcopy(mtod(m, caddr_t) + remain, mtod(n, caddr_t), n->m_len); | ||||
Show All 21 Lines | out: | ||||
for (m = top; m != NULL; m = m->m_next) | for (m = top; m != NULL; m = m->m_next) | ||||
sbfree_ktls_rx(sb, m); | sbfree_ktls_rx(sb, m); | ||||
sb->sb_tlsdcc = len; | sb->sb_tlsdcc = len; | ||||
sb->sb_ccc += len; | sb->sb_ccc += len; | ||||
SBCHECK(sb); | SBCHECK(sb); | ||||
return (top); | return (top); | ||||
} | } | ||||
/* | |||||
* Check if a mbuf chain is fully decrypted at the given offset and | |||||
* length. Returns KTLS_MBUF_CRYPTO_ST_DECRYPTED if all data is | |||||
* decrypted. KTLS_MBUF_CRYPTO_ST_MIXED if there is a mix of encrypted | |||||
* and decrypted data. Else KTLS_MBUF_CRYPTO_ST_ENCRYPTED if all data | |||||
* is encrypted. | |||||
*/ | |||||
int | |||||
ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len) | |||||
{ | |||||
int m_flags_ored = 0; | |||||
int m_flags_anded = -1; | |||||
for (; mb != NULL; mb = mb->m_next) { | |||||
if (offset < mb->m_len) | |||||
break; | |||||
offset -= mb->m_len; | |||||
} | |||||
offset += len; | |||||
for (; mb != NULL; mb = mb->m_next) { | |||||
m_flags_ored |= mb->m_flags; | |||||
m_flags_anded &= mb->m_flags; | |||||
if (offset <= mb->m_len) | |||||
break; | |||||
offset -= mb->m_len; | |||||
} | |||||
MPASS(mb != NULL || offset == 0); | |||||
if ((m_flags_ored ^ m_flags_anded) & M_DECRYPTED) | |||||
return (KTLS_MBUF_CRYPTO_ST_MIXED); | |||||
else | |||||
return ((m_flags_ored & M_DECRYPTED) ? | |||||
KTLS_MBUF_CRYPTO_ST_DECRYPTED : | |||||
KTLS_MBUF_CRYPTO_ST_ENCRYPTED); | |||||
} | |||||
/* | |||||
* ktls_resync_ifnet - get HW TLS RX back on track after packet loss | |||||
*/ | |||||
static int | |||||
ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num) | |||||
{ | |||||
union if_snd_tag_modify_params params; | |||||
struct m_snd_tag *mst; | |||||
struct inpcb *inp; | |||||
struct tcpcb *tp; | |||||
mst = so->so_rcv.sb_tls_info->snd_rcv_tag; | |||||
if (__predict_false(mst == NULL)) | |||||
return (EINVAL); | |||||
inp = sotoinpcb(so); | |||||
if (__predict_false(inp == NULL)) | |||||
return (EINVAL); | |||||
INP_RLOCK(inp); | |||||
if (inp->inp_flags2 & INP_FREED) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | |||||
INP_RUNLOCK(inp); | |||||
return (ECONNRESET); | |||||
} | |||||
tp = intotcpcb(inp); | |||||
MPASS(tp != NULL); | |||||
Done Inline ActionsI think this doesn't handle the case that there are multiple TLS records in the queue? The fetch_info routine tries to handle that case. Or perhaps what happens is that you keep getting TCP sequences that have already passed and can't get in sync until there is only zero or 1 TLS records in the sb_mtls queue? jhb: I think this doesn't handle the case that there are multiple TLS records in the queue? The… | |||||
Done Inline ActionsThis is not how this mechanism works. The hardware speculates into the TLS header detection, and if a valid chain is found based on a past TCP sequence number which is re-loaded every now and then by the HW, the decryption immediately starts, after that the software confirms this TCP sequence number really points to the start of a valid TLS record. The callback mechanism into the driver, simply builds a small database record of past TLS TCP start sequence numbers. Later on the driver queries the hardware, where it currently started decrypting, and if a match is found, we are good. That means this function only provides the TCP SN of the current TLS record being processed and not the last one. hselasky: This is not how this mechanism works. The hardware speculates into the TLS header detection… | |||||
Done Inline ActionsMmm, so what happens if the hardware guesses wrong? That is, from your description it would seem that it would only start sending decrypted data once the software confirms the TCP sequence number (via the modify call with a TLS record length of 0), but that only seems to be invoked currently if some encrypted data is received which doesn't match the description. I would assume you would want/need the newest TLS record (mostly recently received) as you are trying to line up with the start of a future TLS record not received yet. jhb: Mmm, so what happens if the hardware guesses wrong? That is, from your description it would… | |||||
Done Inline ActionsHi John, The TLS record length is not encrypted. When you have a chain of TLS records up to the present time, you only need to confirm one past TLS record started at a given TCP sequence number. The all the subsequent ones should be fine. This works fine up to 4 GBytes of data. v hardware stores the TCP sequence number of a previous TLS record. [ TLS RECORD ..... ] [ TLS RECORD ..... ] [ TLS RECORD ..... ] [ TLS RECORD ..... * ] * The last received TCP sequence number is not of relevance The software keeps track of the past TLS record starts and asks the HW what TCP sequence number it wants to confirm. Then if the TCP sequence number the SW gets exist in its locally kept database, then it tells hardware back, that this TCP sequence number is valid, and if the HW still refers the same TCP sequence number, then HW decryption starts immediately at the next IP packet received. Was that clear? hselasky: Hi John,
The TLS record length is not encrypted. When you have a chain of TLS records up to… | |||||
/* Get the TCP sequence number of the next valid TLS header. */ | |||||
SOCKBUF_LOCK(&so->so_rcv); | |||||
params.tls_rx.tls_hdr_tcp_sn = | |||||
tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len; | |||||
params.tls_rx.tls_rec_length = tls_len; | |||||
params.tls_rx.tls_rec_number = tls_rcd_num; | |||||
SOCKBUF_UNLOCK(&so->so_rcv); | |||||
INP_RUNLOCK(inp); | |||||
MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RX); | |||||
return (mst->sw->snd_tag_modify(mst, ¶ms)); | |||||
} | |||||
static struct mbuf * | |||||
ktls_dup_rx_chain(struct mbuf *m) | |||||
{ | |||||
struct mbuf *top; | |||||
struct mbuf **pp; | |||||
MPASS(m != NULL); | |||||
pp = ⊤ | |||||
do { | |||||
*pp = m_get2(m->m_len, M_WAITOK, MT_DATA, M_NOTREADY); | |||||
if (m->m_flags & M_DECRYPTED) | |||||
m_copydata(m, 0, ((*pp)->m_len = m->m_len), mtod(*pp, caddr_t)); | |||||
else | |||||
memset(mtod(*pp, caddr_t), 0, ((*pp)->m_len = m->m_len)); | |||||
pp = &(*pp)->m_next; | |||||
} while ((m = m->m_next) != NULL); | |||||
*pp = NULL; | |||||
return (top); | |||||
} | |||||
static void | static void | ||||
ktls_decrypt(struct socket *so) | ktls_decrypt(struct socket *so) | ||||
{ | { | ||||
char tls_header[MBUF_PEXT_HDR_LEN]; | char tls_header[MBUF_PEXT_HDR_LEN]; | ||||
struct ktls_session *tls; | struct ktls_session *tls; | ||||
struct sockbuf *sb; | struct sockbuf *sb; | ||||
struct tls_record_layer *hdr; | struct tls_record_layer *hdr; | ||||
struct tls_get_record tgr; | struct tls_get_record tgr; | ||||
struct mbuf *control, *data, *m; | struct mbuf *control, *data, *m; | ||||
uint64_t seqno; | uint64_t seqno; | ||||
int error, remain, tls_len, trail_len; | int error, remain, tls_len, trail_len; | ||||
int state; | |||||
hdr = (struct tls_record_layer *)tls_header; | hdr = (struct tls_record_layer *)tls_header; | ||||
sb = &so->so_rcv; | sb = &so->so_rcv; | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING, | KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING, | ||||
("%s: socket %p not running", __func__, so)); | ("%s: socket %p not running", __func__, so)); | ||||
tls = sb->sb_tls_info; | tls = sb->sb_tls_info; | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | if (data == NULL) | ||||
continue; | continue; | ||||
MPASS(sb->sb_tlsdcc == tls_len); | MPASS(sb->sb_tlsdcc == tls_len); | ||||
seqno = sb->sb_tls_seqno; | seqno = sb->sb_tls_seqno; | ||||
sb->sb_tls_seqno++; | sb->sb_tls_seqno++; | ||||
SBCHECK(sb); | SBCHECK(sb); | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
/* get crypto state for this TLS record */ | |||||
state = ktls_mbuf_crypto_state(data, 0, tls_len); | |||||
Done Inline ActionsIt might be easier to read if 'state' is an enum of some sort, or at least uses #define values. jhb: It might be easier to read if 'state' is an enum of some sort, or at least uses #define values. | |||||
switch (state) { | |||||
struct mbuf **pp; | |||||
Done Inline ActionsI would maybe move these into the case 0 declaring variables here is a bit unusual style. jhb: I would maybe move these into the `case 0` declaring variables here is a bit unusual style. | |||||
struct mbuf *m0; | |||||
struct mbuf *m1; | |||||
case KTLS_MBUF_CRYPTO_ST_MIXED: | |||||
pp = &data; | |||||
m0 = data; | |||||
m1 = ktls_dup_rx_chain(data); | |||||
/* Perform XOR of crypto sequence. */ | |||||
error = tls->sw_recrypt(tls, hdr, m1, seqno); | |||||
if (__predict_false(error != 0)) { | |||||
m_freem(m1); | |||||
break; | |||||
} | |||||
/* Reconstruct encrypted mbuf data. */ | |||||
while (m0 != NULL) { | |||||
if (m0->m_flags & M_DECRYPTED) { | |||||
*pp = m1; | |||||
pp = &(*pp)->m_next; | |||||
m0 = m_free(m0); | |||||
m1 = m1->m_next; | |||||
} else { | |||||
*pp = m0; | |||||
pp = &(*pp)->m_next; | |||||
m0 = m0->m_next; | |||||
m1 = m_free(m1); | |||||
} | |||||
} | |||||
*pp = NULL; | |||||
MPASS(m1 == NULL); | |||||
/* FALLTHROUGH */ | |||||
case KTLS_MBUF_CRYPTO_ST_ENCRYPTED: | |||||
error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len); | error = tls->sw_decrypt(tls, hdr, data, seqno, &trail_len); | ||||
break; | |||||
default: | |||||
error = 0; | |||||
trail_len = tls->params.tls_tlen; /* XXX */ | |||||
break; | |||||
} | |||||
Done Inline ActionsMaybe make this ST_DECRYPTED explicitly for readability? jhb: Maybe make this ST_DECRYPTED explicitly for readability? | |||||
if (error) { | if (error) { | ||||
counter_u64_add(ktls_offload_failed_crypto, 1); | counter_u64_add(ktls_offload_failed_crypto, 1); | ||||
Done Inline ActionsI think you can axe the XXX comments and instead add a larger block comment here: /* * NIC TLS is only supported for AEAD ciphersuites which used a fixed sized trailer. */ jhb: I think you can axe the XXX comments and instead add a larger block comment here:
```
/*… | |||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
if (sb->sb_tlsdcc == 0) { | if (sb->sb_tlsdcc == 0) { | ||||
/* | /* | ||||
* sbcut/drop/flush discarded these | * sbcut/drop/flush discarded these | ||||
* mbufs. | * mbufs. | ||||
*/ | */ | ||||
m_freem(data); | m_freem(data); | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | if (tgr.tls_length == htobe16(0)) { | ||||
remain -= data->m_len; | remain -= data->m_len; | ||||
data = m_free(data); | data = m_free(data); | ||||
} | } | ||||
/* Trim trailer and clear M_NOTREADY. */ | /* Trim trailer and clear M_NOTREADY. */ | ||||
remain = be16toh(tgr.tls_length); | remain = be16toh(tgr.tls_length); | ||||
m = data; | m = data; | ||||
for (m = data; remain > m->m_len; m = m->m_next) { | for (m = data; remain > m->m_len; m = m->m_next) { | ||||
m->m_flags &= ~M_NOTREADY; | m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); | ||||
remain -= m->m_len; | remain -= m->m_len; | ||||
} | } | ||||
m->m_len = remain; | m->m_len = remain; | ||||
m_freem(m->m_next); | m_freem(m->m_next); | ||||
m->m_next = NULL; | m->m_next = NULL; | ||||
m->m_flags &= ~M_NOTREADY; | m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); | ||||
/* Set EOR on the final mbuf. */ | /* Set EOR on the final mbuf. */ | ||||
m->m_flags |= M_EOR; | m->m_flags |= M_EOR; | ||||
} | } | ||||
sbappendcontrol_locked(sb, data, control, 0); | sbappendcontrol_locked(sb, data, control, 0); | ||||
if (__predict_false(state != -1)) { | |||||
Done Inline ActionsIs state != -1 still the right check? jhb: Is state != -1 still the right check? | |||||
Done Inline ActionsSwitched to using define. hselasky: Switched to using define. | |||||
sb->sb_flags |= SB_TLS_RX_RESYNC; | |||||
SOCKBUF_UNLOCK(sb); | |||||
ktls_resync_ifnet(so, tls_len, seqno); | |||||
SOCKBUF_LOCK(sb); | |||||
} else if (__predict_false(sb->sb_flags & SB_TLS_RX_RESYNC)) { | |||||
sb->sb_flags &= ~SB_TLS_RX_RESYNC; | |||||
SOCKBUF_UNLOCK(sb); | |||||
ktls_resync_ifnet(so, 0, seqno); | |||||
SOCKBUF_LOCK(sb); | |||||
} | |||||
} | } | ||||
sb->sb_flags &= ~SB_TLS_RX_RUNNING; | sb->sb_flags &= ~SB_TLS_RX_RUNNING; | ||||
if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc > 0) | if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc > 0) | ||||
so->so_error = EMSGSIZE; | so->so_error = EMSGSIZE; | ||||
sorwakeup_locked(so); | sorwakeup_locked(so); | ||||
▲ Show 20 Lines • Show All 709 Lines • Show Last 20 Lines |
If we’re running under network epoch, this is not necessary. Nhops are guaranteed to exists within the epoch and each nexthop references the transmit ifp.