diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -319,6 +319,7 @@ M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); } mb->m_pkthdr.rcvif = ifp; + mb->m_pkthdr.leaf_rcvif = ifp; if (cqe_is_tunneled(cqe)) { /* @@ -598,6 +599,7 @@ memset(mb->m_data, 255, 14); mb->m_data[14] = rq->ix; mb->m_pkthdr.rcvif = rq->ifp; + mb->m_pkthdr.leaf_rcvif = rq->ifp; rq->ifp->if_input(rq->ifp, mb); } #endif diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c --- a/sys/kern/kern_mbuf.c +++ b/sys/kern/kern_mbuf.c @@ -1645,12 +1645,21 @@ gen = m->m_pkthdr.rcvif->if_idxgen; m->m_pkthdr.rcvidx = idx; m->m_pkthdr.rcvgen = gen; + if (__predict_false(m->m_pkthdr.leaf_rcvif != NULL)) { + idx = m->m_pkthdr.leaf_rcvif->if_index; + gen = m->m_pkthdr.leaf_rcvif->if_idxgen; + } else { + idx = -1; + gen = 0; + } + m->m_pkthdr.leaf_rcvidx = idx; + m->m_pkthdr.leaf_rcvgen = gen; } struct ifnet * m_rcvif_restore(struct mbuf *m) { - struct ifnet *ifp; + struct ifnet *ifp, *leaf_ifp; M_ASSERTPKTHDR(m); NET_EPOCH_ASSERT(); @@ -1659,7 +1668,19 @@ if (ifp == NULL || (ifp->if_flags & IFF_DYING)) return (NULL); - return (m->m_pkthdr.rcvif = ifp); + if (__predict_true(m->m_pkthdr.leaf_rcvidx == (u_short)-1)) { + leaf_ifp = NULL; + } else { + leaf_ifp = ifnet_byindexgen(m->m_pkthdr.leaf_rcvidx, + m->m_pkthdr.leaf_rcvgen); + if (__predict_false(leaf_ifp != NULL && (leaf_ifp->if_flags & IFF_DYING))) + leaf_ifp = NULL; + } + + m->m_pkthdr.leaf_rcvif = leaf_ifp; + m->m_pkthdr.rcvif = ifp; + + return (ifp); } /* diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -299,6 +299,7 @@ static void ktls_cleanup(struct ktls_session *tls); #if defined(INET) || defined(INET6) +static void ktls_reset_receive_tag(void *context, int pending); static void ktls_reset_send_tag(void *context, int pending); #endif static void ktls_work_thread(void *ctx); @@ -503,7 +504,7 @@ #if defined(INET) || defined(INET6) static int ktls_create_session(struct socket *so, struct tls_enable *en, - struct ktls_session **tlsp) + struct ktls_session **tlsp, int direction) { struct ktls_session *tls; int error; @@ -619,7 +620,10 @@ counter_u64_add(ktls_offload_active, 1); refcount_init(&tls->refcount, 1); - TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls); + if (direction == KTLS_RX) + TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls); + else + TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls); tls->wq_index = ktls_get_cpu(so); @@ -745,7 +749,7 @@ } static struct ktls_session * -ktls_clone_session(struct ktls_session *tls) +ktls_clone_session(struct ktls_session *tls, int direction) { struct ktls_session *tls_new; @@ -754,7 +758,12 @@ counter_u64_add(ktls_offload_active, 1); refcount_init(&tls_new->refcount, 1); - TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new); + if (direction == KTLS_RX) + TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_receive_tag, + tls_new); + else + TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, + tls_new); /* Copy fields from existing session. */ tls_new->params = tls->params; @@ -810,6 +819,8 @@ } if (tls->snd_tag != NULL) m_snd_tag_rele(tls->snd_tag); + if (tls->rx_ifp != NULL) + if_rele(tls->rx_ifp); break; #ifdef TCP_OFFLOAD case TCP_TLS_MODE_TOE: @@ -983,28 +994,135 @@ return (error); } +/* + * Allocate an initial TLS receive tag for doing HW decryption of TLS + * data. + * + * This function allocates a new TLS receive tag on whatever interface + * the connection is currently routed over. If the connection ends up + * using a different interface for receive this will get fixed up via + * ktls_input_ifp_mismatch as future packets arrive. + */ static int -ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force) +ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls, + struct m_snd_tag **mstp) { - struct m_snd_tag *mst; + union if_snd_tag_alloc_params params; + struct ifnet *ifp; + struct nhop_object *nh; int error; - error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); - if (error == 0) { - tls->mode = TCP_TLS_MODE_IFNET; - tls->snd_tag = mst; - switch (tls->params.cipher_algorithm) { - case CRYPTO_AES_CBC: - counter_u64_add(ktls_ifnet_cbc, 1); - break; - case CRYPTO_AES_NIST_GCM_16: - counter_u64_add(ktls_ifnet_gcm, 1); - break; - case CRYPTO_CHACHA20_POLY1305: - counter_u64_add(ktls_ifnet_chacha20, 1); - break; + if (!ktls_ocf_recrypt_supported(tls)) + return (ENXIO); + + INP_RLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_RUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_socket == NULL) { + INP_RUNLOCK(inp); + return (ECONNRESET); + } + + /* + * Check administrative controls on ifnet TLS to determine if + * ifnet TLS should be denied. + */ + if (ktls_ifnet_permitted == 0) { + INP_RUNLOCK(inp); + return (ENXIO); + } + + /* + * XXX: As with ktls_alloc_snd_tag, use the cached route in + * the inpcb to find the interface. + */ + nh = inp->inp_route.ro_nh; + if (nh == NULL) { + INP_RUNLOCK(inp); + return (ENXIO); + } + ifp = nh->nh_ifp; + if_ref(ifp); + tls->rx_ifp = ifp; + + params.hdr.type = IF_SND_TAG_TYPE_TLS_RX; + params.hdr.flowid = inp->inp_flowid; + params.hdr.flowtype = inp->inp_flowtype; + params.hdr.numa_domain = inp->inp_numa_domain; + params.tls_rx.inp = inp; + params.tls_rx.tls = tls; + params.tls_rx.vlan_id = 0; + + INP_RUNLOCK(inp); + + if (inp->inp_vflag & INP_IPV6) { + if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0) { + error = EOPNOTSUPP; + goto out; + } + } else { + if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0) { + error = EOPNOTSUPP; + goto out; } } + error = m_snd_tag_alloc(ifp, ¶ms, mstp); + + /* + * If this connection is over a vlan, vlan_snd_tag_alloc + * rewrites vlan_id with the saved interface. Save the VLAN + * ID for use in ktls_reset_receive_tag which allocates new + * receive tags directly from the leaf interface bypassing + * if_vlan. + */ + if (error == 0) + tls->rx_vlan_id = params.tls_rx.vlan_id; +out: + return (error); +} + +static int +ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction, + bool force) +{ + struct m_snd_tag *mst; + int error; + + switch (direction) { + case KTLS_TX: + error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); + if (__predict_false(error != 0)) + goto done; + break; + case KTLS_RX: + KASSERT(!force, ("%s: forced receive tag", __func__)); + error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst); + if (__predict_false(error != 0)) + goto done; + break; + default: + __assert_unreachable(); + } + + tls->mode = TCP_TLS_MODE_IFNET; + tls->snd_tag = mst; + + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_ifnet_cbc, 1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_ifnet_gcm, 1); + break; + case CRYPTO_CHACHA20_POLY1305: + counter_u64_add(ktls_ifnet_chacha20, 1); + break; + default: + break; + } +done: return (error); } @@ -1185,7 +1303,7 @@ if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable) return (ENOTSUP); - error = ktls_create_session(so, en, &tls); + error = ktls_create_session(so, en, &tls, KTLS_RX); if (error) return (error); @@ -1206,10 +1324,13 @@ ktls_check_rx(&so->so_rcv); SOCKBUF_UNLOCK(&so->so_rcv); + /* Prefer TOE -> ifnet TLS -> software TLS. */ #ifdef TCP_OFFLOAD error = ktls_try_toe(so, tls, KTLS_RX); if (error) #endif + error = ktls_try_ifnet(so, tls, KTLS_RX, false); + if (error) ktls_use_sw(tls); counter_u64_add(ktls_offload_total, 1); @@ -1252,7 +1373,7 @@ if (mb_use_ext_pgs == 0) return (ENXIO); - error = ktls_create_session(so, en, &tls); + error = ktls_create_session(so, en, &tls, KTLS_TX); if (error) return (error); @@ -1261,7 +1382,7 @@ error = ktls_try_toe(so, tls, KTLS_TX); if (error) #endif - error = ktls_try_ifnet(so, tls, false); + error = ktls_try_ifnet(so, tls, KTLS_TX, false); if (error) error = ktls_try_sw(so, tls, KTLS_TX); @@ -1418,10 +1539,10 @@ SOCKBUF_UNLOCK(&so->so_snd); INP_WUNLOCK(inp); - tls_new = ktls_clone_session(tls); + tls_new = ktls_clone_session(tls, KTLS_TX); if (mode == TCP_TLS_MODE_IFNET) - error = ktls_try_ifnet(so, tls_new, true); + error = ktls_try_ifnet(so, tls_new, KTLS_TX, true); else error = ktls_try_sw(so, tls_new, KTLS_TX); if (error) { @@ -1479,6 +1600,89 @@ return (0); } +/* + * Try to allocate a new TLS receive tag. This task is scheduled when + * sbappend_ktls_rx detects an input path change. If a new tag is + * allocated, replace the tag in the TLS session. If a new tag cannot + * be allocated, let the session fall back to software decryption. + */ +static void +ktls_reset_receive_tag(void *context, int pending) +{ + union if_snd_tag_alloc_params params; + struct ktls_session *tls; + struct m_snd_tag *mst; + struct inpcb *inp; + struct ifnet *ifp; + struct socket *so; + int error; + + MPASS(pending == 1); + + tls = context; + so = tls->so; + inp = so->so_pcb; + ifp = NULL; + + INP_RLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_RUNLOCK(inp); + goto out; + } + + SOCKBUF_LOCK(&so->so_rcv); + m_snd_tag_rele(tls->snd_tag); + tls->snd_tag = NULL; + + ifp = tls->rx_ifp; + if_ref(ifp); + SOCKBUF_UNLOCK(&so->so_rcv); + + params.hdr.type = IF_SND_TAG_TYPE_TLS_RX; + params.hdr.flowid = inp->inp_flowid; + params.hdr.flowtype = inp->inp_flowtype; + params.hdr.numa_domain = inp->inp_numa_domain; + params.tls_rx.inp = inp; + params.tls_rx.tls = tls; + params.tls_rx.vlan_id = tls->rx_vlan_id; + INP_RUNLOCK(inp); + + if (inp->inp_vflag & INP_IPV6) { + if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0) + goto out; + } else { + if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0) + goto out; + } + + error = m_snd_tag_alloc(ifp, ¶ms, &mst); + if (error == 0) { + SOCKBUF_LOCK(&so->so_rcv); + tls->snd_tag = mst; + SOCKBUF_UNLOCK(&so->so_rcv); + + counter_u64_add(ktls_ifnet_reset, 1); + } else { + /* + * Just fall back to software decryption if a tag + * cannot be allocated leaving the connection intact. + * If a future input path change switches to another + * interface this connection will resume ifnet TLS. + */ + counter_u64_add(ktls_ifnet_reset_failed, 1); + } + +out: + mtx_pool_lock(mtxpool_sleep, tls); + tls->reset_pending = false; + mtx_pool_unlock(mtxpool_sleep, tls); + + if (ifp != NULL) + if_rele(ifp); + sorele(so); + ktls_free(tls); +} + /* * Try to allocate a new TLS send tag. This task is scheduled when * ip_output detects a route change while trying to transmit a packet @@ -1565,6 +1769,37 @@ ktls_free(tls); } +void +ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp) +{ + struct ktls_session *tls; + struct socket *so; + + SOCKBUF_LOCK_ASSERT(sb); + KASSERT(sb->sb_flags & SB_TLS_RX, ("%s: sockbuf %p isn't TLS RX", + __func__, sb)); + so = __containerof(sb, struct socket, so_rcv); + + tls = sb->sb_tls_info; + if_rele(tls->rx_ifp); + if_ref(ifp); + tls->rx_ifp = ifp; + + /* + * See if we should schedule a task to update the receive tag for + * this session. + */ + mtx_pool_lock(mtxpool_sleep, tls); + if (!tls->reset_pending) { + (void) ktls_hold(tls); + soref(so); + tls->so = so; + tls->reset_pending = true; + taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task); + } + mtx_pool_unlock(mtxpool_sleep, tls); +} + int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls) { @@ -1615,10 +1850,11 @@ return (0); } - MPASS(tls->snd_tag != NULL); - MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT); - mst = tls->snd_tag; + + MPASS(mst != NULL); + MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT); + return (mst->sw->snd_tag_modify(mst, ¶ms)); } #endif @@ -1907,7 +2143,7 @@ return (NULL); } } - n->m_flags |= M_NOTREADY; + n->m_flags |= (m->m_flags & (M_NOTREADY | M_DECRYPTED)); /* Store remainder in 'n'. */ n->m_len = m->m_len - remain; @@ -1992,6 +2228,86 @@ return (0); } +/* + * Check if a mbuf chain is fully decrypted at the given offset and + * length. Returns KTLS_MBUF_CRYPTO_ST_DECRYPTED if all data is + * decrypted. KTLS_MBUF_CRYPTO_ST_MIXED if there is a mix of encrypted + * and decrypted data. Else KTLS_MBUF_CRYPTO_ST_ENCRYPTED if all data + * is encrypted. + */ +ktls_mbuf_crypto_st_t +ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len) +{ + int m_flags_ored = 0; + int m_flags_anded = -1; + + for (; mb != NULL; mb = mb->m_next) { + if (offset < mb->m_len) + break; + offset -= mb->m_len; + } + offset += len; + + for (; mb != NULL; mb = mb->m_next) { + m_flags_ored |= mb->m_flags; + m_flags_anded &= mb->m_flags; + + if (offset <= mb->m_len) + break; + offset -= mb->m_len; + } + MPASS(mb != NULL || offset == 0); + + if ((m_flags_ored ^ m_flags_anded) & M_DECRYPTED) + return (KTLS_MBUF_CRYPTO_ST_MIXED); + else + return ((m_flags_ored & M_DECRYPTED) ? + KTLS_MBUF_CRYPTO_ST_DECRYPTED : + KTLS_MBUF_CRYPTO_ST_ENCRYPTED); +} + +/* + * ktls_resync_ifnet - get HW TLS RX back on track after packet loss + */ +static int +ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num) +{ + union if_snd_tag_modify_params params; + struct m_snd_tag *mst; + struct inpcb *inp; + struct tcpcb *tp; + + mst = so->so_rcv.sb_tls_info->snd_tag; + if (__predict_false(mst == NULL)) + return (EINVAL); + + inp = sotoinpcb(so); + if (__predict_false(inp == NULL)) + return (EINVAL); + + INP_RLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_RUNLOCK(inp); + return (ECONNRESET); + } + + tp = intotcpcb(inp); + MPASS(tp != NULL); + + /* Get the TCP sequence number of the next valid TLS header. */ + SOCKBUF_LOCK(&so->so_rcv); + params.tls_rx.tls_hdr_tcp_sn = + tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len; + params.tls_rx.tls_rec_length = tls_len; + params.tls_rx.tls_seq_number = tls_rcd_num; + SOCKBUF_UNLOCK(&so->so_rcv); + + INP_RUNLOCK(inp); + + MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RX); + return (mst->sw->snd_tag_modify(mst, ¶ms)); +} + static void ktls_decrypt(struct socket *so) { @@ -2001,6 +2317,7 @@ struct tls_record_layer *hdr; struct tls_get_record tgr; struct mbuf *control, *data, *m; + ktls_mbuf_crypto_st_t state; uint64_t seqno; int error, remain, tls_len, trail_len; bool tls13; @@ -2073,13 +2390,46 @@ SBCHECK(sb); SOCKBUF_UNLOCK(sb); - error = ktls_ocf_decrypt(tls, hdr, data, seqno, &trail_len); - if (error == 0) { - if (tls13) + /* get crypto state for this TLS record */ + state = ktls_mbuf_crypto_state(data, 0, tls_len); + + switch (state) { + case KTLS_MBUF_CRYPTO_ST_MIXED: + error = ktls_ocf_recrypt(tls, hdr, data, seqno); + if (error) + break; + /* FALLTHROUGH */ + case KTLS_MBUF_CRYPTO_ST_ENCRYPTED: + error = ktls_ocf_decrypt(tls, hdr, data, seqno, + &trail_len); + if (__predict_true(error == 0)) { + if (tls13) { + error = tls13_find_record_type(tls, data, + tls_len, &trail_len, &record_type); + } else { + record_type = hdr->tls_type; + } + } + break; + case KTLS_MBUF_CRYPTO_ST_DECRYPTED: + /* + * NIC TLS is only supported for AEAD + * ciphersuites which used a fixed sized + * trailer. + */ + if (tls13) { + trail_len = tls->params.tls_tlen - 1; error = tls13_find_record_type(tls, data, tls_len, &trail_len, &record_type); - else + } else { + trail_len = tls->params.tls_tlen; + error = 0; record_type = hdr->tls_type; + } + break; + default: + error = EINVAL; + break; } if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); @@ -2160,19 +2510,31 @@ remain = be16toh(tgr.tls_length); m = data; for (m = data; remain > m->m_len; m = m->m_next) { - m->m_flags &= ~M_NOTREADY; + m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); remain -= m->m_len; } m->m_len = remain; m_freem(m->m_next); m->m_next = NULL; - m->m_flags &= ~M_NOTREADY; + m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); /* Set EOR on the final mbuf. */ m->m_flags |= M_EOR; } sbappendcontrol_locked(sb, data, control, 0); + + if (__predict_false(state != KTLS_MBUF_CRYPTO_ST_DECRYPTED)) { + sb->sb_flags |= SB_TLS_RX_RESYNC; + SOCKBUF_UNLOCK(sb); + ktls_resync_ifnet(so, tls_len, seqno); + SOCKBUF_LOCK(sb); + } else if (__predict_false(sb->sb_flags & SB_TLS_RX_RESYNC)) { + sb->sb_flags &= ~SB_TLS_RX_RESYNC; + SOCKBUF_UNLOCK(sb); + ktls_resync_ifnet(so, 0, seqno); + SOCKBUF_LOCK(sb); + } } sb->sb_flags &= ~SB_TLS_RX_RUNNING; diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -180,11 +180,11 @@ */ #if defined(__LP64__) CTASSERT(offsetof(struct mbuf, m_dat) == 32); -CTASSERT(sizeof(struct pkthdr) == 56); +CTASSERT(sizeof(struct pkthdr) == 64); CTASSERT(sizeof(struct m_ext) == 160); #else CTASSERT(offsetof(struct mbuf, m_dat) == 24); -CTASSERT(sizeof(struct pkthdr) == 48); +CTASSERT(sizeof(struct pkthdr) == 52); #if defined(__powerpc__) && defined(BOOKE) /* PowerPC booke has 64-bit physical pointers. */ CTASSERT(sizeof(struct m_ext) == 184); diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c --- a/sys/kern/uipc_sockbuf.c +++ b/sys/kern/uipc_sockbuf.c @@ -54,6 +54,8 @@ #include #include +#include + /* * Function pointer set by the AIO routines so that the socket buffer code * can call back into the AIO module if it is loaded. @@ -963,17 +965,40 @@ static void sbappend_ktls_rx(struct sockbuf *sb, struct mbuf *m) { + struct ifnet *ifp = NULL; struct mbuf *n; + int flags = M_NOTREADY; SBLASTMBUFCHK(sb); + /* Mbuf chain must start with a packet header. */ + MPASS((m->m_flags & M_PKTHDR) != 0); + /* Remove all packet headers and mbuf tags to get a pure data chain. */ - m_demote(m, 1, 0); + for (n = m; n != NULL; n = n->m_next) { + if (n->m_flags & M_PKTHDR) { + ifp = m->m_pkthdr.leaf_rcvif; + if ((n->m_pkthdr.csum_flags & CSUM_TLS_MASK) == + CSUM_TLS_DECRYPTED) { + /* Mark all mbufs in this packet decrypted. */ + flags = M_NOTREADY | M_DECRYPTED; + } else { + flags = M_NOTREADY; + } + m_demote_pkthdr(n); + } + + n->m_flags &= M_DEMOTEFLAGS; + n->m_flags |= flags; + + MPASS((n->m_flags & M_NOTREADY) != 0); + } - for (n = m; n != NULL; n = n->m_next) - n->m_flags |= M_NOTREADY; sbcompress_ktls_rx(sb, m, sb->sb_mtlstail); ktls_check_rx(sb); + if (ifp != NULL && sb->sb_tls_info->rx_ifp != NULL && + sb->sb_tls_info->rx_ifp != ifp) + ktls_input_ifp_mismatch(sb, ifp); } #endif @@ -1427,7 +1452,8 @@ if (n && M_WRITABLE(n) && ((sb->sb_flags & SB_NOCOALESCE) == 0) && - !(n->m_flags & (M_EXTPG)) && + !((m->m_flags ^ n->m_flags) & M_DECRYPTED) && + !(n->m_flags & M_EXTPG) && m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ m->m_len <= M_TRAILINGSPACE(n)) { m_copydata(m, 0, m->m_len, mtodo(n, n->m_len)); diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -1836,6 +1836,7 @@ struct lagg_snd_tag *lst; struct lagg_port *lp; struct ifnet *lp_ifp; + struct m_snd_tag *mst; int error; switch (params->hdr.type) { @@ -1851,6 +1852,10 @@ case IF_SND_TAG_TYPE_TLS: sw = &lagg_snd_tag_tls_sw; break; + case IF_SND_TAG_TYPE_TLS_RX: + /* Return tag from port interface directly. */ + sw = NULL; + break; #ifdef RATELIMIT case IF_SND_TAG_TYPE_TLS_RATE_LIMIT: sw = &lagg_snd_tag_tls_rl_sw; @@ -1876,22 +1881,30 @@ if_ref(lp_ifp); NET_EPOCH_EXIT(et); - lst = malloc(sizeof(*lst), M_LAGG, M_NOWAIT); - if (lst == NULL) { - if_rele(lp_ifp); - return (ENOMEM); - } + if (sw != NULL) { + lst = malloc(sizeof(*lst), M_LAGG, M_NOWAIT); + if (lst == NULL) { + if_rele(lp_ifp); + return (ENOMEM); + } + } else + lst = NULL; - error = m_snd_tag_alloc(lp_ifp, params, &lst->tag); + error = m_snd_tag_alloc(lp_ifp, params, &mst); if_rele(lp_ifp); if (error) { free(lst, M_LAGG); return (error); } - m_snd_tag_init(&lst->com, ifp, sw); + if (sw != NULL) { + m_snd_tag_init(&lst->com, ifp, sw); + lst->tag = mst; + + *ppmt = &lst->com; + } else + *ppmt = mst; - *ppmt = &lst->com; return (0); } diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c --- a/sys/net/if_vlan.c +++ b/sys/net/if_vlan.c @@ -2204,8 +2204,12 @@ struct vlan_snd_tag *vst; struct ifvlan *ifv; struct ifnet *parent; + struct m_snd_tag *mst; int error; + NET_EPOCH_ENTER(et); + ifv = ifp->if_softc; + switch (params->hdr.type) { #ifdef RATELIMIT case IF_SND_TAG_TYPE_UNLIMITED: @@ -2219,6 +2223,12 @@ case IF_SND_TAG_TYPE_TLS: sw = &vlan_snd_tag_tls_sw; break; + case IF_SND_TAG_TYPE_TLS_RX: + sw = NULL; + if (params->tls_rx.vlan_id != 0) + goto failure; + params->tls_rx.vlan_id = ifv->ifv_vid; + break; #ifdef RATELIMIT case IF_SND_TAG_TYPE_TLS_RATE_LIMIT: sw = &vlan_snd_tag_tls_rl_sw; @@ -2226,39 +2236,46 @@ #endif #endif default: - return (EOPNOTSUPP); + goto failure; } - NET_EPOCH_ENTER(et); - ifv = ifp->if_softc; if (ifv->ifv_trunk != NULL) parent = PARENT(ifv); else parent = NULL; - if (parent == NULL) { - NET_EPOCH_EXIT(et); - return (EOPNOTSUPP); - } + if (parent == NULL) + goto failure; if_ref(parent); NET_EPOCH_EXIT(et); - vst = malloc(sizeof(*vst), M_VLAN, M_NOWAIT); - if (vst == NULL) { - if_rele(parent); - return (ENOMEM); - } + if (sw != NULL) { + vst = malloc(sizeof(*vst), M_VLAN, M_NOWAIT); + if (vst == NULL) { + if_rele(parent); + return (ENOMEM); + } + } else + vst = NULL; - error = m_snd_tag_alloc(parent, params, &vst->tag); + error = m_snd_tag_alloc(parent, params, &mst); if_rele(parent); if (error) { free(vst, M_VLAN); return (error); } - m_snd_tag_init(&vst->com, ifp, sw); + if (sw != NULL) { + m_snd_tag_init(&vst->com, ifp, sw); + vst->tag = mst; + + *ppmt = &vst->com; + } else + *ppmt = mst; - *ppmt = &vst->com; return (0); +failure: + NET_EPOCH_EXIT(et); + return (EOPNOTSUPP); } static struct m_snd_tag * diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h --- a/sys/sys/ktls.h +++ b/sys/sys/ktls.h @@ -187,7 +187,12 @@ struct task reset_tag_task; struct task disable_ifnet_task; - struct inpcb *inp; + union { + struct inpcb *inp; /* Used by transmit tasks. */ + struct socket *so; /* Used by receive task. */ + }; + struct ifnet *rx_ifp; + u_short rx_vlan_id; bool reset_pending; bool disable_ifnet_pending; bool sync_dispatch; @@ -200,7 +205,14 @@ extern unsigned int ktls_ifnet_max_rexmit_pct; +typedef enum { + KTLS_MBUF_CRYPTO_ST_MIXED = 0, + KTLS_MBUF_CRYPTO_ST_ENCRYPTED = 1, + KTLS_MBUF_CRYPTO_ST_DECRYPTED = -1, +} ktls_mbuf_crypto_st_t; + void ktls_check_rx(struct sockbuf *sb); +ktls_mbuf_crypto_st_t ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len); void ktls_disable_ifnet(void *arg); int ktls_enable_rx(struct socket *so, struct tls_enable *en); int ktls_enable_tx(struct socket *so, struct tls_enable *en); @@ -215,6 +227,7 @@ int ktls_set_tx_mode(struct socket *so, int mode); int ktls_get_tx_mode(struct socket *so, int *modep); int ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq); +void ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp); int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls); #ifdef RATELIMIT int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate); diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -164,6 +164,13 @@ uint16_t rcvgen; /* ... and generation count */ }; }; + union { + struct ifnet *leaf_rcvif; /* leaf rcv interface */ + struct { + uint16_t leaf_rcvidx; /* leaf rcv interface index ... */ + uint16_t leaf_rcvgen; /* ... and generation count */ + }; + }; SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */ int32_t len; /* total packet length */ diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h --- a/sys/sys/sockbuf.h +++ b/sys/sys/sockbuf.h @@ -53,6 +53,7 @@ #define SB_STOP 0x1000 /* backpressure indicator */ #define SB_AIO_RUNNING 0x2000 /* AIO operation running */ #define SB_TLS_IFNET 0x4000 /* has used / is using ifnet KTLS */ +#define SB_TLS_RX_RESYNC 0x8000 /* KTLS RX lost HW sync */ #define SBS_CANTSENDMORE 0x0010 /* can't send more data to peer */ #define SBS_CANTRCVMORE 0x0020 /* can't receive more data from peer */