D32356.id106364.diff
No OneTemporary
Actions

Size

24 KB

Referenced Files

None

Subscribers

None

D32356.id106364.diff
View Options

	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	@@ -319,6 +319,7 @@
	M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
	}
	mb->m_pkthdr.rcvif = ifp;
	+ mb->m_pkthdr.leaf_rcvif = ifp;

	if (cqe_is_tunneled(cqe)) {
	/*
	@@ -598,6 +599,7 @@
	memset(mb->m_data, 255, 14);
	mb->m_data[14] = rq->ix;
	mb->m_pkthdr.rcvif = rq->ifp;
	+ mb->m_pkthdr.leaf_rcvif = rq->ifp;
	rq->ifp->if_input(rq->ifp, mb);
	}
	#endif
	diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
	--- a/sys/kern/kern_mbuf.c
	+++ b/sys/kern/kern_mbuf.c
	@@ -1645,12 +1645,21 @@
	gen = m->m_pkthdr.rcvif->if_idxgen;
	m->m_pkthdr.rcvidx = idx;
	m->m_pkthdr.rcvgen = gen;
	+ if (m->m_pkthdr.leaf_rcvif != NULL) {
	+ idx = m->m_pkthdr.leaf_rcvif->if_index;
	+ gen = m->m_pkthdr.leaf_rcvif->if_idxgen;
	+ } else {
	+ idx = -1;
	+ gen = 0;
	+ }
	+ m->m_pkthdr.leaf_rcvidx = idx;
	+ m->m_pkthdr.leaf_rcvgen = gen;
	}

	struct ifnet *
	m_rcvif_restore(struct mbuf *m)
	{
	- struct ifnet *ifp;
	+ struct ifnet ifp, leaf_ifp;

	M_ASSERTPKTHDR(m);
	NET_EPOCH_ASSERT();
	@@ -1659,6 +1668,15 @@
	if (ifp == NULL \|\| (ifp->if_flags & IFF_DYING))
	return (NULL);

	+ if (m->m_pkthdr.leaf_rcvidx == (u_short)-1)
	+ leaf_ifp = NULL;
	+ else {
	+ leaf_ifp = ifnet_byindexgen(m->m_pkthdr.leaf_rcvidx,
	+ m->m_pkthdr.leaf_rcvgen);
	+ if (leaf_ifp != NULL && (leaf_ifp->if_flags & IFF_DYING))
	+ leaf_ifp = NULL;
	+ }
	+
	return (m->m_pkthdr.rcvif = ifp);
	}

	diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
	--- a/sys/kern/uipc_ktls.c
	+++ b/sys/kern/uipc_ktls.c
	@@ -299,6 +299,7 @@

	static void ktls_cleanup(struct ktls_session *tls);
	#if defined(INET) \|\| defined(INET6)
	+static void ktls_reset_receive_tag(void *context, int pending);
	static void ktls_reset_send_tag(void *context, int pending);
	#endif
	static void ktls_work_thread(void *ctx);
	@@ -503,7 +504,7 @@
	#if defined(INET) \|\| defined(INET6)
	static int
	ktls_create_session(struct socket so, struct tls_enable en,
	- struct ktls_session **tlsp)
	+ struct ktls_session **tlsp, int direction)
	{
	struct ktls_session *tls;
	int error;
	@@ -619,7 +620,10 @@
	counter_u64_add(ktls_offload_active, 1);

	refcount_init(&tls->refcount, 1);
	- TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
	+ if (direction == KTLS_RX)
	+ TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls);
	+ else
	+ TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);

	tls->wq_index = ktls_get_cpu(so);

	@@ -745,7 +749,7 @@
	}

	static struct ktls_session *
	-ktls_clone_session(struct ktls_session *tls)
	+ktls_clone_session(struct ktls_session *tls, int direction)
	{
	struct ktls_session *tls_new;

	@@ -754,7 +758,12 @@
	counter_u64_add(ktls_offload_active, 1);

	refcount_init(&tls_new->refcount, 1);
	- TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new);
	+ if (direction == KTLS_RX)
	+ TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_receive_tag,
	+ tls_new);
	+ else
	+ TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag,
	+ tls_new);

	/* Copy fields from existing session. */
	tls_new->params = tls->params;
	@@ -810,6 +819,8 @@
	}
	if (tls->snd_tag != NULL)
	m_snd_tag_rele(tls->snd_tag);
	+ if (tls->rx_ifp != NULL)
	+ if_rele(tls->rx_ifp);
	break;
	#ifdef TCP_OFFLOAD
	case TCP_TLS_MODE_TOE:
	@@ -983,28 +994,135 @@
	return (error);
	}

	+/*
	+ * Allocate an initial TLS receive tag for doing HW decryption of TLS
	+ * data.
	+ *
	+ * This function allocates a new TLS receive tag on whatever interface
	+ * the connection is currently routed over. If the connection ends up
	+ * using a different interface for receive this will get fixed up via
	+ * ktls_input_ifp_mismatch as future packets arrive.
	+ */
	static int
	-ktls_try_ifnet(struct socket so, struct ktls_session tls, bool force)
	+ktls_alloc_rcv_tag(struct inpcb inp, struct ktls_session tls,
	+ struct m_snd_tag **mstp)
	{
	- struct m_snd_tag *mst;
	+ union if_snd_tag_alloc_params params;
	+ struct ifnet *ifp;
	+ struct nhop_object *nh;
	int error;

	- error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
	- if (error == 0) {
	- tls->mode = TCP_TLS_MODE_IFNET;
	- tls->snd_tag = mst;
	- switch (tls->params.cipher_algorithm) {
	- case CRYPTO_AES_CBC:
	- counter_u64_add(ktls_ifnet_cbc, 1);
	- break;
	- case CRYPTO_AES_NIST_GCM_16:
	- counter_u64_add(ktls_ifnet_gcm, 1);
	- break;
	- case CRYPTO_CHACHA20_POLY1305:
	- counter_u64_add(ktls_ifnet_chacha20, 1);
	- break;
	+ if (!ktls_ocf_recrypt_supported(tls))
	+ return (ENXIO);
	+
	+ INP_RLOCK(inp);
	+ if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	+ INP_RUNLOCK(inp);
	+ return (ECONNRESET);
	+ }
	+ if (inp->inp_socket == NULL) {
	+ INP_RUNLOCK(inp);
	+ return (ECONNRESET);
	+ }
	+
	+ /*
	+ * Check administrative controls on ifnet TLS to determine if
	+ * ifnet TLS should be denied.
	+ */
	+ if (ktls_ifnet_permitted == 0) {
	+ INP_RUNLOCK(inp);
	+ return (ENXIO);
	+ }
	+
	+ /*
	+ * XXX: As with ktls_alloc_snd_tag, use the cached route in
	+ * the inpcb to find the interface.
	+ */
	+ nh = inp->inp_route.ro_nh;
	+ if (nh == NULL) {
	+ INP_RUNLOCK(inp);
	+ return (ENXIO);
	+ }
	+ ifp = nh->nh_ifp;
	+ if_ref(ifp);
	+ tls->rx_ifp = ifp;
	+
	+ params.hdr.type = IF_SND_TAG_TYPE_TLS_RX;
	+ params.hdr.flowid = inp->inp_flowid;
	+ params.hdr.flowtype = inp->inp_flowtype;
	+ params.hdr.numa_domain = inp->inp_numa_domain;
	+ params.tls_rx.inp = inp;
	+ params.tls_rx.tls = tls;
	+ params.tls_rx.vlan_id = 0;
	+
	+ INP_RUNLOCK(inp);
	+
	+ if (inp->inp_vflag & INP_IPV6) {
	+ if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0) {
	+ error = EOPNOTSUPP;
	+ goto out;
	+ }
	+ } else {
	+ if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0) {
	+ error = EOPNOTSUPP;
	+ goto out;
	}
	}
	+ error = m_snd_tag_alloc(ifp, &params, mstp);
	+
	+ /*
	+ * If this connection is over a vlan, vlan_snd_tag_alloc
	+ * rewrites vlan_id with the saved interface. Save the VLAN
	+ * ID for use in ktls_reset_receive_tag which allocates new
	+ * receive tags directly from the leaf interface bypassing
	+ * if_vlan.
	+ */
	+ if (error == 0)
	+ tls->rx_vlan_id = params.tls_rx.vlan_id;
	+out:
	+ return (error);
	+}
	+
	+static int
	+ktls_try_ifnet(struct socket so, struct ktls_session tls, int direction,
	+ bool force)
	+{
	+ struct m_snd_tag *mst;
	+ int error;
	+
	+ switch (direction) {
	+ case KTLS_TX:
	+ error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
	+ if (__predict_false(error != 0))
	+ goto done;
	+ break;
	+ case KTLS_RX:
	+ KASSERT(!force, ("%s: forced receive tag", __func__));
	+ error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst);
	+ if (__predict_false(error != 0))
	+ goto done;
	+ break;
	+ default:
	+ __assert_unreachable();
	+ }
	+
	+ tls->mode = TCP_TLS_MODE_IFNET;
	+ tls->snd_tag = mst;
	+
	+ switch (tls->params.cipher_algorithm) {
	+ case CRYPTO_AES_CBC:
	+ counter_u64_add(ktls_ifnet_cbc, 1);
	+ break;
	+ case CRYPTO_AES_NIST_GCM_16:
	+ counter_u64_add(ktls_ifnet_gcm, 1);
	+ break;
	+ case CRYPTO_CHACHA20_POLY1305:
	+ counter_u64_add(ktls_ifnet_chacha20, 1);
	+ break;
	+ default:
	+ break;
	+ }
	+done:
	return (error);
	}

	@@ -1185,7 +1303,7 @@
	if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
	return (ENOTSUP);

	- error = ktls_create_session(so, en, &tls);
	+ error = ktls_create_session(so, en, &tls, KTLS_RX);
	if (error)
	return (error);

	@@ -1206,10 +1324,13 @@
	ktls_check_rx(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);

	+ /* Prefer TOE -> ifnet TLS -> software TLS. */
	#ifdef TCP_OFFLOAD
	error = ktls_try_toe(so, tls, KTLS_RX);
	if (error)
	#endif
	+ error = ktls_try_ifnet(so, tls, KTLS_RX, false);
	+ if (error)
	ktls_use_sw(tls);

	counter_u64_add(ktls_offload_total, 1);
	@@ -1252,7 +1373,7 @@
	if (mb_use_ext_pgs == 0)
	return (ENXIO);

	- error = ktls_create_session(so, en, &tls);
	+ error = ktls_create_session(so, en, &tls, KTLS_TX);
	if (error)
	return (error);

	@@ -1261,7 +1382,7 @@
	error = ktls_try_toe(so, tls, KTLS_TX);
	if (error)
	#endif
	- error = ktls_try_ifnet(so, tls, false);
	+ error = ktls_try_ifnet(so, tls, KTLS_TX, false);
	if (error)
	error = ktls_try_sw(so, tls, KTLS_TX);

	@@ -1418,10 +1539,10 @@
	SOCKBUF_UNLOCK(&so->so_snd);
	INP_WUNLOCK(inp);

	- tls_new = ktls_clone_session(tls);
	+ tls_new = ktls_clone_session(tls, KTLS_TX);

	if (mode == TCP_TLS_MODE_IFNET)
	- error = ktls_try_ifnet(so, tls_new, true);
	+ error = ktls_try_ifnet(so, tls_new, KTLS_TX, true);
	else
	error = ktls_try_sw(so, tls_new, KTLS_TX);
	if (error) {
	@@ -1479,6 +1600,89 @@
	return (0);
	}

	+/*
	+ * Try to allocate a new TLS receive tag. This task is scheduled when
	+ * sbappend_ktls_rx detects an input path change. If a new tag is
	+ * allocated, replace the tag in the TLS session. If a new tag cannot
	+ * be allocated, let the session fall back to software decryption.
	+ */
	+static void
	+ktls_reset_receive_tag(void *context, int pending)
	+{
	+ union if_snd_tag_alloc_params params;
	+ struct ktls_session *tls;
	+ struct m_snd_tag *mst;
	+ struct inpcb *inp;
	+ struct ifnet *ifp;
	+ struct socket *so;
	+ int error;
	+
	+ MPASS(pending == 1);
	+
	+ tls = context;
	+ so = tls->so;
	+ inp = so->so_pcb;
	+ ifp = NULL;
	+
	+ INP_RLOCK(inp);
	+ if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	+ INP_RUNLOCK(inp);
	+ goto out;
	+ }
	+
	+ SOCKBUF_LOCK(&so->so_rcv);
	+ m_snd_tag_rele(tls->snd_tag);
	+ tls->snd_tag = NULL;
	+
	+ ifp = tls->rx_ifp;
	+ if_ref(ifp);
	+ SOCKBUF_UNLOCK(&so->so_rcv);
	+
	+ params.hdr.type = IF_SND_TAG_TYPE_TLS_RX;
	+ params.hdr.flowid = inp->inp_flowid;
	+ params.hdr.flowtype = inp->inp_flowtype;
	+ params.hdr.numa_domain = inp->inp_numa_domain;
	+ params.tls_rx.inp = inp;
	+ params.tls_rx.tls = tls;
	+ params.tls_rx.vlan_id = tls->rx_vlan_id;
	+ INP_RUNLOCK(inp);
	+
	+ if (inp->inp_vflag & INP_IPV6) {
	+ if ((ifp->if_capenable2 & IFCAP2_RXTLS6) == 0)
	+ goto out;
	+ } else {
	+ if ((ifp->if_capenable2 & IFCAP2_RXTLS4) == 0)
	+ goto out;
	+ }
	+
	+ error = m_snd_tag_alloc(ifp, &params, &mst);
	+ if (error == 0) {
	+ SOCKBUF_LOCK(&so->so_rcv);
	+ tls->snd_tag = mst;
	+ SOCKBUF_UNLOCK(&so->so_rcv);
	+
	+ counter_u64_add(ktls_ifnet_reset, 1);
	+ } else {
	+ /*
	+ * Just fall back to software decryption if a tag
	+ * cannot be allocated leaving the connection intact.
	+ * If a future input path change switches to another
	+ * interface this connection will resume ifnet TLS.
	+ */
	+ counter_u64_add(ktls_ifnet_reset_failed, 1);
	+ }
	+
	+out:
	+ mtx_pool_lock(mtxpool_sleep, tls);
	+ tls->reset_pending = false;
	+ mtx_pool_unlock(mtxpool_sleep, tls);
	+
	+ if (ifp != NULL)
	+ if_rele(ifp);
	+ sorele(so);
	+ ktls_free(tls);
	+}
	+
	/*
	* Try to allocate a new TLS send tag. This task is scheduled when
	* ip_output detects a route change while trying to transmit a packet
	@@ -1565,6 +1769,37 @@
	ktls_free(tls);
	}

	+void
	+ktls_input_ifp_mismatch(struct sockbuf sb, struct ifnet ifp)
	+{
	+ struct ktls_session *tls;
	+ struct socket *so;
	+
	+ SOCKBUF_LOCK_ASSERT(sb);
	+ KASSERT(sb->sb_flags & SB_TLS_RX, ("%s: sockbuf %p isn't TLS RX",
	+ __func__, sb));
	+ so = __containerof(sb, struct socket, so_rcv);
	+
	+ tls = sb->sb_tls_info;
	+ if_rele(tls->rx_ifp);
	+ if_ref(ifp);
	+ tls->rx_ifp = ifp;
	+
	+ /*
	+ * See if we should schedule a task to update the receive tag for
	+ * this session.
	+ */
	+ mtx_pool_lock(mtxpool_sleep, tls);
	+ if (!tls->reset_pending) {
	+ (void) ktls_hold(tls);
	+ soref(so);
	+ tls->so = so;
	+ tls->reset_pending = true;
	+ taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);
	+ }
	+ mtx_pool_unlock(mtxpool_sleep, tls);
	+}
	+
	int
	ktls_output_eagain(struct inpcb inp, struct ktls_session tls)
	{
	@@ -1615,10 +1850,11 @@
	return (0);
	}

	- MPASS(tls->snd_tag != NULL);
	- MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT);
	-
	mst = tls->snd_tag;
	+
	+ MPASS(mst != NULL);
	+ MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT);
	+
	return (mst->sw->snd_tag_modify(mst, &params));
	}
	#endif
	@@ -1907,7 +2143,7 @@
	return (NULL);
	}
	}
	- n->m_flags \|= M_NOTREADY;
	+ n->m_flags \|= (m->m_flags & (M_NOTREADY \| M_DECRYPTED));

	/* Store remainder in 'n'. */
	n->m_len = m->m_len - remain;
	@@ -1992,6 +2228,86 @@
	return (0);
	}

	+/*
	+ * Check if a mbuf chain is fully decrypted at the given offset and
	+ * length. Returns KTLS_MBUF_CRYPTO_ST_DECRYPTED if all data is
	+ * decrypted. KTLS_MBUF_CRYPTO_ST_MIXED if there is a mix of encrypted
	+ * and decrypted data. Else KTLS_MBUF_CRYPTO_ST_ENCRYPTED if all data
	+ * is encrypted.
	+ */
	+ktls_mbuf_crypto_st_t
	+ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len)
	+{
	+ int m_flags_ored = 0;
	+ int m_flags_anded = -1;
	+
	+ for (; mb != NULL; mb = mb->m_next) {
	+ if (offset < mb->m_len)
	+ break;
	+ offset -= mb->m_len;
	+ }
	+ offset += len;
	+
	+ for (; mb != NULL; mb = mb->m_next) {
	+ m_flags_ored \|= mb->m_flags;
	+ m_flags_anded &= mb->m_flags;
	+
	+ if (offset <= mb->m_len)
	+ break;
	+ offset -= mb->m_len;
	+ }
	+ MPASS(mb != NULL \|\| offset == 0);
	+
	+ if ((m_flags_ored ^ m_flags_anded) & M_DECRYPTED)
	+ return (KTLS_MBUF_CRYPTO_ST_MIXED);
	+ else
	+ return ((m_flags_ored & M_DECRYPTED) ?
	+ KTLS_MBUF_CRYPTO_ST_DECRYPTED :
	+ KTLS_MBUF_CRYPTO_ST_ENCRYPTED);
	+}
	+
	+/*
	+ * ktls_resync_ifnet - get HW TLS RX back on track after packet loss
	+ */
	+static int
	+ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num)
	+{
	+ union if_snd_tag_modify_params params;
	+ struct m_snd_tag *mst;
	+ struct inpcb *inp;
	+ struct tcpcb *tp;
	+
	+ mst = so->so_rcv.sb_tls_info->snd_tag;
	+ if (__predict_false(mst == NULL))
	+ return (EINVAL);
	+
	+ inp = sotoinpcb(so);
	+ if (__predict_false(inp == NULL))
	+ return (EINVAL);
	+
	+ INP_RLOCK(inp);
	+ if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	+ INP_RUNLOCK(inp);
	+ return (ECONNRESET);
	+ }
	+
	+ tp = intotcpcb(inp);
	+ MPASS(tp != NULL);
	+
	+ /* Get the TCP sequence number of the next valid TLS header. */
	+ SOCKBUF_LOCK(&so->so_rcv);
	+ params.tls_rx.tls_hdr_tcp_sn =
	+ tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len;
	+ params.tls_rx.tls_rec_length = tls_len;
	+ params.tls_rx.tls_seq_number = tls_rcd_num;
	+ SOCKBUF_UNLOCK(&so->so_rcv);
	+
	+ INP_RUNLOCK(inp);
	+
	+ MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RX);
	+ return (mst->sw->snd_tag_modify(mst, &params));
	+}
	+
	static void
	ktls_decrypt(struct socket *so)
	{
	@@ -2001,6 +2317,7 @@
	struct tls_record_layer *hdr;
	struct tls_get_record tgr;
	struct mbuf control, data, *m;
	+ ktls_mbuf_crypto_st_t state;
	uint64_t seqno;
	int error, remain, tls_len, trail_len;
	bool tls13;
	@@ -2073,13 +2390,46 @@
	SBCHECK(sb);
	SOCKBUF_UNLOCK(sb);

	- error = ktls_ocf_decrypt(tls, hdr, data, seqno, &trail_len);
	- if (error == 0) {
	- if (tls13)
	+ /* get crypto state for this TLS record */
	+ state = ktls_mbuf_crypto_state(data, 0, tls_len);
	+
	+ switch (state) {
	+ case KTLS_MBUF_CRYPTO_ST_MIXED:
	+ error = ktls_ocf_recrypt(tls, hdr, data, seqno);
	+ if (error)
	+ break;
	+ /* FALLTHROUGH */
	+ case KTLS_MBUF_CRYPTO_ST_ENCRYPTED:
	+ error = ktls_ocf_decrypt(tls, hdr, data, seqno,
	+ &trail_len);
	+ if (__predict_true(error == 0)) {
	+ if (tls13) {
	+ error = tls13_find_record_type(tls, data,
	+ tls_len, &trail_len, &record_type);
	+ } else {
	+ record_type = hdr->tls_type;
	+ }
	+ }
	+ break;
	+ case KTLS_MBUF_CRYPTO_ST_DECRYPTED:
	+ /*
	+ * NIC TLS is only supported for AEAD
	+ * ciphersuites which used a fixed sized
	+ * trailer.
	+ */
	+ if (tls13) {
	+ trail_len = tls->params.tls_tlen - 1;
	error = tls13_find_record_type(tls, data,
	tls_len, &trail_len, &record_type);
	- else
	+ } else {
	+ trail_len = tls->params.tls_tlen;
	+ error = 0;
	record_type = hdr->tls_type;
	+ }
	+ break;
	+ default:
	+ error = EINVAL;
	+ break;
	}
	if (error) {
	counter_u64_add(ktls_offload_failed_crypto, 1);
	@@ -2160,19 +2510,31 @@
	remain = be16toh(tgr.tls_length);
	m = data;
	for (m = data; remain > m->m_len; m = m->m_next) {
	- m->m_flags &= ~M_NOTREADY;
	+ m->m_flags &= ~(M_NOTREADY \| M_DECRYPTED);
	remain -= m->m_len;
	}
	m->m_len = remain;
	m_freem(m->m_next);
	m->m_next = NULL;
	- m->m_flags &= ~M_NOTREADY;
	+ m->m_flags &= ~(M_NOTREADY \| M_DECRYPTED);

	/* Set EOR on the final mbuf. */
	m->m_flags \|= M_EOR;
	}

	sbappendcontrol_locked(sb, data, control, 0);
	+
	+ if (__predict_false(state != KTLS_MBUF_CRYPTO_ST_DECRYPTED)) {
	+ sb->sb_flags \|= SB_TLS_RX_RESYNC;
	+ SOCKBUF_UNLOCK(sb);
	+ ktls_resync_ifnet(so, tls_len, seqno);
	+ SOCKBUF_LOCK(sb);
	+ } else if (__predict_false(sb->sb_flags & SB_TLS_RX_RESYNC)) {
	+ sb->sb_flags &= ~SB_TLS_RX_RESYNC;
	+ SOCKBUF_UNLOCK(sb);
	+ ktls_resync_ifnet(so, 0, seqno);
	+ SOCKBUF_LOCK(sb);
	+ }
	}

	sb->sb_flags &= ~SB_TLS_RX_RUNNING;
	diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
	--- a/sys/kern/uipc_mbuf.c
	+++ b/sys/kern/uipc_mbuf.c
	@@ -180,11 +180,11 @@
	*/
	#if defined(__LP64__)
	CTASSERT(offsetof(struct mbuf, m_dat) == 32);
	-CTASSERT(sizeof(struct pkthdr) == 56);
	+CTASSERT(sizeof(struct pkthdr) == 64);
	CTASSERT(sizeof(struct m_ext) == 160);
	#else
	CTASSERT(offsetof(struct mbuf, m_dat) == 24);
	-CTASSERT(sizeof(struct pkthdr) == 48);
	+CTASSERT(sizeof(struct pkthdr) == 52);
	#if defined(__powerpc__) && defined(BOOKE)
	/* PowerPC booke has 64-bit physical pointers. */
	CTASSERT(sizeof(struct m_ext) == 184);
	diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
	--- a/sys/kern/uipc_sockbuf.c
	+++ b/sys/kern/uipc_sockbuf.c
	@@ -54,6 +54,8 @@
	#include <sys/sx.h>
	#include <sys/sysctl.h>

	+#include <netinet/in.h>
	+
	/*
	* Function pointer set by the AIO routines so that the socket buffer code
	* can call back into the AIO module if it is loaded.
	@@ -963,17 +965,40 @@
	static void
	sbappend_ktls_rx(struct sockbuf sb, struct mbuf m)
	{
	+ struct ifnet *ifp;
	struct mbuf *n;
	+ int flags = M_NOTREADY;

	SBLASTMBUFCHK(sb);

	+ /* Mbuf chain must start with a packet header. */
	+ MPASS((m->m_flags & M_PKTHDR) != 0);
	+
	/* Remove all packet headers and mbuf tags to get a pure data chain. */
	- m_demote(m, 1, 0);
	+ for (n = m; n != NULL; n = n->m_next) {
	+ if (n->m_flags & M_PKTHDR) {
	+ ifp = m->m_pkthdr.leaf_rcvif;
	+ if ((n->m_pkthdr.csum_flags & CSUM_TLS_MASK) ==
	+ CSUM_TLS_DECRYPTED) {
	+ /* Mark all mbufs in this packet decrypted. */
	+ flags = M_NOTREADY \| M_DECRYPTED;
	+ } else {
	+ flags = M_NOTREADY;
	+ }
	+ m_demote_pkthdr(n);
	+ }
	+
	+ n->m_flags &= M_DEMOTEFLAGS;
	+ n->m_flags \|= flags;
	+
	+ MPASS((n->m_flags & M_NOTREADY) != 0);
	+ }

	- for (n = m; n != NULL; n = n->m_next)
	- n->m_flags \|= M_NOTREADY;
	sbcompress_ktls_rx(sb, m, sb->sb_mtlstail);
	ktls_check_rx(sb);
	+ if (ifp != NULL && sb->sb_tls_info->rx_ifp != NULL &&
	+ sb->sb_tls_info->rx_ifp != ifp)
	+ ktls_input_ifp_mismatch(sb, ifp);
	}
	#endif

	@@ -1427,7 +1452,8 @@
	if (n &&
	M_WRITABLE(n) &&
	((sb->sb_flags & SB_NOCOALESCE) == 0) &&
	- !(n->m_flags & (M_EXTPG)) &&
	+ !((m->m_flags ^ n->m_flags) & M_DECRYPTED) &&
	+ !(n->m_flags & M_EXTPG) &&
	m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
	m->m_len <= M_TRAILINGSPACE(n)) {
	m_copydata(m, 0, m->m_len, mtodo(n, n->m_len));
	diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c
	--- a/sys/net/if_lagg.c
	+++ b/sys/net/if_lagg.c
	@@ -1836,6 +1836,7 @@
	struct lagg_snd_tag *lst;
	struct lagg_port *lp;
	struct ifnet *lp_ifp;
	+ struct m_snd_tag *mst;
	int error;

	switch (params->hdr.type) {
	@@ -1851,6 +1852,10 @@
	case IF_SND_TAG_TYPE_TLS:
	sw = &lagg_snd_tag_tls_sw;
	break;
	+ case IF_SND_TAG_TYPE_TLS_RX:
	+ /* Return tag from port interface directly. */
	+ sw = NULL;
	+ break;
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
	sw = &lagg_snd_tag_tls_rl_sw;
	@@ -1876,22 +1881,30 @@
	if_ref(lp_ifp);
	NET_EPOCH_EXIT(et);

	- lst = malloc(sizeof(*lst), M_LAGG, M_NOWAIT);
	- if (lst == NULL) {
	- if_rele(lp_ifp);
	- return (ENOMEM);
	- }
	+ if (sw != NULL) {
	+ lst = malloc(sizeof(*lst), M_LAGG, M_NOWAIT);
	+ if (lst == NULL) {
	+ if_rele(lp_ifp);
	+ return (ENOMEM);
	+ }
	+ } else
	+ lst = NULL;

	- error = m_snd_tag_alloc(lp_ifp, params, &lst->tag);
	+ error = m_snd_tag_alloc(lp_ifp, params, &mst);
	if_rele(lp_ifp);
	if (error) {
	free(lst, M_LAGG);
	return (error);
	}

	- m_snd_tag_init(&lst->com, ifp, sw);
	+ if (sw != NULL) {
	+ m_snd_tag_init(&lst->com, ifp, sw);
	+ lst->tag = mst;
	+
	+ *ppmt = &lst->com;
	+ } else
	+ *ppmt = mst;

	- *ppmt = &lst->com;
	return (0);
	}

	diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c
	--- a/sys/net/if_vlan.c
	+++ b/sys/net/if_vlan.c
	@@ -2204,8 +2204,12 @@
	struct vlan_snd_tag *vst;
	struct ifvlan *ifv;
	struct ifnet *parent;
	+ struct m_snd_tag *mst;
	int error;

	+ NET_EPOCH_ENTER(et);
	+ ifv = ifp->if_softc;
	+
	switch (params->hdr.type) {
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_UNLIMITED:
	@@ -2219,6 +2223,12 @@
	case IF_SND_TAG_TYPE_TLS:
	sw = &vlan_snd_tag_tls_sw;
	break;
	+ case IF_SND_TAG_TYPE_TLS_RX:
	+ sw = NULL;
	+ if (params->tls_rx.vlan_id != 0)
	+ goto failure;
	+ params->tls_rx.vlan_id = ifv->ifv_vid;
	+ break;
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
	sw = &vlan_snd_tag_tls_rl_sw;
	@@ -2226,39 +2236,46 @@
	#endif
	#endif
	default:
	- return (EOPNOTSUPP);
	+ goto failure;
	}

	- NET_EPOCH_ENTER(et);
	- ifv = ifp->if_softc;
	if (ifv->ifv_trunk != NULL)
	parent = PARENT(ifv);
	else
	parent = NULL;
	- if (parent == NULL) {
	- NET_EPOCH_EXIT(et);
	- return (EOPNOTSUPP);
	- }
	+ if (parent == NULL)
	+ goto failure;
	if_ref(parent);
	NET_EPOCH_EXIT(et);

	- vst = malloc(sizeof(*vst), M_VLAN, M_NOWAIT);
	- if (vst == NULL) {
	- if_rele(parent);
	- return (ENOMEM);
	- }
	+ if (sw != NULL) {
	+ vst = malloc(sizeof(*vst), M_VLAN, M_NOWAIT);
	+ if (vst == NULL) {
	+ if_rele(parent);
	+ return (ENOMEM);
	+ }
	+ } else
	+ vst = NULL;

	- error = m_snd_tag_alloc(parent, params, &vst->tag);
	+ error = m_snd_tag_alloc(parent, params, &mst);
	if_rele(parent);
	if (error) {
	free(vst, M_VLAN);
	return (error);
	}

	- m_snd_tag_init(&vst->com, ifp, sw);
	+ if (sw != NULL) {
	+ m_snd_tag_init(&vst->com, ifp, sw);
	+ vst->tag = mst;
	+
	+ *ppmt = &vst->com;
	+ } else
	+ *ppmt = mst;

	- *ppmt = &vst->com;
	return (0);
	+failure:
	+ NET_EPOCH_EXIT(et);
	+ return (EOPNOTSUPP);
	}

	static struct m_snd_tag *
	diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h
	--- a/sys/sys/ktls.h
	+++ b/sys/sys/ktls.h
	@@ -187,7 +187,12 @@

	struct task reset_tag_task;
	struct task disable_ifnet_task;
	- struct inpcb *inp;
	+ union {
	+ struct inpcb inp; / Used by transmit tasks. */
	+ struct socket so; / Used by receive task. */
	+ };
	+ struct ifnet *rx_ifp;
	+ u_short rx_vlan_id;
	bool reset_pending;
	bool disable_ifnet_pending;
	bool sync_dispatch;
	@@ -200,7 +205,14 @@

	extern unsigned int ktls_ifnet_max_rexmit_pct;

	+typedef enum {
	+ KTLS_MBUF_CRYPTO_ST_MIXED = 0,
	+ KTLS_MBUF_CRYPTO_ST_ENCRYPTED = 1,
	+ KTLS_MBUF_CRYPTO_ST_DECRYPTED = -1,
	+} ktls_mbuf_crypto_st_t;
	+
	void ktls_check_rx(struct sockbuf *sb);
	+ktls_mbuf_crypto_st_t ktls_mbuf_crypto_state(struct mbuf *mb, int offset, int len);
	void ktls_disable_ifnet(void *arg);
	int ktls_enable_rx(struct socket so, struct tls_enable en);
	int ktls_enable_tx(struct socket so, struct tls_enable en);
	@@ -215,6 +227,7 @@
	int ktls_set_tx_mode(struct socket *so, int mode);
	int ktls_get_tx_mode(struct socket so, int modep);
	int ktls_get_rx_sequence(struct inpcb inp, uint32_t tcpseq, uint64_t *tlsseq);
	+void ktls_input_ifp_mismatch(struct sockbuf sb, struct ifnet ifp);
	int ktls_output_eagain(struct inpcb inp, struct ktls_session tls);
	#ifdef RATELIMIT
	int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate);
	diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
	--- a/sys/sys/mbuf.h
	+++ b/sys/sys/mbuf.h
	@@ -164,6 +164,13 @@
	uint16_t rcvgen; /* ... and generation count */
	};
	};
	+ union {
	+ struct ifnet leaf_rcvif; / leaf rcv interface */
	+ struct {
	+ uint16_t leaf_rcvidx; /* leaf rcv interface index ... */
	+ uint16_t leaf_rcvgen; /* ... and generation count */
	+ };
	+ };
	SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
	int32_t len; /* total packet length */

	diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h
	--- a/sys/sys/sockbuf.h
	+++ b/sys/sys/sockbuf.h
	@@ -53,6 +53,7 @@
	#define SB_STOP 0x1000 /* backpressure indicator */
	#define SB_AIO_RUNNING 0x2000 /* AIO operation running */
	#define SB_TLS_IFNET 0x4000 /* has used / is using ifnet KTLS */
	+#define SB_TLS_RX_RESYNC 0x8000 /* KTLS RX lost HW sync */

	#define SBS_CANTSENDMORE 0x0010 /* can't send more data to peer */
	#define SBS_CANTRCVMORE 0x0020 /* can't receive more data from peer */

File Metadata

Mime Type: text/plain
Expires: Mon, Dec 22, 2:24 AM (10 h, 28 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 27131285
Default Alt Text: D32356.id106364.diff (24 KB)

D32356.id106364.diffNo OneTemporaryActions

D32356.id106364.diffView Options

File Metadata

Event Timeline

D32356.id106364.diff
No OneTemporary
Actions

D32356.id106364.diff
View Options