D57299.id179039.diff
No OneTemporary
Actions

Size

27 KB

Referenced Files

None

Subscribers

None

D57299.id179039.diff
View Options

	diff --git a/sys/dev/netmap/if_ptnet.c b/sys/dev/netmap/if_ptnet.c
	--- a/sys/dev/netmap/if_ptnet.c
	+++ b/sys/dev/netmap/if_ptnet.c
	@@ -1464,8 +1464,16 @@
	* two 8-bytes-wide writes. */
	memset(nmbuf, 0, PTNET_HDR_SIZE);
	if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
	- mhead = virtio_net_tx_offload(ifp, mhead, false,
	- vh);
	+ /*
	+ * Translate the CSUM_* flags in the mbuf to the
	+ * corresponding flags in the VirtIO header.
	+ *
	+ * Always indicate that ECN has not been
	+ * negotiated and VirtIO modern mode is not
	+ * used because bhyve does not do this.
	+ */
	+ virtio_net_tx_offload(ifp, &mhead, vh, false,
	+ false);
	if (unlikely(!mhead)) {
	/* Packet dropped because errors
	* occurred while preparing the vnet
	@@ -1842,7 +1850,14 @@
	}
	}

	- if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) {
	+ /*
	+ * Translate the VirtIO header flags to the corresponding
	+ * CSUM_* flags in the mbuf.
	+ */
	+ if (unlikely(have_vnet_hdr &&
	+ ((vh->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM \|
	+ VIRTIO_NET_HDR_F_DATA_VALID)) != 0) &&
	+ (virtio_net_rx_csum(mhead, vh) != 0))) {
	m_freem(mhead);
	nm_prlim(1, "Csum offload error: dropping");
	pq->stats.iqdrops ++;
	diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c
	--- a/sys/dev/virtio/network/if_vtnet.c
	+++ b/sys/dev/virtio/network/if_vtnet.c
	@@ -134,14 +134,6 @@
	static int vtnet_rxq_replace_buf(struct vtnet_rxq , struct mbuf , int);
	static int vtnet_rxq_enqueue_buf(struct vtnet_rxq , struct mbuf );
	static int vtnet_rxq_new_buf(struct vtnet_rxq *);
	-#if defined(INET) \|\| defined(INET6)
	-static void vtnet_rxq_csum_needs_csum(struct vtnet_rxq , struct mbuf ,
	- bool, int, struct virtio_net_hdr *);
	-static void vtnet_rxq_csum_data_valid(struct vtnet_rxq , struct mbuf ,
	- int);
	-static int vtnet_rxq_csum(struct vtnet_rxq , struct mbuf ,
	- struct virtio_net_hdr *);
	-#endif
	static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int);
	static void vtnet_rxq_discard_buf(struct vtnet_rxq , struct mbuf );
	static int vtnet_rxq_merged_eof(struct vtnet_rxq , struct mbuf , int);
	@@ -156,13 +148,6 @@
	static int vtnet_txq_below_threshold(struct vtnet_txq *);
	static int vtnet_txq_notify(struct vtnet_txq *);
	static void vtnet_txq_free_mbufs(struct vtnet_txq *);
	-static int vtnet_txq_offload_ctx(struct vtnet_txq , struct mbuf ,
	- int , int , int *);
	-static int vtnet_txq_offload_tso(struct vtnet_txq , struct mbuf , int,
	- int, struct virtio_net_hdr *);
	-static struct mbuf *
	- vtnet_txq_offload(struct vtnet_txq , struct mbuf ,
	- struct virtio_net_hdr *);
	static int vtnet_txq_enqueue_buf(struct vtnet_txq , struct mbuf *,
	struct vtnet_tx_header *);
	static int vtnet_txq_encap(struct vtnet_txq , struct mbuf *, int);
	@@ -1774,124 +1759,6 @@
	return (error);
	}

	-#if defined(INET) \|\| defined(INET6)
	-static void
	-vtnet_rxq_csum_needs_csum(struct vtnet_rxq rxq, struct mbuf m, bool isipv6,
	- int protocol, struct virtio_net_hdr *hdr)
	-{
	- /*
	- * The packet is likely from another VM on the same host or from the
	- * host that itself performed checksum offloading so Tx/Rx is basically
	- * a memcpy and the checksum has little value so far.
	- */
	-
	- KASSERT(protocol == IPPROTO_TCP \|\| protocol == IPPROTO_UDP,
	- ("%s: unsupported IP protocol %d", __func__, protocol));
	-
	- /*
	- * Just forward the order to compute the checksum by setting
	- * the corresponding mbuf flag (e.g., CSUM_TCP).
	- */
	- switch (protocol) {
	- case IPPROTO_TCP:
	- m->m_pkthdr.csum_flags \|= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP);
	- break;
	- case IPPROTO_UDP:
	- m->m_pkthdr.csum_flags \|= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP);
	- break;
	- }
	- m->m_pkthdr.csum_data = hdr->csum_offset;
	-}
	-
	-static void
	-vtnet_rxq_csum_data_valid(struct vtnet_rxq rxq, struct mbuf m, int protocol)
	-{
	- KASSERT(protocol == IPPROTO_TCP \|\| protocol == IPPROTO_UDP,
	- ("%s: unsupported IP protocol %d", __func__, protocol));
	-
	- m->m_pkthdr.csum_flags \|= CSUM_DATA_VALID \| CSUM_PSEUDO_HDR;
	- m->m_pkthdr.csum_data = 0xFFFF;
	-}
	-
	-static int
	-vtnet_rxq_csum(struct vtnet_rxq rxq, struct mbuf m,
	- struct virtio_net_hdr *hdr)
	-{
	- const struct ether_header *eh;
	- struct vtnet_softc *sc;
	- int hoff, protocol;
	- uint16_t etype;
	- bool isipv6;
	-
	- KASSERT(hdr->flags &
	- (VIRTIO_NET_HDR_F_NEEDS_CSUM \| VIRTIO_NET_HDR_F_DATA_VALID),
	- ("%s: missing checksum offloading flag %x", __func__, hdr->flags));
	-
	- eh = mtod(m, const struct ether_header *);
	- etype = ntohs(eh->ether_type);
	- if (etype == ETHERTYPE_VLAN) {
	- /* TODO BMV: Handle QinQ. */
	- const struct ether_vlan_header *evh =
	- mtod(m, const struct ether_vlan_header *);
	- etype = ntohs(evh->evl_proto);
	- hoff = sizeof(struct ether_vlan_header);
	- } else
	- hoff = sizeof(struct ether_header);
	-
	- sc = rxq->vtnrx_sc;
	-
	- /* Check whether ethernet type is IP or IPv6, and get protocol. */
	- switch (etype) {
	-#if defined(INET)
	- case ETHERTYPE_IP:
	- if (__predict_false(m->m_len < hoff + sizeof(struct ip))) {
	- sc->vtnet_stats.rx_csum_inaccessible_ipproto++;
	- return (1);
	- } else {
	- struct ip ip = (struct ip )(m->m_data + hoff);
	- protocol = ip->ip_p;
	- }
	- isipv6 = false;
	- break;
	-#endif
	-#if defined(INET6)
	- case ETHERTYPE_IPV6:
	- if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr))
	- \|\| ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) {
	- sc->vtnet_stats.rx_csum_inaccessible_ipproto++;
	- return (1);
	- }
	- isipv6 = true;
	- break;
	-#endif
	- default:
	- sc->vtnet_stats.rx_csum_bad_ethtype++;
	- return (1);
	- }
	-
	- /* Check whether protocol is TCP or UDP. */
	- switch (protocol) {
	- case IPPROTO_TCP:
	- case IPPROTO_UDP:
	- break;
	- default:
	- /*
	- * FreeBSD does not support checksum offloading of this
	- * protocol here.
	- */
	- sc->vtnet_stats.rx_csum_bad_ipproto++;
	- return (1);
	- }
	-
	- if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
	- vtnet_rxq_csum_needs_csum(rxq, m, isipv6, protocol, hdr);
	- else /* VIRTIO_NET_HDR_F_DATA_VALID */
	- vtnet_rxq_csum_data_valid(rxq, m, protocol);
	-
	- return (0);
	-}
	-#endif
	-
	static void
	vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)
	{
	@@ -2022,10 +1889,29 @@
	if (hdr->flags &
	(VIRTIO_NET_HDR_F_NEEDS_CSUM \| VIRTIO_NET_HDR_F_DATA_VALID)) {
	#if defined(INET) \|\| defined(INET6)
	- if (vtnet_rxq_csum(rxq, m, hdr) == 0)
	+ int ret;
	+
	+ /*
	+ * Translate the VirtIO header flags to the corresponding
	+ * CSUM_* flags in the mbuf.
	+ */
	+ ret = virtio_net_rx_csum(m, hdr);
	+ if (ret == 0)
	rxq->vtnrx_stats.vrxs_csum++;
	- else
	+ else {
	+ switch (ret) {
	+ case VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO:
	+ sc->vtnet_stats.rx_csum_inaccessible_ipproto++;
	+ break;
	+ case VIRTIO_NET_RX_CSUM_BAD_ETHTYPE:
	+ sc->vtnet_stats.rx_csum_bad_ethtype++;
	+ break;
	+ case VIRTIO_NET_RX_CSUM_BAD_IPPROTO:
	+ sc->vtnet_stats.rx_csum_bad_ipproto++;
	+ break;
	+ }
	rxq->vtnrx_stats.vrxs_csum_failed++;
	+ }
	#else
	sc->vtnet_stats.rx_csum_bad_ethtype++;
	rxq->vtnrx_stats.vrxs_csum_failed++;
	@@ -2351,166 +2237,6 @@
	("%s: mbufs remaining in tx queue %p", __func__, txq));
	}

	-/*
	- * BMV: This can go away once we finally have offsets in the mbuf header.
	- */
	-static int
	-vtnet_txq_offload_ctx(struct vtnet_txq txq, struct mbuf m, int *etype,
	- int proto, int start)
	-{
	- struct vtnet_softc *sc;
	- struct ether_vlan_header *evh;
	-#if defined(INET) \|\| defined(INET6)
	- int offset;
	-#endif
	-
	- sc = txq->vtntx_sc;
	-
	- evh = mtod(m, struct ether_vlan_header *);
	- if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
	- /* BMV: We should handle nested VLAN tags too. */
	- *etype = ntohs(evh->evl_proto);
	-#if defined(INET) \|\| defined(INET6)
	- offset = sizeof(struct ether_vlan_header);
	-#endif
	- } else {
	- *etype = ntohs(evh->evl_encap_proto);
	-#if defined(INET) \|\| defined(INET6)
	- offset = sizeof(struct ether_header);
	-#endif
	- }
	-
	- switch (*etype) {
	-#if defined(INET)
	- case ETHERTYPE_IP: {
	- struct ip *ip, iphdr;
	- if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
	- m_copydata(m, offset, sizeof(struct ip),
	- (caddr_t) &iphdr);
	- ip = &iphdr;
	- } else
	- ip = (struct ip *)(m->m_data + offset);
	- *proto = ip->ip_p;
	- *start = offset + (ip->ip_hl << 2);
	- break;
	- }
	-#endif
	-#if defined(INET6)
	- case ETHERTYPE_IPV6:
	- *proto = -1;
	- *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
	- /* Assert the network stack sent us a valid packet. */
	- KASSERT(*start > offset,
	- ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
	- start, offset, proto));
	- break;
	-#endif
	- default:
	- sc->vtnet_stats.tx_csum_unknown_ethtype++;
	- return (EINVAL);
	- }
	-
	- return (0);
	-}
	-
	-static int
	-vtnet_txq_offload_tso(struct vtnet_txq txq, struct mbuf m, int eth_type,
	- int offset, struct virtio_net_hdr *hdr)
	-{
	- static struct timeval lastecn;
	- static int curecn;
	- struct vtnet_softc *sc;
	- struct tcphdr *tcp, tcphdr;
	-
	- sc = txq->vtntx_sc;
	-
	- if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
	- m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
	- tcp = &tcphdr;
	- } else
	- tcp = (struct tcphdr *)(m->m_data + offset);
	-
	- hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2));
	- hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz);
	- hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
	- VIRTIO_NET_HDR_GSO_TCPV6;
	-
	- if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) {
	- /*
	- * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In
	- * FreeBSD, ECN support is not on a per-interface basis,
	- * but globally via the net.inet.tcp.ecn.enable sysctl
	- * knob. The default is off.
	- */
	- if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
	- if (ppsratecheck(&lastecn, &curecn, 1))
	- if_printf(sc->vtnet_ifp,
	- "TSO with ECN not negotiated with host\n");
	- return (ENOTSUP);
	- }
	- hdr->gso_type \|= VIRTIO_NET_HDR_GSO_ECN;
	- }
	-
	- txq->vtntx_stats.vtxs_tso++;
	-
	- return (0);
	-}
	-
	-static struct mbuf *
	-vtnet_txq_offload(struct vtnet_txq txq, struct mbuf m,
	- struct virtio_net_hdr *hdr)
	-{
	- struct vtnet_softc *sc;
	- int flags, etype, csum_start, proto, error;
	-
	- sc = txq->vtntx_sc;
	- flags = m->m_pkthdr.csum_flags;
	-
	- error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start);
	- if (error)
	- goto drop;
	-
	- if (flags & (VTNET_CSUM_OFFLOAD \| VTNET_CSUM_OFFLOAD_IPV6)) {
	- /* Sanity check the parsed mbuf matches the offload flags. */
	- if (__predict_false((flags & VTNET_CSUM_OFFLOAD &&
	- etype != ETHERTYPE_IP) \|\| (flags & VTNET_CSUM_OFFLOAD_IPV6
	- && etype != ETHERTYPE_IPV6))) {
	- sc->vtnet_stats.tx_csum_proto_mismatch++;
	- goto drop;
	- }
	-
	- hdr->flags \|= VIRTIO_NET_HDR_F_NEEDS_CSUM;
	- hdr->csum_start = vtnet_gtoh16(sc, csum_start);
	- hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data);
	- txq->vtntx_stats.vtxs_csum++;
	- }
	-
	- if (flags & (CSUM_IP_TSO \| CSUM_IP6_TSO)) {
	- /*
	- * Sanity check the parsed mbuf IP protocol is TCP, and
	- * VirtIO TSO reqires the checksum offloading above.
	- */
	- if (__predict_false(proto != IPPROTO_TCP)) {
	- sc->vtnet_stats.tx_tso_not_tcp++;
	- goto drop;
	- } else if (__predict_false((hdr->flags &
	- VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
	- sc->vtnet_stats.tx_tso_without_csum++;
	- goto drop;
	- }
	-
	- error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
	- if (error)
	- goto drop;
	- }
	-
	- return (m);
	-
	-drop:
	- m_freem(m);
	- return (NULL);
	-}
	-
	static int
	vtnet_txq_enqueue_buf(struct vtnet_txq txq, struct mbuf *m_head,
	struct vtnet_tx_header *txhdr)
	@@ -2597,11 +2323,38 @@
	}

	if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) {
	- m = vtnet_txq_offload(txq, m, hdr);
	+ int ret;
	+
	+ /*
	+ * Translate the CSUM_* flags in the mbuf to the corresponding
	+ * flags in the VirtIO header.
	+ */
	+ ret = virtio_net_tx_offload(txq->vtntx_sc->vtnet_ifp, &m, hdr,
	+ (txq->vtntx_sc->vtnet_flags & VTNET_FLAG_TSO_ECN),
	+ vtnet_modern(txq->vtntx_sc));
	+ switch (ret) {
	+ case VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE:
	+ txq->vtntx_sc->vtnet_stats.tx_csum_unknown_ethtype++;
	+ break;
	+ case VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH:
	+ txq->vtntx_sc->vtnet_stats.tx_csum_proto_mismatch++;
	+ break;
	+ case VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP:
	+ txq->vtntx_sc->vtnet_stats.tx_tso_not_tcp++;
	+ break;
	+ case VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM:
	+ txq->vtntx_sc->vtnet_stats.tx_tso_without_csum++;
	+ break;
	+ }
	if ((*m_head = m) == NULL) {
	error = ENOBUFS;
	goto fail;
	}
	+ if (m->m_pkthdr.csum_flags &
	+ (VTNET_CSUM_OFFLOAD \| VTNET_CSUM_OFFLOAD_IPV6))
	+ txq->vtntx_stats.vtxs_csum++;
	+ if (m->m_pkthdr.csum_flags & (CSUM_IP_TSO \| CSUM_IP6_TSO))
	+ txq->vtntx_stats.vtxs_tso++;
	}

	error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
	diff --git a/sys/dev/virtio/network/virtio_net.h b/sys/dev/virtio/network/virtio_net.h
	--- a/sys/dev/virtio/network/virtio_net.h
	+++ b/sys/dev/virtio/network/virtio_net.h
	@@ -31,6 +31,11 @@
	#ifndef _VIRTIO_NET_H
	#define _VIRTIO_NET_H

	+#include "opt_inet.h"
	+#include "opt_inet6.h"
	+
	+#include <dev/virtio/virtio_endian.h>
	+
	/* The feature bitmap for virtio net */
	#define VIRTIO_NET_F_CSUM (1ULL << 0) /* Host handles pkts w/ partial csum */
	#define VIRTIO_NET_F_GUEST_CSUM (1ULL << 1) /* Guest handles pkts w/ partial csum*/
	@@ -260,161 +265,146 @@
	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5
	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0

	-/*
	- * Use the checksum offset in the VirtIO header to set the
	- * correct CSUM_* flags.
	- */
	-static inline int
	-virtio_net_rx_csum_by_offset(struct mbuf *m, uint16_t eth_type, int ip_start,
	- struct virtio_net_hdr *hdr)
	-{
	#if defined(INET) \|\| defined(INET6)
	- int offset = hdr->csum_start + hdr->csum_offset;
	-#endif
	+static inline void
	+virtio_net_rx_csum_needs_csum(struct mbuf *m, bool isipv6, int protocol,
	+ struct virtio_net_hdr *hdr)
	+{
	+ /*
	+ * The packet is likely from another VM on the same host or from the
	+ * host that itself performed checksum offloading so Tx/Rx is basically
	+ * a memcpy and the checksum has little value so far.
	+ */

	- /* Only do a basic sanity check on the offset. */
	- switch (eth_type) {
	-#if defined(INET)
	- case ETHERTYPE_IP:
	- if (__predict_false(offset < ip_start + sizeof(struct ip)))
	- return (1);
	- break;
	-#endif
	-#if defined(INET6)
	- case ETHERTYPE_IPV6:
	- if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
	- return (1);
	- break;
	-#endif
	- default:
	- /* Here we should increment the rx_csum_bad_ethtype counter. */
	- return (1);
	- }
	+ KASSERT(protocol == IPPROTO_TCP \|\| protocol == IPPROTO_UDP,
	+ ("%s: unsupported IP protocol %d", __func__, protocol));

	/*
	- * Use the offset to determine the appropriate CSUM_* flags. This is
	- * a bit dirty, but we can get by with it since the checksum offsets
	- * happen to be different. We assume the host host does not do IPv4
	- * header checksum offloading.
	+ * Just forward the order to compute the checksum by setting
	+ * the corresponding mbuf flag (e.g., CSUM_TCP).
	*/
	- switch (hdr->csum_offset) {
	- case offsetof(struct udphdr, uh_sum):
	- case offsetof(struct tcphdr, th_sum):
	- m->m_pkthdr.csum_flags \|= CSUM_DATA_VALID \| CSUM_PSEUDO_HDR;
	- m->m_pkthdr.csum_data = 0xFFFF;
	+ switch (protocol) {
	+ case IPPROTO_TCP:
	+ m->m_pkthdr.csum_flags \|= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP);
	+ break;
	+ case IPPROTO_UDP:
	+ m->m_pkthdr.csum_flags \|= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP);
	break;
	- default:
	- /* Here we should increment the rx_csum_bad_offset counter. */
	- return (1);
	}
	+ m->m_pkthdr.csum_data = hdr->csum_offset;
	+}

	- return (0);
	+static inline void
	+virtio_net_rx_csum_data_valid(struct mbuf *m, int protocol)
	+{
	+ KASSERT(protocol == IPPROTO_TCP \|\| protocol == IPPROTO_UDP,
	+ ("%s: unsupported IP protocol %d", __func__, protocol));
	+
	+ m->m_pkthdr.csum_flags \|= CSUM_DATA_VALID \| CSUM_PSEUDO_HDR;
	+ m->m_pkthdr.csum_data = 0xFFFF;
	}

	+#define VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO 1
	+#define VIRTIO_NET_RX_CSUM_BAD_ETHTYPE 2
	+#define VIRTIO_NET_RX_CSUM_BAD_IPPROTO 3
	+
	+/*
	+ * For a packet received over the VirtIO channel, it checks the given
	+ * VirtIO header and sets the appropriate CSUM_* flags in the given mbuf.
	+ *
	+ * Unfortunately, the information provided is not directly useful to us. The
	+ * VirtIO header gives the offset of the checksum, which is all Linux needs, but
	+ * this is not how FreeBSD does things. We are forced to peek inside the packet
	+ * a bit.
	+ *
	+ * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
	+ * could accept the offsets and let the stack figure it out.
	+ *
	+ * @param m mbuf of the packet where CSUM_* flags might need to be set.
	+ * @param hdr VirtIO header of the received packet that needs to be checked.
	+ *
	+ * @return 0 on success, or one of the VIRTIO_NET_RX_CSUM_* error codes.
	+ */
	static inline int
	-virtio_net_rx_csum_by_parse(struct mbuf *m, uint16_t eth_type, int ip_start,
	- struct virtio_net_hdr *hdr)
	+virtio_net_rx_csum(struct mbuf m, struct virtio_net_hdr hdr)
	{
	- int offset, proto;
	+ const struct ether_header *eh;
	+ int hoff, protocol;
	+ uint16_t etype;
	+ bool isipv6;
	+
	+ KASSERT(hdr->flags &
	+ (VIRTIO_NET_HDR_F_NEEDS_CSUM \| VIRTIO_NET_HDR_F_DATA_VALID),
	+ ("%s: missing checksum offloading flag %x", __func__, hdr->flags));
	+
	+ eh = mtod(m, const struct ether_header *);
	+ etype = ntohs(eh->ether_type);
	+ if (etype == ETHERTYPE_VLAN) {
	+ /* TODO BMV: Handle QinQ. */
	+ const struct ether_vlan_header *evh =
	+ mtod(m, const struct ether_vlan_header *);
	+ etype = ntohs(evh->evl_proto);
	+ hoff = sizeof(struct ether_vlan_header);
	+ } else
	+ hoff = sizeof(struct ether_header);

	- switch (eth_type) {
	+ /* Check whether ethernet type is IP or IPv6, and get protocol. */
	+ switch (etype) {
	#if defined(INET)
	- case ETHERTYPE_IP: {
	- struct ip *ip;
	- if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
	- return (1);
	- ip = (struct ip *)(m->m_data + ip_start);
	- proto = ip->ip_p;
	- offset = ip_start + (ip->ip_hl << 2);
	+ case ETHERTYPE_IP:
	+ if (__predict_false(m->m_len < hoff + sizeof(struct ip))) {
	+ return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO);
	+ } else {
	+ struct ip ip = (struct ip )(m->m_data + hoff);
	+ protocol = ip->ip_p;
	+ }
	+ isipv6 = false;
	break;
	- }
	#endif
	#if defined(INET6)
	case ETHERTYPE_IPV6:
	- if (__predict_false(m->m_len < ip_start +
	- sizeof(struct ip6_hdr)))
	- return (1);
	- offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
	- if (__predict_false(offset < 0))
	- return (1);
	+ if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr))
	+ \|\| ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) {
	+ return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO);
	+ }
	+ isipv6 = true;
	break;
	#endif
	default:
	- /* Here we should increment the rx_csum_bad_ethtype counter. */
	- return (1);
	+ return (VIRTIO_NET_RX_CSUM_BAD_ETHTYPE);
	}

	- switch (proto) {
	+ /* Check whether protocol is TCP or UDP. */
	+ switch (protocol) {
	case IPPROTO_TCP:
	- if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
	- return (1);
	- m->m_pkthdr.csum_flags \|= CSUM_DATA_VALID \| CSUM_PSEUDO_HDR;
	- m->m_pkthdr.csum_data = 0xFFFF;
	- break;
	case IPPROTO_UDP:
	- if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
	- return (1);
	- m->m_pkthdr.csum_flags \|= CSUM_DATA_VALID \| CSUM_PSEUDO_HDR;
	- m->m_pkthdr.csum_data = 0xFFFF;
	break;
	default:
	/*
	- * For the remaining protocols, FreeBSD does not support
	- * checksum offloading, so the checksum will be recomputed.
	+ * FreeBSD does not support checksum offloading of this
	+ * protocol here.
	*/
	-#if 0
	- if_printf(ifp, "cksum offload of unsupported "
	- "protocol eth_type=%#x proto=%d csum_start=%d "
	- "csum_offset=%d\n", __func__, eth_type, proto,
	- hdr->csum_start, hdr->csum_offset);
	-#endif
	- break;
	+ return (VIRTIO_NET_RX_CSUM_BAD_IPPROTO);
	}

	+ if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
	+ virtio_net_rx_csum_needs_csum(m, isipv6, protocol, hdr);
	+ else /* VIRTIO_NET_HDR_F_DATA_VALID */
	+ virtio_net_rx_csum_data_valid(m, protocol);
	+
	return (0);
	}
	+#endif
	+
	+#define VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE 1
	+#define VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH 2
	+#define VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP 3
	+#define VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM 4
	+#define VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED 5

	/*
	- * Set the appropriate CSUM_* flags. Unfortunately, the information
	- * provided is not directly useful to us. The VirtIO header gives the
	- * offset of the checksum, which is all Linux needs, but this is not
	- * how FreeBSD does things. We are forced to peek inside the packet
	- * a bit.
	- *
	- * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
	- * could accept the offsets and let the stack figure it out.
	+ * BMV: This can go away once we finally have offsets in the mbuf header.
	*/
	-static inline int
	-virtio_net_rx_csum(struct mbuf m, struct virtio_net_hdr hdr)
	-{
	- struct ether_header *eh;
	- struct ether_vlan_header *evh;
	- uint16_t eth_type;
	- int offset, error;
	-
	- if ((hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM \|
	- VIRTIO_NET_HDR_F_DATA_VALID)) == 0) {
	- return (0);
	- }
	-
	- eh = mtod(m, struct ether_header *);
	- eth_type = ntohs(eh->ether_type);
	- if (eth_type == ETHERTYPE_VLAN) {
	- /* BMV: We should handle nested VLAN tags too. */
	- evh = mtod(m, struct ether_vlan_header *);
	- eth_type = ntohs(evh->evl_proto);
	- offset = sizeof(struct ether_vlan_header);
	- } else
	- offset = sizeof(struct ether_header);
	-
	- if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
	- error = virtio_net_rx_csum_by_offset(m, eth_type, offset, hdr);
	- else
	- error = virtio_net_rx_csum_by_parse(m, eth_type, offset, hdr);
	-
	- return (error);
	-}
	-
	static inline int
	virtio_net_tx_offload_ctx(struct mbuf m, int etype, int proto, int start)
	{
	@@ -463,16 +453,15 @@
	break;
	#endif
	default:
	- /* Here we should increment the tx_csum_bad_ethtype counter. */
	- return (EINVAL);
	+ return (VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE);
	}

	return (0);
	}

	static inline int
	-virtio_net_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type,
	- int offset, bool allow_ecn, struct virtio_net_hdr *hdr)
	+virtio_net_tx_offload_tso(struct ifnet ifp, struct mbuf m, int eth_type,
	+ int offset, struct virtio_net_hdr *hdr, bool tso_ecn, bool modern)
	{
	static struct timeval lastecn;
	static int curecn;
	@@ -484,79 +473,99 @@
	} else
	tcp = (struct tcphdr *)(m->m_data + offset);

	- hdr->hdr_len = offset + (tcp->th_off << 2);
	- hdr->gso_size = m->m_pkthdr.tso_segsz;
	+ hdr->hdr_len = virtio_gtoh16(modern, offset + (tcp->th_off << 2));
	+ hdr->gso_size = virtio_gtoh16(modern, m->m_pkthdr.tso_segsz);
	hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
	VIRTIO_NET_HDR_GSO_TCPV6;

	- if (tcp_get_flags(tcp) & TH_CWR) {
	+ if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) {
	/*
	- * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
	- * ECN support is not on a per-interface basis, but globally via
	- * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
	+ * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In
	+ * FreeBSD, ECN support is not on a per-interface basis,
	+ * but globally via the net.inet.tcp.ecn.enable sysctl
	+ * knob. The default is off.
	*/
	- if (!allow_ecn) {
	+ if (!tso_ecn) {
	if (ppsratecheck(&lastecn, &curecn, 1))
	if_printf(ifp,
	"TSO with ECN not negotiated with host\n");
	- return (ENOTSUP);
	+ return (VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED);
	}
	hdr->gso_type \|= VIRTIO_NET_HDR_GSO_ECN;
	}

	- /* Here we should increment tx_tso counter. */
	-
	return (0);
	}

	-static inline struct mbuf *
	-virtio_net_tx_offload(if_t ifp, struct mbuf *m, bool allow_ecn,
	- struct virtio_net_hdr *hdr)
	+/*
	+ * For a packet to be transmitted over the VirtIO channel, it checks the
	+ * CSUM_* flags in the mbuf and sets the appropriate flags in the VirtIO header.
	+ * In case of an error, it frees the mbuf and sets the pointer referenced by mp
	+ * to NULL.
	+ *
	+ * @param ifp ifnet struct of outgoing interface.
	+ * @param mp mbuf on which the CSUM_* flags needs to be checked.
	+ * @param hdr VirtIO header to be filled for the outgoing packet.
	+ * @param tso_ecn true if ECN have been negotiated between host and guest.
	+ * @param modern true if VirtIO modern mode is used.
	+ *
	+ * @return 0 on success, or one of the VIRTIO_NET_TX_OFFLOAD_* error codes.
	+ */
	+static inline int
	+virtio_net_tx_offload(struct ifnet ifp, struct mbuf *mp,
	+ struct virtio_net_hdr *hdr, bool tso_ecn, bool modern)
	{
	int flags, etype, csum_start, proto, error;
	+ struct mbuf *m;

	+ m = *mp;
	flags = m->m_pkthdr.csum_flags;

	error = virtio_net_tx_offload_ctx(m, &etype, &proto, &csum_start);
	- if (error)
	+ if (error != 0)
	goto drop;

	- if ((etype == ETHERTYPE_IP && (flags & (CSUM_TCP \| CSUM_UDP))) \|\|
	- (etype == ETHERTYPE_IPV6 &&
	- (flags & (CSUM_TCP_IPV6 \| CSUM_UDP_IPV6)))) {
	- /*
	- * We could compare the IP protocol vs the CSUM_ flag too,
	- * but that really should not be necessary.
	- */
	+ if (flags & (CSUM_TCP \| CSUM_UDP \| CSUM_TCP_IPV6 \| CSUM_UDP_IPV6)) {
	+ /* Sanity check the parsed mbuf matches the offload flags. */
	+ if (__predict_false((flags & (CSUM_TCP \| CSUM_UDP) &&
	+ etype != ETHERTYPE_IP) \|\|
	+ (flags & (CSUM_TCP_IPV6 \| CSUM_UDP_IPV6) &&
	+ etype != ETHERTYPE_IPV6))) {
	+ error = VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH;
	+ goto drop;
	+ }
	+
	hdr->flags \|= VIRTIO_NET_HDR_F_NEEDS_CSUM;
	- hdr->csum_start = csum_start;
	- hdr->csum_offset = m->m_pkthdr.csum_data;
	- /* Here we should increment the tx_csum counter. */
	+ hdr->csum_start = virtio_gtoh16(modern, csum_start);
	+ hdr->csum_offset = virtio_gtoh16(modern, m->m_pkthdr.csum_data);
	}

	- if (flags & CSUM_TSO) {
	+ if (flags & (CSUM_IP_TSO \| CSUM_IP6_TSO)) {
	+ /*
	+ * Sanity check the parsed mbuf IP protocol is TCP, and
	+ * VirtIO TSO reqires the checksum offloading above.
	+ */
	if (__predict_false(proto != IPPROTO_TCP)) {
	- /* Likely failed to correctly parse the mbuf.
	- * Here we should increment the tx_tso_not_tcp
	- * counter. */
	+ error = VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP;
	+ goto drop;
	+ } else if (__predict_false((hdr->flags &
	+ VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
	+ error = VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM;
	goto drop;
	}

	- KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
	- ("%s: mbuf %p TSO without checksum offload %#x",
	- __func__, m, flags));
	-
	error = virtio_net_tx_offload_tso(ifp, m, etype, csum_start,
	- allow_ecn, hdr);
	- if (error)
	+ hdr, tso_ecn, modern);
	+ if (error != 0)
	goto drop;
	}

	- return (m);
	+ return (error);

	drop:
	m_freem(m);
	- return (NULL);
	+ *mp = NULL;
	+ return (error);
	}

	#endif /* _VIRTIO_NET_H */
	diff --git a/sys/net/if_tuntap.c b/sys/net/if_tuntap.c
	--- a/sys/net/if_tuntap.c
	+++ b/sys/net/if_tuntap.c
	@@ -1882,7 +1882,16 @@

	bzero(&vhdr, sizeof(vhdr));
	if (m->m_pkthdr.csum_flags & TAP_ALL_OFFLOAD) {
	- m = virtio_net_tx_offload(ifp, m, false, &vhdr.hdr);
	+ /*
	+ * Translate the CSUM_* flags in the mbuf to the
	+ * corresponding flags in the VirtIO header.
	+ *
	+ * Always indicate that ECN has not been negotiated
	+ * and VirtIO modern mode is not used because bhyve
	+ * does not do this.
	+ */
	+ virtio_net_tx_offload(ifp, &m, &vhdr.hdr, false,
	+ false);
	}

	TUNDEBUG(ifp, "txvhdr: f %u, gt %u, hl %u, "
	@@ -1927,7 +1936,13 @@
	}

	if (vhdr != NULL) {
	- if (virtio_net_rx_csum(m, &vhdr->hdr)) {
	+ /*
	+ * Translate the VirtIO header flags to the corresponding
	+ * CSUM_* flags in the mbuf.
	+ */
	+ if (((vhdr->hdr.flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM \|
	+ VIRTIO_NET_HDR_F_DATA_VALID)) != 0) &&
	+ (virtio_net_rx_csum(m, &vhdr->hdr) != 0)) {
	m_freem(m);
	return (0);
	}

File Metadata

Mime Type: text/plain
Expires: Fri, Jun 19, 1:16 PM (5 h, 49 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 34080647
Default Alt Text: D57299.id179039.diff (27 KB)

D57299.id179039.diffNo OneTemporaryActions

D57299.id179039.diffView Options

File Metadata

Event Timeline

D57299.id179039.diff
No OneTemporary
Actions

D57299.id179039.diff
View Options