Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/virtio/network/if_vtnet.c
Show First 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | |||||
#include <netinet/in_systm.h> | #include <netinet/in_systm.h> | ||||
#include <netinet/in.h> | #include <netinet/in.h> | ||||
#include <netinet/ip.h> | #include <netinet/ip.h> | ||||
#include <netinet/ip6.h> | #include <netinet/ip6.h> | ||||
#include <netinet6/ip6_var.h> | #include <netinet6/ip6_var.h> | ||||
#include <netinet/udp.h> | #include <netinet/udp.h> | ||||
#include <netinet/tcp.h> | #include <netinet/tcp.h> | ||||
#include <netinet/tcp_lro.h> | |||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/resource.h> | #include <machine/resource.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/rman.h> | #include <sys/rman.h> | ||||
#include <dev/virtio/virtio.h> | #include <dev/virtio/virtio.h> | ||||
#include <dev/virtio/virtqueue.h> | #include <dev/virtio/virtqueue.h> | ||||
Show All 23 Lines | |||||
static void vtnet_setup_features(struct vtnet_softc *); | static void vtnet_setup_features(struct vtnet_softc *); | ||||
static int vtnet_init_rxq(struct vtnet_softc *, int); | static int vtnet_init_rxq(struct vtnet_softc *, int); | ||||
static int vtnet_init_txq(struct vtnet_softc *, int); | static int vtnet_init_txq(struct vtnet_softc *, int); | ||||
static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); | static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); | ||||
static void vtnet_free_rxtx_queues(struct vtnet_softc *); | static void vtnet_free_rxtx_queues(struct vtnet_softc *); | ||||
static int vtnet_alloc_rx_filters(struct vtnet_softc *); | static int vtnet_alloc_rx_filters(struct vtnet_softc *); | ||||
static void vtnet_free_rx_filters(struct vtnet_softc *); | static void vtnet_free_rx_filters(struct vtnet_softc *); | ||||
static int vtnet_alloc_virtqueues(struct vtnet_softc *); | static int vtnet_alloc_virtqueues(struct vtnet_softc *); | ||||
static int vtnet_alloc_interface(struct vtnet_softc *); | |||||
static int vtnet_setup_interface(struct vtnet_softc *); | static int vtnet_setup_interface(struct vtnet_softc *); | ||||
static int vtnet_ioctl_mtu(struct vtnet_softc *, int); | static int vtnet_ioctl_mtu(struct vtnet_softc *, int); | ||||
static int vtnet_ioctl_ifflags(struct vtnet_softc *); | static int vtnet_ioctl_ifflags(struct vtnet_softc *); | ||||
static int vtnet_ioctl_multi(struct vtnet_softc *); | static int vtnet_ioctl_multi(struct vtnet_softc *); | ||||
static int vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *); | static int vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *); | ||||
static int vtnet_ioctl(struct ifnet *, u_long, caddr_t); | static int vtnet_ioctl(struct ifnet *, u_long, caddr_t); | ||||
static uint64_t vtnet_get_counter(struct ifnet *, ift_counter); | static uint64_t vtnet_get_counter(struct ifnet *, ift_counter); | ||||
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | |||||
static void vtnet_vlan_tag_remove(struct mbuf *); | static void vtnet_vlan_tag_remove(struct mbuf *); | ||||
static void vtnet_set_rx_process_limit(struct vtnet_softc *); | static void vtnet_set_rx_process_limit(struct vtnet_softc *); | ||||
static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, | static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, | ||||
struct sysctl_oid_list *, struct vtnet_rxq *); | struct sysctl_oid_list *, struct vtnet_rxq *); | ||||
static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, | static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, | ||||
struct sysctl_oid_list *, struct vtnet_txq *); | struct sysctl_oid_list *, struct vtnet_txq *); | ||||
static void vtnet_setup_queue_sysctl(struct vtnet_softc *); | static void vtnet_setup_queue_sysctl(struct vtnet_softc *); | ||||
static void vtnet_load_tunables(struct vtnet_softc *); | |||||
static void vtnet_setup_sysctl(struct vtnet_softc *); | static void vtnet_setup_sysctl(struct vtnet_softc *); | ||||
static int vtnet_rxq_enable_intr(struct vtnet_rxq *); | static int vtnet_rxq_enable_intr(struct vtnet_rxq *); | ||||
static void vtnet_rxq_disable_intr(struct vtnet_rxq *); | static void vtnet_rxq_disable_intr(struct vtnet_rxq *); | ||||
static int vtnet_txq_enable_intr(struct vtnet_txq *); | static int vtnet_txq_enable_intr(struct vtnet_txq *); | ||||
static void vtnet_txq_disable_intr(struct vtnet_txq *); | static void vtnet_txq_disable_intr(struct vtnet_txq *); | ||||
static void vtnet_enable_rx_interrupts(struct vtnet_softc *); | static void vtnet_enable_rx_interrupts(struct vtnet_softc *); | ||||
static void vtnet_enable_tx_interrupts(struct vtnet_softc *); | static void vtnet_enable_tx_interrupts(struct vtnet_softc *); | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||
SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, | SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, | ||||
&vtnet_tso_maxlen, 0, "TSO burst limit"); | &vtnet_tso_maxlen, 0, "TSO burst limit"); | ||||
static int vtnet_rx_process_limit = 1024; | static int vtnet_rx_process_limit = 1024; | ||||
TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); | TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); | ||||
SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, | SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, | ||||
&vtnet_rx_process_limit, 0, "Limits RX segments processed in a single pass"); | &vtnet_rx_process_limit, 0, "Limits RX segments processed in a single pass"); | ||||
static int vtnet_lro_entry_count = 128; | |||||
SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, | |||||
&vtnet_lro_entry_count, 0, "Software LRO entry count"); | |||||
/* Enable sorted LRO, and the depth of the mbuf queue. */ | |||||
static int vtnet_lro_mbufq_depth = 0; | |||||
SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, | |||||
&vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue"); | |||||
static uma_zone_t vtnet_tx_header_zone; | static uma_zone_t vtnet_tx_header_zone; | ||||
static struct virtio_feature_desc vtnet_feature_desc[] = { | static struct virtio_feature_desc vtnet_feature_desc[] = { | ||||
{ VIRTIO_NET_F_CSUM, "TxChecksum" }, | { VIRTIO_NET_F_CSUM, "TxChecksum" }, | ||||
{ VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, | { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, | ||||
{ VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "CtrlRxOffloads" }, | { VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "CtrlRxOffloads" }, | ||||
{ VIRTIO_NET_F_MAC, "MAC" }, | { VIRTIO_NET_F_MAC, "MAC" }, | ||||
{ VIRTIO_NET_F_GSO, "TxGSO" }, | { VIRTIO_NET_F_GSO, "TxGSO" }, | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | vtnet_attach(device_t dev) | ||||
sc = device_get_softc(dev); | sc = device_get_softc(dev); | ||||
sc->vtnet_dev = dev; | sc->vtnet_dev = dev; | ||||
virtio_set_feature_desc(dev, vtnet_feature_desc); | virtio_set_feature_desc(dev, vtnet_feature_desc); | ||||
VTNET_CORE_LOCK_INIT(sc); | VTNET_CORE_LOCK_INIT(sc); | ||||
callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); | callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); | ||||
vtnet_load_tunables(sc); | |||||
error = vtnet_alloc_interface(sc); | |||||
if (error) { | |||||
device_printf(dev, "cannot allocate interface\n"); | |||||
goto fail; | |||||
} | |||||
vtnet_setup_sysctl(sc); | vtnet_setup_sysctl(sc); | ||||
vtnet_setup_features(sc); | vtnet_setup_features(sc); | ||||
error = vtnet_alloc_rx_filters(sc); | error = vtnet_alloc_rx_filters(sc); | ||||
if (error) { | if (error) { | ||||
device_printf(dev, "cannot allocate Rx filters\n"); | device_printf(dev, "cannot allocate Rx filters\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 201 Lines • ▼ Show 20 Lines | if (virtio_with_feature(dev, VTNET_LRO_FEATURES) && | ||||
* enough to hold the maximum TCP packet, the Ethernet header, | * enough to hold the maximum TCP packet, the Ethernet header, | ||||
* and the header. This requires up to 34 descriptors with | * and the header. This requires up to 34 descriptors with | ||||
* MCLBYTES clusters. If we do not have indirect descriptors, | * MCLBYTES clusters. If we do not have indirect descriptors, | ||||
* LRO is disabled since the virtqueue will not contain very | * LRO is disabled since the virtqueue will not contain very | ||||
* many receive buffers. | * many receive buffers. | ||||
*/ | */ | ||||
if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { | if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"LRO disabled since both mergeable buffers and " | "Host LRO disabled since both mergeable buffers " | ||||
"indirect descriptors were not negotiated\n"); | "and indirect descriptors were not negotiated\n"); | ||||
features &= ~VTNET_LRO_FEATURES; | features &= ~VTNET_LRO_FEATURES; | ||||
negotiated_features = | negotiated_features = | ||||
virtio_negotiate_features(dev, features); | virtio_negotiate_features(dev, features); | ||||
} else | } else | ||||
sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; | sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; | ||||
} | } | ||||
sc->vtnet_features = negotiated_features; | sc->vtnet_features = negotiated_features; | ||||
Show All 40 Lines | vtnet_setup_features(struct vtnet_softc *sc) | ||||
if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) | if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) | ||||
sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE; | sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE; | ||||
else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) | else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) | ||||
sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG; | sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG; | ||||
else | else | ||||
sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE; | sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE; | ||||
/* | |||||
* Favor "hardware" LRO if negotiated, but support software LRO as | |||||
* a fallback; there is usually little benefit (or worse) with both. | |||||
*/ | |||||
if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 && | |||||
virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0) | |||||
sc->vtnet_flags |= VTNET_FLAG_SW_LRO; | |||||
if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || | if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || | ||||
virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || | virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || | ||||
virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) | virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) | ||||
sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX; | sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX; | ||||
else | else | ||||
sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN; | sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN; | ||||
sc->vtnet_max_vq_pairs = 1; | sc->vtnet_max_vq_pairs = 1; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | vtnet_init_rxq(struct vtnet_softc *sc, int id) | ||||
rxq->vtnrx_sc = sc; | rxq->vtnrx_sc = sc; | ||||
rxq->vtnrx_id = id; | rxq->vtnrx_id = id; | ||||
rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); | rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); | ||||
if (rxq->vtnrx_sg == NULL) | if (rxq->vtnrx_sg == NULL) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
#if defined(INET) || defined(INET6) | |||||
if (vtnet_software_lro(sc)) { | |||||
if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp, | |||||
sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0) | |||||
return (ENOMEM); | |||||
} | |||||
#endif | |||||
NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); | NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); | ||||
rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, | rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, | ||||
taskqueue_thread_enqueue, &rxq->vtnrx_tq); | taskqueue_thread_enqueue, &rxq->vtnrx_tq); | ||||
return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); | return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); | ||||
} | } | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
vtnet_destroy_rxq(struct vtnet_rxq *rxq) | vtnet_destroy_rxq(struct vtnet_rxq *rxq) | ||||
{ | { | ||||
rxq->vtnrx_sc = NULL; | rxq->vtnrx_sc = NULL; | ||||
rxq->vtnrx_id = -1; | rxq->vtnrx_id = -1; | ||||
#if defined(INET) || defined(INET6) | |||||
tcp_lro_free(&rxq->vtnrx_lro); | |||||
#endif | |||||
if (rxq->vtnrx_sg != NULL) { | if (rxq->vtnrx_sg != NULL) { | ||||
sglist_free(rxq->vtnrx_sg); | sglist_free(rxq->vtnrx_sg); | ||||
rxq->vtnrx_sg = NULL; | rxq->vtnrx_sg = NULL; | ||||
} | } | ||||
if (mtx_initialized(&rxq->vtnrx_mtx) != 0) | if (mtx_initialized(&rxq->vtnrx_mtx) != 0) | ||||
mtx_destroy(&rxq->vtnrx_mtx); | mtx_destroy(&rxq->vtnrx_mtx); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines | vtnet_alloc_virtqueues(struct vtnet_softc *sc) | ||||
error = virtio_alloc_virtqueues(dev, flags, nvqs, info); | error = virtio_alloc_virtqueues(dev, flags, nvqs, info); | ||||
free(info, M_TEMP); | free(info, M_TEMP); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
vtnet_setup_interface(struct vtnet_softc *sc) | vtnet_alloc_interface(struct vtnet_softc *sc) | ||||
{ | { | ||||
device_t dev; | device_t dev; | ||||
struct pfil_head_args pa; | |||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
dev = sc->vtnet_dev; | dev = sc->vtnet_dev; | ||||
ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER); | ifp = if_alloc(IFT_ETHER); | ||||
if (ifp == NULL) { | if (ifp == NULL) | ||||
device_printf(dev, "cannot allocate ifnet structure\n"); | return (ENOMEM); | ||||
return (ENOSPC); | |||||
} | |||||
if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | sc->vtnet_ifp = ifp; | ||||
ifp->if_softc = sc; | ifp->if_softc = sc; | ||||
if_initname(ifp, device_get_name(dev), device_get_unit(dev)); | |||||
return (0); | |||||
} | |||||
static int | |||||
vtnet_setup_interface(struct vtnet_softc *sc) | |||||
{ | |||||
device_t dev; | |||||
struct pfil_head_args pa; | |||||
struct ifnet *ifp; | |||||
dev = sc->vtnet_dev; | |||||
ifp = sc->vtnet_ifp; | |||||
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | | ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | | ||||
IFF_KNOWSEPOCH; | IFF_KNOWSEPOCH; | ||||
ifp->if_baudrate = IF_Gbps(10); | ifp->if_baudrate = IF_Gbps(10); | ||||
ifp->if_init = vtnet_init; | ifp->if_init = vtnet_init; | ||||
ifp->if_ioctl = vtnet_ioctl; | ifp->if_ioctl = vtnet_ioctl; | ||||
ifp->if_get_counter = vtnet_get_counter; | ifp->if_get_counter = vtnet_get_counter; | ||||
#ifndef VTNET_LEGACY_TX | #ifndef VTNET_LEGACY_TX | ||||
ifp->if_transmit = vtnet_txq_mq_start; | ifp->if_transmit = vtnet_txq_mq_start; | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | #ifdef notyet | ||||
/* BMV: Rx checksums not distinguished between IPv4 and IPv6. */ | /* BMV: Rx checksums not distinguished between IPv4 and IPv6. */ | ||||
ifp->if_capabilities |= IFCAP_RXCSUM_IPV6; | ifp->if_capabilities |= IFCAP_RXCSUM_IPV6; | ||||
#endif | #endif | ||||
if (vtnet_tunable_int(sc, "fixup_needs_csum", | if (vtnet_tunable_int(sc, "fixup_needs_csum", | ||||
vtnet_fixup_needs_csum) != 0) | vtnet_fixup_needs_csum) != 0) | ||||
sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM; | sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM; | ||||
if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) || | /* Support either "hardware" or software LRO. */ | ||||
virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) || | |||||
virtio_with_feature(dev, VIRTIO_NET_F_GUEST_ECN)) | |||||
ifp->if_capabilities |= IFCAP_LRO; | ifp->if_capabilities |= IFCAP_LRO; | ||||
} | } | ||||
if (ifp->if_capabilities & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) { | if (ifp->if_capabilities & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) { | ||||
/* | /* | ||||
* VirtIO does not support VLAN tagging, but we can fake | * VirtIO does not support VLAN tagging, but we can fake | ||||
* it by inserting and removing the 802.1Q header during | * it by inserting and removing the 802.1Q header during | ||||
* transmit and receive. We are then able to do checksum | * transmit and receive. We are then able to do checksum | ||||
* offloading of VLAN frames. | * offloading of VLAN frames. | ||||
▲ Show 20 Lines • Show All 171 Lines • ▼ Show 20 Lines | if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) { | ||||
* These Rx features require the negotiated features to | * These Rx features require the negotiated features to | ||||
* be updated. Avoid a full reinit if possible. | * be updated. Avoid a full reinit if possible. | ||||
*/ | */ | ||||
if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | ||||
update = 1; | update = 1; | ||||
else | else | ||||
reinit = 1; | reinit = 1; | ||||
/* BMV: Avoid needless renegotiation for just software LRO. */ | |||||
if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) == | |||||
IFCAP_LRO && vtnet_software_lro(sc)) | |||||
reinit = update = 0; | |||||
if (mask & IFCAP_RXCSUM) | if (mask & IFCAP_RXCSUM) | ||||
ifp->if_capenable ^= IFCAP_RXCSUM; | ifp->if_capenable ^= IFCAP_RXCSUM; | ||||
if (mask & IFCAP_RXCSUM_IPV6) | if (mask & IFCAP_RXCSUM_IPV6) | ||||
ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; | ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; | ||||
if (mask & IFCAP_LRO) | if (mask & IFCAP_LRO) | ||||
ifp->if_capenable ^= IFCAP_LRO; | ifp->if_capenable ^= IFCAP_LRO; | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 590 Lines • ▼ Show 20 Lines | |||||
fail: | fail: | ||||
sc->vtnet_stats.rx_mergeable_failed++; | sc->vtnet_stats.rx_mergeable_failed++; | ||||
m_freem(m_head); | m_freem(m_head); | ||||
return (1); | return (1); | ||||
} | } | ||||
#if defined(INET) || defined(INET6) | |||||
static int | |||||
vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m) | |||||
{ | |||||
struct lro_ctrl *lro; | |||||
lro = &rxq->vtnrx_lro; | |||||
if (lro->lro_mbuf_max != 0) { | |||||
tcp_lro_queue_mbuf(lro, m); | |||||
return (0); | |||||
} | |||||
return (tcp_lro_rx(lro, m, 0)); | |||||
} | |||||
#endif | |||||
static void | static void | ||||
vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, | vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, | ||||
struct virtio_net_hdr *hdr) | struct virtio_net_hdr *hdr) | ||||
{ | { | ||||
struct vtnet_softc *sc; | struct vtnet_softc *sc; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
sc = rxq->vtnrx_sc; | sc = rxq->vtnrx_sc; | ||||
Show All 21 Lines | if (vtnet_rxq_csum(rxq, m, hdr) == 0) | ||||
rxq->vtnrx_stats.vrxs_csum++; | rxq->vtnrx_stats.vrxs_csum++; | ||||
else | else | ||||
rxq->vtnrx_stats.vrxs_csum_failed++; | rxq->vtnrx_stats.vrxs_csum_failed++; | ||||
} | } | ||||
rxq->vtnrx_stats.vrxs_ipackets++; | rxq->vtnrx_stats.vrxs_ipackets++; | ||||
rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; | rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; | ||||
VTNET_RXQ_UNLOCK(rxq); | #if defined(INET) || defined(INET6) | ||||
if (vtnet_software_lro(sc) && ifp->if_capenable & IFCAP_LRO) { | |||||
if (vtnet_lro_rx(rxq, m) == 0) | |||||
return; | |||||
} | |||||
#endif | |||||
(*ifp->if_input)(ifp, m); | (*ifp->if_input)(ifp, m); | ||||
VTNET_RXQ_LOCK(rxq); | |||||
} | } | ||||
static int | static int | ||||
vtnet_rxq_eof(struct vtnet_rxq *rxq) | vtnet_rxq_eof(struct vtnet_rxq *rxq) | ||||
{ | { | ||||
struct virtio_net_hdr lhdr, *hdr; | struct virtio_net_hdr lhdr, *hdr; | ||||
struct vtnet_softc *sc; | struct vtnet_softc *sc; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | if (PFIL_HOOKED_IN(sc->vtnet_pfil)) { | ||||
continue; | continue; | ||||
default: | default: | ||||
KASSERT(pfil == PFIL_PASS, | KASSERT(pfil == PFIL_PASS, | ||||
("Filter returned %d!", pfil)); | ("Filter returned %d!", pfil)); | ||||
} | } | ||||
} | } | ||||
vtnet_rxq_input(rxq, m, &lhdr); | vtnet_rxq_input(rxq, m, &lhdr); | ||||
/* Must recheck after dropping the Rx lock. */ | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
break; | |||||
} | } | ||||
if (deq > 0) | if (deq > 0) { | ||||
#if defined(INET) || defined(INET6) | |||||
tcp_lro_flush_all(&rxq->vtnrx_lro); | |||||
#endif | |||||
virtqueue_notify(vq); | virtqueue_notify(vq); | ||||
} | |||||
return (count > 0 ? 0 : EAGAIN); | return (count > 0 ? 0 : EAGAIN); | ||||
} | } | ||||
static void | static void | ||||
vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries) | vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries) | ||||
{ | { | ||||
struct vtnet_softc *sc; | struct vtnet_softc *sc; | ||||
▲ Show 20 Lines • Show All 1,189 Lines • ▼ Show 20 Lines | vtnet_update_rx_offloads(struct vtnet_softc *sc) | ||||
if (ifp->if_capabilities & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { | if (ifp->if_capabilities & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { | ||||
if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) | if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) | ||||
features |= VIRTIO_NET_F_GUEST_CSUM; | features |= VIRTIO_NET_F_GUEST_CSUM; | ||||
else | else | ||||
features &= ~VIRTIO_NET_F_GUEST_CSUM; | features &= ~VIRTIO_NET_F_GUEST_CSUM; | ||||
} | } | ||||
if (ifp->if_capabilities & IFCAP_LRO) { | if (ifp->if_capabilities & IFCAP_LRO && !vtnet_software_lro(sc)) { | ||||
if (ifp->if_capenable & IFCAP_LRO) | if (ifp->if_capenable & IFCAP_LRO) | ||||
features |= VTNET_LRO_FEATURES; | features |= VTNET_LRO_FEATURES; | ||||
else | else | ||||
features &= ~VTNET_LRO_FEATURES; | features &= ~VTNET_LRO_FEATURES; | ||||
} | } | ||||
error = vtnet_ctrl_guest_offloads(sc, | error = vtnet_ctrl_guest_offloads(sc, | ||||
features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 | | features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 | | ||||
▲ Show 20 Lines • Show All 861 Lines • ▼ Show 20 Lines | vtnet_setup_sysctl(struct vtnet_softc *sc) | ||||
SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs", | SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs", | ||||
CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0, | CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0, | ||||
"Number of requested virtqueue pairs"); | "Number of requested virtqueue pairs"); | ||||
SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", | SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", | ||||
CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, | CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, | ||||
"Number of active virtqueue pairs"); | "Number of active virtqueue pairs"); | ||||
vtnet_setup_stat_sysctl(ctx, child, sc); | vtnet_setup_stat_sysctl(ctx, child, sc); | ||||
} | |||||
static void | |||||
vtnet_load_tunables(struct vtnet_softc *sc) | |||||
{ | |||||
sc->vtnet_lro_entry_count = vtnet_tunable_int(sc, | |||||
"lro_entry_count", vtnet_lro_entry_count); | |||||
if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES) | |||||
sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES; | |||||
sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc, | |||||
"lro_mbufq_depth", vtnet_lro_mbufq_depth); | |||||
} | } | ||||
static int | static int | ||||
vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) | vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) | ||||
{ | { | ||||
return (virtqueue_enable_intr(rxq->vtnrx_vq)); | return (virtqueue_enable_intr(rxq->vtnrx_vq)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 152 Lines • Show Last 20 Lines |