Index: sys/dev/virtio/network/if_vtnet.c
===================================================================
--- sys/dev/virtio/network/if_vtnet.c
+++ sys/dev/virtio/network/if_vtnet.c
@@ -233,6 +233,7 @@
 static void	vtnet_setup_txq_sysctl(struct sysctl_ctx_list *,
 		    struct sysctl_oid_list *, struct vtnet_txq *);
 static void	vtnet_setup_queue_sysctl(struct vtnet_softc *);
+static void	vtnet_load_tunables(struct vtnet_softc *);
 static void	vtnet_setup_sysctl(struct vtnet_softc *);
 
 static int	vtnet_rxq_enable_intr(struct vtnet_rxq *);
@@ -295,6 +296,15 @@
     &vtnet_rx_process_limit, 0,
     "Number of RX segments processed in one pass");
 
+static int vtnet_lro_entry_count = 128;
+SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN,
+    &vtnet_lro_entry_count, 0, "Software LRO entry count");
+
+/* Enable sorted LRO, and the depth of the mbuf queue. */
+static int vtnet_lro_mbufq_depth = 0;
+SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN,
+    &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue");
+
 static uma_zone_t vtnet_tx_header_zone;
 
 static struct virtio_feature_desc vtnet_feature_desc[] = {
@@ -429,6 +439,7 @@
 
 	VTNET_CORE_LOCK_INIT(sc);
 	callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
+	vtnet_load_tunables(sc);
 
 	error = vtnet_alloc_interface(sc);
 	if (error) {
@@ -682,8 +693,8 @@
 		 */
 		if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
 			device_printf(dev,
-			    "LRO disabled since both mergeable buffers and "
-			    "indirect descriptors were not negotiated\n");
+			    "Host LRO disabled since both mergeable buffers "
+			    "and indirect descriptors were not negotiated\n");
 			features &= ~VTNET_LRO_FEATURES;
 			negotiated_features =
 			    virtio_negotiate_features(dev, features);
@@ -740,6 +751,14 @@
 	else
 		sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE;
 
+	/*
+	 * Favor "hardware" LRO if negotiated, but support software LRO as
+	 * a fallback; there is usually little benefit (or worse) with both.
+	 */
+	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 &&
+	    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0)
+		sc->vtnet_flags |= VTNET_FLAG_SW_LRO;
+
 	if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) ||
 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
 	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
@@ -809,9 +828,11 @@
 		return (ENOMEM);
 
 #if defined(INET) || defined(INET6)
-	if (tcp_lro_init(&rxq->vtnrx_lro) != 0)
-		return (ENOMEM);
-	rxq->vtnrx_lro.ifp = sc->vtnet_ifp;
+	if (vtnet_software_lro(sc)) {
+		if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp,
+		    sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0)
+			return (ENOMEM);
+	}
 #endif
 
 	NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
@@ -1138,10 +1159,8 @@
 		    vtnet_fixup_needs_csum) != 0)
 			sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM;
 
-		if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
-		    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) ||
-		    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_ECN))
-			ifp->if_capabilities |= IFCAP_LRO;
+		/* Support either "hardware" or software LRO. */
+		ifp->if_capabilities |= IFCAP_LRO;
 	}
 
 	if (ifp->if_capabilities & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) {
@@ -1329,6 +1348,11 @@
 		else
 			reinit = 1;
 
+		/* BMV: Avoid needless renegotiation for just software LRO. */
+		if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) ==
+		    IFCAP_LRO && vtnet_software_lro(sc))
+			reinit = update = 0;
+
 		if (mask & IFCAP_RXCSUM)
 			ifp->if_capenable ^= IFCAP_RXCSUM;
 		if (mask & IFCAP_RXCSUM_IPV6)
@@ -1935,6 +1959,23 @@
 	return (1);
 }
 
+#if defined(INET) || defined(INET6)
+static int
+vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m)
+{
+	struct lro_ctrl *lro;
+
+	lro = &rxq->vtnrx_lro;
+
+	if (lro->lro_mbuf_max != 0) {
+		tcp_lro_queue_mbuf(lro, m);
+		return (0);
+	}
+
+	return (tcp_lro_rx(lro, m, 0));
+}
+#endif
+
 static void
 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
     struct virtio_net_hdr *hdr)
@@ -1973,8 +2014,8 @@
 	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;
 
 #if defined(INET) || defined(INET6)
-	if (ifp->if_capenable & IFCAP_LRO && rxq->vtnrx_lro.lro_cnt != 0) {
-		if (tcp_lro_rx(&rxq->vtnrx_lro, m, 0) == 0)
+	if (vtnet_software_lro(sc) && ifp->if_capenable & IFCAP_LRO) {
+		if (vtnet_lro_rx(rxq, m) == 0)
 			return;
 	}
 #endif
@@ -3294,7 +3335,7 @@
 			features &= ~VIRTIO_NET_F_GUEST_CSUM;
 	}
 
-	if (ifp->if_capabilities & IFCAP_LRO) {
+	if (ifp->if_capabilities & IFCAP_LRO && !vtnet_software_lro(sc)) {
 		if (ifp->if_capenable & IFCAP_LRO)
 			features |= VTNET_LRO_FEATURES;
 		else
@@ -4169,6 +4210,19 @@
 	vtnet_setup_stat_sysctl(ctx, child, sc);
 }
 
+static void
+vtnet_load_tunables(struct vtnet_softc *sc)
+{
+
+	sc->vtnet_lro_entry_count = vtnet_tunable_int(sc,
+	    "lro_entry_count", vtnet_lro_entry_count);
+	if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES)
+		sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES;
+
+	sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc,
+	    "lro_mbufq_depth", vtnet_lro_mbufq_depth);
+}
+
 static int
 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
 {
Index: sys/dev/virtio/network/if_vtnetvar.h
===================================================================
--- sys/dev/virtio/network/if_vtnetvar.h
+++ sys/dev/virtio/network/if_vtnetvar.h
@@ -79,7 +79,7 @@
 	struct vtnet_rxq_stats	 vtnrx_stats;
 	struct taskqueue	*vtnrx_tq;
 	struct task		 vtnrx_intrtask;
-	struct lro_ctrl	 	 vtnrx_lro;
+	struct lro_ctrl		 vtnrx_lro;
 #ifdef DEV_NETMAP
 	uint32_t		 vtnrx_nm_refill;
 	struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;
@@ -156,6 +156,7 @@
 #define VTNET_FLAG_EVENT_IDX	 0x0800
 #define VTNET_FLAG_SUSPENDED	 0x1000
 #define VTNET_FLAG_FIXUP_NEEDS_CSUM 0x2000
+#define VTNET_FLAG_SW_LRO	 0x4000
 
 	int			 vtnet_link_active;
 	int			 vtnet_hdr_size;
@@ -169,6 +170,8 @@
 	int			 vtnet_act_vq_pairs;
 	int			 vtnet_req_vq_pairs;
 	int			 vtnet_max_vq_pairs;
+	int			 vtnet_lro_entry_count;
+	int			 vtnet_lro_mbufq_depth;
 
 	struct virtqueue	*vtnet_ctrl_vq;
 	struct vtnet_mac_filter	*vtnet_mac_filter;
@@ -193,6 +196,12 @@
 	return ((sc->vtnet_flags & VTNET_FLAG_MODERN) != 0);
 }
 
+static bool
+vtnet_software_lro(struct vtnet_softc *sc)
+{
+	return ((sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0);
+}
+
 /*
  * Maximum number of queue pairs we will autoconfigure to.
  */