Index: sys/dev/ixgbe/if_ix.c =================================================================== --- sys/dev/ixgbe/if_ix.c +++ sys/dev/ixgbe/if_ix.c @@ -165,7 +165,8 @@ static void ixgbe_free_pci_resources(if_ctx_t ctx); static int ixgbe_msix_link(void *arg); -static int ixgbe_msix_que(void *arg); +static int ixgbe_msix_rx_que(void *arg); +static int ixgbe_msix_tx_que(void *arg); static void ixgbe_initialize_rss_mapping(struct adapter *adapter); static void ixgbe_initialize_receive_units(if_ctx_t ctx); static void ixgbe_initialize_transmit_units(if_ctx_t ctx); @@ -357,6 +358,11 @@ SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &ixgbe_enable_aim, 0, "Enable adaptive interrupt moderation"); +static int ixgbe_enable_tx_irq = FALSE; +TUNABLE_INT("hw.ix.enable_tx_irq", &ixgbe_enable_tx_irq); +SYSCTL_INT(_hw_ix, OID_AUTO, enable_tx_irq, CTLFLAG_RDTUN, + &ixgbe_enable_tx_irq, 0, "Enable IRQ for Tx"); + #if 0 /* Keep running tab on them for sanity check */ static int ixgbe_total_ports; @@ -904,6 +910,10 @@ hw->subsystem_vendor_id = pci_get_subvendor(dev); hw->subsystem_device_id = pci_get_subdevice(dev); + if (ixgbe_enable_tx_irq) { + iflib_enable_tx_irq(ctx); + } + /* Do base PCI setup - map BAR0 */ if (ixgbe_allocate_pci_resources(ctx)) { device_printf(dev, "Allocation of PCI resources failed\n"); @@ -1602,6 +1612,9 @@ ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", CTLFLAG_RD, &txr->tso_tx, "TSO"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", + CTLFLAG_RD, &txr->tx_bytes, + "Queue Bytes Transmitted"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", CTLFLAG_RD, &txr->total_packets, "Queue Packets Transmitted"); @@ -2051,11 +2064,11 @@ snprintf(buf, sizeof(buf), "rxq%d", i); error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, - IFLIB_INTR_RXTX, ixgbe_msix_que, rx_que, rx_que->rxr.me, buf); + IFLIB_INTR_RXTX, ixgbe_msix_rx_que, rx_que, rx_que->rxr.me, buf); if (error) { device_printf(iflib_get_dev(ctx), - "Failed to allocate que int %d err: %d", i, error); + "Failed to allocate Rx que irq %d err: %d", i, error); adapter->num_rx_queues = i + 1; goto fail; } @@ -2084,10 +2097,24 @@ for (int i = 0; i < adapter->num_tx_queues; i++) { snprintf(buf, sizeof(buf), "txq%d", i); tx_que = &adapter->tx_queues[i]; - tx_que->msix = i % adapter->num_rx_queues; - iflib_softirq_alloc_generic(ctx, - &adapter->rx_queues[tx_que->msix].que_irq, - IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); + if (ixgbe_enable_tx_irq) { + rid = vector + 1; + error = iflib_irq_alloc_generic(ctx, &tx_que->que_irq, rid, + IFLIB_INTR_TX, ixgbe_msix_tx_que, tx_que, tx_que->txr.me, buf); + if (error) { + device_printf(iflib_get_dev(ctx), + "Failed to allocate Tx que irq %d err: %d", i, error); + adapter->num_tx_queues = i + 1; + goto fail; + } + tx_que->msix = vector; + vector++; + } else { + tx_que->msix = i % adapter->num_rx_queues; + iflib_softirq_alloc_generic(ctx, + &adapter->rx_queues[tx_que->msix].que_irq, + IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); + } } rid = vector + 1; error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, @@ -2106,36 +2133,20 @@ rx_que = adapter->rx_queues; for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++) iflib_irq_free(ctx, &rx_que->que_irq); + if (ixgbe_enable_tx_irq) { + tx_que = adapter->tx_queues; + for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) { + iflib_irq_free(ctx, &tx_que->que_irq); + } + } return (error); } /* ixgbe_if_msix_intr_assign */ -static inline void -ixgbe_perform_aim(struct adapter *adapter, struct ix_rx_queue *que) +static inline uint32_t +ixgbe_calculate_new_itr(struct adapter *adapter, uint32_t itr) { - uint32_t newitr = 0; - struct rx_ring *rxr = &que->rxr; - - /* - * Do Adaptive Interrupt Moderation: - * - Write out last calculated setting - * - Calculate based on average size over - * the last interval. - */ - if (que->eitr_setting) { - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(que->msix), - que->eitr_setting); - } - - que->eitr_setting = 0; - /* Idle, do nothing */ - if (rxr->bytes == 0) { - return; - } - - if ((rxr->bytes) && (rxr->packets)) { - newitr = (rxr->bytes / rxr->packets); - } + uint32_t newitr = itr; newitr += 24; /* account for hardware frame, crc */ /* set an upper boundary */ @@ -2154,25 +2165,71 @@ newitr |= IXGBE_EITR_CNT_WDIS; } - /* save for next interrupt */ - que->eitr_setting = newitr; - - /* Reset state */ - rxr->bytes = 0; - rxr->packets = 0; - - return; + return newitr; } /********************************************************************* - * ixgbe_msix_que - MSI-X Queue Interrupt Service routine + * MSI-X Queue Interrupt Service routine **********************************************************************/ static int -ixgbe_msix_que(void *arg) +ixgbe_msix_rx_que(void *arg) { struct ix_rx_queue *que = arg; - struct adapter *adapter = que->adapter; - struct ifnet *ifp = iflib_get_ifp(que->adapter->ctx); + struct adapter *adapter = que->adapter; + struct ifnet *ifp = iflib_get_ifp(que->adapter->ctx); + struct rx_ring *rxr = &que->rxr; + uint32_t newitr = 0; + + /* Protect against spurious interrupts */ + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + return (FILTER_HANDLED); + + ixgbe_disable_queue(adapter, que->msix); + ++que->irqs; + + /* Check for AIM */ + if (adapter->enable_aim) { + /* + * Do Adaptive Interrupt Moderation: + * - Write out last calculated setting + * - Calculate based on average size over + * the last interval. + */ + if (que->eitr_setting) { + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(que->msix), + que->eitr_setting); + } + + que->eitr_setting = 0; + /* Idle, do nothing */ + if (rxr->bytes == 0) { + return (FILTER_SCHEDULE_THREAD); + } + + if ((rxr->bytes) && (rxr->packets)) { + newitr = (rxr->bytes / rxr->packets); + } + + newitr = ixgbe_calculate_new_itr(adapter, newitr); + + /* save for next interrupt */ + que->eitr_setting = newitr; + + /* Reset state */ + rxr->bytes = 0; + rxr->packets = 0; + } + return (FILTER_SCHEDULE_THREAD); +} /* ixgbe_msix_rx_que */ + +static int +ixgbe_msix_tx_que(void *arg) +{ + struct ix_tx_queue *que = arg; + struct adapter *adapter = que->adapter; + struct ifnet *ifp = iflib_get_ifp(que->adapter->ctx); + struct tx_ring *txr = &que->txr; + uint32_t newitr = 0; /* Protect against spurious interrupts */ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) @@ -2183,11 +2240,39 @@ /* Check for AIM */ if (adapter->enable_aim) { - ixgbe_perform_aim(adapter, que); + /* + * Do Adaptive Interrupt Moderation: + * - Write out last calculated setting + * - Calculate based on average size over + * the last interval. + */ + if (que->eitr_setting) { + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(que->msix), + que->eitr_setting); + } + + que->eitr_setting = 0; + /* Idle, do nothing */ + if (txr->bytes == 0) { + return (FILTER_SCHEDULE_THREAD); + } + + if ((txr->bytes) && (txr->packets)) { + newitr = (txr->bytes / txr->packets); + } + + newitr = ixgbe_calculate_new_itr(adapter, newitr); + + /* save for next interrupt */ + que->eitr_setting = newitr; + + /* Reset state */ + txr->bytes = 0; + txr->packets = 0; } return (FILTER_SCHEDULE_THREAD); -} /* ixgbe_msix_que */ +} /* ixgbe_msix_tx_que */ /************************************************************************ * ixgbe_media_status - Media Ioctl callback @@ -3233,6 +3318,11 @@ /* ... and the TX */ ixgbe_set_ivar(adapter, txr->me, tx_que->msix, 1); + + if (ixgbe_enable_tx_irq) { + /* Set an Initial EITR value */ + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(tx_que->msix), newitr); + } } /* For the Link interrupt */ ixgbe_set_ivar(adapter, 1, adapter->vector, -1); @@ -3946,6 +4036,15 @@ } } + if (ixgbe_enable_tx_irq) { + struct ix_tx_queue *tx_que = adapter->tx_queues; + if (tx_que != NULL) { + for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) { + iflib_irq_free(ctx, &tx_que->que_irq); + } + } + } + if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(adapter->pci_mem), adapter->pci_mem); Index: sys/dev/ixgbe/ix_txrx.c =================================================================== --- sys/dev/ixgbe/ix_txrx.c +++ sys/dev/ixgbe/ix_txrx.c @@ -238,10 +238,12 @@ } txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | flags); - txr->bytes += pi->ipi_len; + txr->bytes += pi->ipi_len; // for AIM pi->ipi_new_pidx = i; ++txr->total_packets; + txr->tx_bytes = txr->bytes; + txr->packets = txr->total_packets; // for AIM return (0); } /* ixgbe_isc_txd_encap */ Index: sys/dev/ixgbe/ixgbe.h =================================================================== --- sys/dev/ixgbe/ixgbe.h +++ sys/dev/ixgbe/ixgbe.h @@ -297,6 +297,7 @@ u32 packets; /* Soft Stats */ u64 tso_tx; + u64 tx_bytes; u64 total_packets; }; @@ -349,7 +350,11 @@ struct ix_tx_queue { struct adapter *adapter; u32 msix; /* This queue's MSIX vector */ + u32 eitr_setting; + struct resource *res; struct tx_ring txr; + struct if_irq que_irq; + u64 irqs; }; #define IXGBE_MAX_VF_MC 30 /* Max number of multicast entries */ Index: sys/net/iflib.h =================================================================== --- sys/net/iflib.h +++ sys/net/iflib.h @@ -193,7 +193,8 @@ int isc_vectors; int isc_nrxqsets; int isc_ntxqsets; - uint16_t __spare0__; + uint8_t isc_tx_irq; + uint8_t __spare0__; uint32_t __spare1__; int isc_msix_bar; /* can be model specific - initialize in attach_pre */ int isc_tx_nsegments; /* can be model specific - initialize in attach_pre */ @@ -420,21 +421,17 @@ * field accessors */ void *iflib_get_softc(if_ctx_t ctx); - device_t iflib_get_dev(if_ctx_t ctx); - if_t iflib_get_ifp(if_ctx_t ctx); - struct ifmedia *iflib_get_media(if_ctx_t ctx); - if_softc_ctx_t iflib_get_softc_ctx(if_ctx_t ctx); if_shared_ctx_t iflib_get_sctx(if_ctx_t ctx); +uint32_t iflib_get_rx_mbuf_sz(if_ctx_t ctx); void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]); void iflib_request_reset(if_ctx_t ctx); uint8_t iflib_in_detach(if_ctx_t ctx); - -uint32_t iflib_get_rx_mbuf_sz(if_ctx_t ctx); +void iflib_enable_tx_irq(if_ctx_t ctx); /* * If the driver can plug cleanly in to newbus use these Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -643,8 +643,11 @@ &iflib_encap_txd_encap_fail, 0, "# driver encap failures"); static int iflib_task_fn_rxs; +static int iflib_task_fn_txs; static int iflib_rx_intr_enables; static int iflib_fast_intrs; +static int iflib_fast_intr_rxtxs; +static int iflib_fast_intr_ctxs; static int iflib_rx_unavail; static int iflib_rx_ctx_inactive; static int iflib_rx_if_input; @@ -654,10 +657,16 @@ SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD, &iflib_task_fn_rxs, 0, "# task_fn_rx calls"); +SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_tx, CTLFLAG_RD, + &iflib_task_fn_txs, 0, "# task_fn_tx calls"); SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD, &iflib_rx_intr_enables, 0, "# RX intr enables"); SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD, &iflib_fast_intrs, 0, "# fast_intr calls"); +SYSCTL_INT(_net_iflib, OID_AUTO, fast_intr_rxtxs, CTLFLAG_RD, + &iflib_fast_intr_rxtxs, 0, "# fast_intr_rxtx calls"); +SYSCTL_INT(_net_iflib, OID_AUTO, fast_intr_ctxs, CTLFLAG_RD, + &iflib_fast_intr_ctxs, 0, "# fast_intr_ctx calls"); SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD, &iflib_rx_unavail, 0, "# times rxeof called with no available data"); SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD, @@ -679,9 +688,9 @@ iflib_txq_drain_notready = iflib_encap_load_mbuf_fail = iflib_encap_pad_mbuf_fail = iflib_encap_txq_avail_fail = iflib_encap_txd_encap_fail = - iflib_task_fn_rxs = iflib_rx_intr_enables = iflib_fast_intrs = - iflib_rx_unavail = - iflib_rx_ctx_inactive = iflib_rx_if_input = + iflib_task_fn_rxs = iflib_task_fn_txs = iflib_rx_intr_enables = + iflib_fast_intrs = iflib_fast_intr_rxtxs = iflib_fast_intr_ctxs = + iflib_rx_unavail = iflib_rx_ctx_inactive = iflib_rx_if_input = iflib_rxd_flush = 0; } @@ -1566,7 +1575,7 @@ qidx_t txqid; bool intr_enable, intr_legacy; - DBG_COUNTER_INC(fast_intrs); + DBG_COUNTER_INC(fast_intr_rxtxs); if (info->ifi_filter != NULL) { result = info->ifi_filter(info->ifi_filter_arg); if ((result & FILTER_SCHEDULE_THREAD) == 0) @@ -1617,7 +1626,7 @@ struct grouptask *gtask = info->ifi_task; int result; - DBG_COUNTER_INC(fast_intrs); + DBG_COUNTER_INC(fast_intr_ctxs); if (info->ifi_filter != NULL) { result = info->ifi_filter(info->ifi_filter_arg); if ((result & FILTER_SCHEDULE_THREAD) == 0) @@ -6680,6 +6689,9 @@ } vectors = rx_queues + admincnt; + if (scctx->isc_tx_irq) { + vectors += tx_queues; + } if (msgs < vectors) { device_printf(dev, "insufficient number of MSI-X vectors " @@ -6687,9 +6699,9 @@ goto msi; } - device_printf(dev, "Using %d RX queues %d TX queues\n", rx_queues, - tx_queues); msgs = vectors; + device_printf(dev, "Using %d RX queues %d TX queues msgs %d\n", + rx_queues, tx_queues, msgs); if ((err = pci_alloc_msix(dev, &vectors)) == 0) { if (vectors != msgs) { device_printf(dev, @@ -7052,6 +7064,12 @@ STATE_UNLOCK(ctx); } +void +iflib_enable_tx_irq(if_ctx_t ctx) +{ + ctx->ifc_softc_ctx.isc_tx_irq = 1; +} + #ifndef __NO_STRICT_ALIGNMENT static struct mbuf * iflib_fixup_rx(struct mbuf *m)