Index: sys/dev/axgbe/if_axgbe_pci.c =================================================================== --- sys/dev/axgbe/if_axgbe_pci.c +++ sys/dev/axgbe/if_axgbe_pci.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -62,6 +63,8 @@ MALLOC_DEFINE(M_AXGBE, "axgbe", "axgbe data"); extern struct if_txrx axgbe_txrx; +static int axgbe_sph_enable; +static int axgbe_single_fl; /* Function prototypes */ static void *axgbe_register(device_t); @@ -252,16 +255,11 @@ .isc_vendor_info = axgbe_vendor_info_array, .isc_driver_version = XGBE_DRV_VERSION, - .isc_nrxd_min = {XGBE_RX_DESC_CNT_MIN, XGBE_RX_DESC_CNT_MIN}, - .isc_nrxd_default = {XGBE_RX_DESC_CNT_DEFAULT, XGBE_RX_DESC_CNT_DEFAULT}, - .isc_nrxd_max = {XGBE_RX_DESC_CNT_MAX, XGBE_RX_DESC_CNT_MAX}, .isc_ntxd_min = {XGBE_TX_DESC_CNT_MIN}, .isc_ntxd_default = {XGBE_TX_DESC_CNT_DEFAULT}, .isc_ntxd_max = {XGBE_TX_DESC_CNT_MAX}, - .isc_nfl = 2, .isc_ntxqs = 1, - .isc_nrxqs = 2, .isc_flags = IFLIB_TSO_INIT_IP | IFLIB_NEED_SCRATCH | IFLIB_NEED_ZERO_CSUM | IFLIB_NEED_ETHER_PAD, }; @@ -269,6 +267,70 @@ static void * axgbe_register(device_t dev) { + int axgbe_nfl; + int axgbe_nrxqs; + int error; + char *value = NULL; + + value = kern_getenv("dev.ax.sph_enable"); + if (value) { + axgbe_sph_enable = strtol(value, NULL, 10); + freeenv(value); + } else { + /* + * No tunable found, generate one with default values + * Note: only a reboot will reveal the new kenv + */ + error = kern_setenv("dev.ax.sph_enable", "1"); + if (error) { + printf("Error setting tunable, using default driver values\n"); + } + axgbe_sph_enable = 1; + } + + value = kern_getenv("dev.ax.single_fl"); + if (value) { + axgbe_single_fl = strtol(value, NULL, 10); + freeenv(value); + } else { + /* + * No tunable found, generate one with default values + * Note: only a reboot will reveal the new kenv + */ + error = kern_setenv("dev.ax.single_fl", "0"); + if (error) { + printf("Error setting tunable, using default driver values\n"); + } + axgbe_single_fl = 0; + } + + if (axgbe_single_fl) { + axgbe_nfl = 1; + axgbe_nrxqs = 1; + } else { + axgbe_nfl = 2; + axgbe_nrxqs = 2; + } + + axgbe_sctx_init.isc_nfl = axgbe_nfl; + axgbe_sctx_init.isc_nrxqs = axgbe_nrxqs; + if (axgbe_single_fl) { + axgbe_sctx_init.isc_nrxd_min[0] = XGBE_RX_DESC_CNT_MIN; + + axgbe_sctx_init.isc_nrxd_default[0] = XGBE_RX_DESC_CNT_DEFAULT; + + axgbe_sctx_init.isc_nrxd_max[0] = XGBE_RX_DESC_CNT_MAX; + } else { + axgbe_sctx_init.isc_nrxd_min[0] = XGBE_RX_DESC_CNT_MIN; + axgbe_sctx_init.isc_nrxd_min[1] = XGBE_RX_DESC_CNT_MIN; + + axgbe_sctx_init.isc_nrxd_default[0] = XGBE_RX_DESC_CNT_DEFAULT; + axgbe_sctx_init.isc_nrxd_default[1] = XGBE_RX_DESC_CNT_DEFAULT; + + axgbe_sctx_init.isc_nrxd_max[0] = XGBE_RX_DESC_CNT_MAX; + axgbe_sctx_init.isc_nrxd_max[1] = XGBE_RX_DESC_CNT_MAX; + } + return (&axgbe_sctx_init); } @@ -1292,6 +1354,11 @@ if_softc_ctx_t scctx = sc->scctx; int i, ret; + /* set split header support based on tunable */ + pdata->sph_enabled = axgbe_sph_enable; + /* configure driver for netmap test based on tunable */ + pdata->single_fl = axgbe_single_fl; + /* Initialize ECC timestamps */ pdata->tx_sec_period = ticks; pdata->tx_ded_period = ticks; @@ -1404,6 +1471,8 @@ axgbe_sysctl_init(pdata); + axgbe_pci_init(pdata); + return (0); } /* axgbe_if_attach_post */ @@ -1491,7 +1560,12 @@ struct xgbe_phy_if *phy_if = &pdata->phy_if; struct xgbe_hw_if *hw_if = &pdata->hw_if; int ret = 0; - + + if (!__predict_false((test_bit(XGBE_DOWN, &pdata->dev_state)))) { + axgbe_printf(1, "%s: Starting when XGBE_UP\n", __func__); + return; + } + hw_if->init(pdata); ret = phy_if->phy_start(pdata); @@ -1656,7 +1730,11 @@ MPASS(scctx->isc_nrxqsets > 0); MPASS(scctx->isc_nrxqsets == nrxqsets); - MPASS(nrxqs == 2); + if (pdata->single_fl) { + MPASS(nrxqs == 1); + } else { + MPASS(nrxqs == 2); + } axgbe_printf(1, "%s: rxqsets %d/%d rxqs %d\n", __func__, scctx->isc_nrxqsets, nrxqsets, nrxqs); @@ -1922,7 +2000,7 @@ channel = pdata->channel[i]; snprintf(buf, sizeof(buf), "rxq%d", i); - error = iflib_irq_alloc_generic(ctx, &irq, rid, IFLIB_INTR_RXTX, + error = iflib_irq_alloc_generic(ctx, &irq, rid, IFLIB_INTR_RX, axgbe_msix_que, channel, channel->queue_index, buf); if (error) { @@ -2258,15 +2336,40 @@ static int axgbe_if_promisc_set(if_ctx_t ctx, int flags) { - struct axgbe_if_softc *sc = iflib_get_softc(ctx); + struct axgbe_if_softc *sc = iflib_get_softc(ctx); + struct xgbe_prv_data *pdata = &sc->pdata; + struct ifnet *ifp = pdata->netdev; - if (XGMAC_IOREAD_BITS(&sc->pdata, MAC_PFR, PR) == 1) - return (0); + axgbe_printf(1, "%s: MAC_PFR 0x%x drv_flags 0x%x if_flags 0x%x\n", + __func__, XGMAC_IOREAD(pdata, MAC_PFR), ifp->if_drv_flags, ifp->if_flags); - XGMAC_IOWRITE_BITS(&sc->pdata, MAC_PFR, PR, 1); - XGMAC_IOWRITE_BITS(&sc->pdata, MAC_PFR, VTFE, 0); + if (ifp->if_flags & IFF_PPROMISC) { - return (0); + axgbe_printf(1, "User requested to enter promisc mode\n"); + + if (XGMAC_IOREAD_BITS(pdata, MAC_PFR, PR) == 1) { + axgbe_printf(1, "Already in promisc mode\n"); + return (0); + } + + axgbe_printf(1, "Entering promisc mode\n"); + XGMAC_IOWRITE_BITS(pdata, MAC_PFR, PR, 1); + XGMAC_IOWRITE_BITS(pdata, MAC_PFR, VTFE, 0); + } else { + + axgbe_printf(1, "User requested to leave promisc mode\n"); + + if (XGMAC_IOREAD_BITS(pdata, MAC_PFR, PR) == 0) { + axgbe_printf(1, "Already not in promisc mode\n"); + return (0); + } + + axgbe_printf(1, "Leaving promisc mode\n"); + XGMAC_IOWRITE_BITS(pdata, MAC_PFR, PR, 0); + XGMAC_IOWRITE_BITS(pdata, MAC_PFR, VTFE, 1); + } + + return (0); } static uint64_t Index: sys/dev/axgbe/xgbe-desc.c =================================================================== --- sys/dev/axgbe/xgbe-desc.c +++ sys/dev/axgbe/xgbe-desc.c @@ -191,6 +191,9 @@ rdata->rdesc = rdesc; rdata->rdata_paddr = rdesc_paddr; + rdata->fl_hdr_idx = -1; + rdata->fl_data_idx = -1; + rdesc++; rdesc_paddr += sizeof(struct xgbe_ring_desc); } Index: sys/dev/axgbe/xgbe-dev.c =================================================================== --- sys/dev/axgbe/xgbe-dev.c +++ sys/dev/axgbe/xgbe-dev.c @@ -293,12 +293,14 @@ { unsigned int i; + int tso_enabled = (if_getcapenable(pdata->netdev) & IFCAP_TSO); + for (i = 0; i < pdata->channel_count; i++) { if (!pdata->channel[i]->tx_ring) break; - axgbe_printf(0, "Enabling TSO in channel %d\n", i); - XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_TCR, TSE, 1); + axgbe_printf(1, "TSO in channel %d %s\n", i, tso_enabled ? "enabled" : "disabled"); + XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_TCR, TSE, tso_enabled ? 1 : 0); } } @@ -306,15 +308,33 @@ xgbe_config_sph_mode(struct xgbe_prv_data *pdata) { unsigned int i; + int sph_enable_flag = XGMAC_IOREAD_BITS(pdata, MAC_HWF1R, SPHEN); + + axgbe_printf(1, "sph_enabled %d sph feature enabled?: %d\n", + pdata->sph_enabled, sph_enable_flag); + + if (pdata->sph_enabled && sph_enable_flag) + axgbe_printf(0, "SPH Enabled\n"); for (i = 0; i < pdata->channel_count; i++) { if (!pdata->channel[i]->rx_ring) break; + if (pdata->sph_enabled && sph_enable_flag) { + /* Enable split header feature */ + XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_CR, SPH, 1); + } else { + /* Disable split header feature */ + XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_CR, SPH, 0); + } - XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_CR, SPH, 1); + /* per-channel confirmation of SPH being disabled/enabled */ + int val = XGMAC_DMA_IOREAD_BITS(pdata->channel[i], DMA_CH_CR, SPH); + axgbe_printf(1, "%s: SPH %s in channel %d - val:%d\n", __func__, + (val ? "enabled" : "disabled"), i, val); } - - XGMAC_IOWRITE_BITS(pdata, MAC_RCR, HDSMS, XGBE_SPH_HDSMS_SIZE); + + if (pdata->sph_enabled && sph_enable_flag) + XGMAC_IOWRITE_BITS(pdata, MAC_RCR, HDSMS, XGBE_SPH_HDSMS_SIZE); } static int @@ -953,8 +973,8 @@ { unsigned int pr_mode, am_mode; - pr_mode = ((pdata->netdev->if_drv_flags & IFF_PPROMISC) != 0); - am_mode = ((pdata->netdev->if_drv_flags & IFF_ALLMULTI) != 0); + pr_mode = ((pdata->netdev->if_flags & IFF_PPROMISC) != 0); + am_mode = ((pdata->netdev->if_flags & IFF_ALLMULTI) != 0); xgbe_set_promiscuous_mode(pdata, pr_mode); xgbe_set_all_multicast_mode(pdata, am_mode); @@ -1352,7 +1372,6 @@ if (XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, CTXT)) { /* TODO - Timestamp Context Descriptor */ - XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, CONTEXT, 1); XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, Index: sys/dev/axgbe/xgbe-sysctl.c =================================================================== --- sys/dev/axgbe/xgbe-sysctl.c +++ sys/dev/axgbe/xgbe-sysctl.c @@ -1614,6 +1614,16 @@ SYSCTL_ADD_UINT(clist, top, OID_AUTO, "axgbe_debug_level", CTLFLAG_RWTUN, &pdata->debug_level, 0, "axgbe log level -- higher is verbose"); + SYSCTL_ADD_UINT(clist, top, OID_AUTO, "sph_enabled", + CTLFLAG_RDTUN, &pdata->sph_enabled, 1, + "shows if the split header feature is enabled - change this by setting " + "dev.ax.sph_enable=\"{0|1}\" in /boot/loader.conf"); + + SYSCTL_ADD_UINT(clist, top, OID_AUTO, "single_fl", + CTLFLAG_RDTUN, &pdata->single_fl, 0, + "shows if drvier is exercised with netmap - change this by setting " + "dev.ax.single_fl=\"{0|1}\" in /boot/loader.conf"); + SYSCTL_ADD_PROC(clist, top, OID_AUTO, "xgmac_register", CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, pdata, 0, sysctl_xgmac_reg_addr_handler, "IU", Index: sys/dev/axgbe/xgbe-txrx.c =================================================================== --- sys/dev/axgbe/xgbe-txrx.c +++ sys/dev/axgbe/xgbe-txrx.c @@ -466,17 +466,23 @@ unsigned int inte; uint8_t count = iru->iru_count; int i, j; + bool config_intr = false; axgbe_printf(1, "--> %s: rxq %d fl %d pidx %d count %d ring cur %d " "dirty %d\n", __func__, iru->iru_qsidx, iru->iru_flidx, iru->iru_pidx, count, ring->cur, ring->dirty); - for (i = iru->iru_pidx, j = 0 ; j < count ; i++, j++) { + if (!pdata->single_fl) + j = iru->iru_pidx; + else + j = ring->dirty; + + for (i = 0; count > 0; i++, count--) { - if (i == XGBE_RX_DESC_CNT_DEFAULT) - i = 0; + if (j == sc->scctx->isc_nrxd[0]) + j = 0; - rdata = XGBE_GET_DESC_DATA(ring, i); + rdata = XGBE_GET_DESC_DATA(ring, j); rdesc = rdata->rdesc; if (__predict_false(XGMAC_GET_BITS_LE(rdesc->desc3, @@ -486,28 +492,56 @@ } /* Assuming split header is enabled */ - if (iru->iru_flidx == 0) { + if (!pdata->single_fl) { + if (iru->iru_flidx == 0) { + + /* Fill header/buffer1 address */ + rdesc->desc0 = + cpu_to_le32(lower_32_bits(iru->iru_paddrs[i])); + rdesc->desc1 = + cpu_to_le32(upper_32_bits(iru->iru_paddrs[i])); + } else { - /* Fill header/buffer1 address */ - rdesc->desc0 = - cpu_to_le32(lower_32_bits(iru->iru_paddrs[j])); - rdesc->desc1 = - cpu_to_le32(upper_32_bits(iru->iru_paddrs[j])); + /* Fill data/buffer2 address */ + rdesc->desc2 = + cpu_to_le32(lower_32_bits(iru->iru_paddrs[i])); + rdesc->desc3 = + cpu_to_le32(upper_32_bits(iru->iru_paddrs[i])); + + config_intr = true; + } + + j++; } else { + if (rdata->fl_hdr_idx == -1) { + /* Fill header/buffer1 address */ + rdesc->desc0 = + cpu_to_le32(lower_32_bits(iru->iru_paddrs[i])); + rdesc->desc1 = + cpu_to_le32(upper_32_bits(iru->iru_paddrs[i])); + rdata->fl_hdr_idx = iru->iru_idxs[i]; + } else { + /* Fill data/buffer2 address */ + rdesc->desc2 = + cpu_to_le32(lower_32_bits(iru->iru_paddrs[i])); + rdesc->desc3 = + cpu_to_le32(upper_32_bits(iru->iru_paddrs[i])); + rdata->fl_data_idx = iru->iru_idxs[i]; + + config_intr = true; + + j++; + } + } - /* Fill data/buffer2 address */ - rdesc->desc2 = - cpu_to_le32(lower_32_bits(iru->iru_paddrs[j])); - rdesc->desc3 = - cpu_to_le32(upper_32_bits(iru->iru_paddrs[j])); + if (config_intr) { if (!rx_usecs && !rx_frames) { /* No coalescing, interrupt for every descriptor */ inte = 1; } else { /* Set interrupt based on Rx frame coalescing setting */ - if (rx_frames && - !(((ring->dirty + 1) &(ring->rdesc_count - 1)) % rx_frames)) + if (rx_frames && !((ring->dirty + 1) % rx_frames)) inte = 1; else inte = 0; @@ -520,6 +554,8 @@ wmb(); ring->dirty = ((ring->dirty + 1) & (ring->rdesc_count - 1)); + + config_intr = false; } } @@ -539,12 +575,18 @@ axgbe_printf(1, "--> %s: rxq %d fl %d pidx %d cur %d dirty %d\n", __func__, qsidx, flidx, pidx, ring->cur, ring->dirty); - if (flidx == 1) { + rdata = XGBE_GET_DESC_DATA(ring, pidx); - rdata = XGBE_GET_DESC_DATA(ring, pidx); - - XGMAC_DMA_IOWRITE(channel, DMA_CH_RDTR_LO, - lower_32_bits(rdata->rdata_paddr)); + /* + * update RX descriptor tail pointer in hardware to indicate + * that new buffers are present in the allocated memory region + */ + if (!pdata->single_fl) { + if (flidx == 1) { + XGMAC_DMA_IOWRITE(channel, DMA_CH_RDTR_LO, lower_32_bits(rdata->rdata_paddr)); + } + } else { + XGMAC_DMA_IOWRITE(channel, DMA_CH_RDTR_LO, lower_32_bits(rdata->rdata_paddr)); } wmb(); @@ -560,12 +602,17 @@ struct xgbe_ring_data *rdata; struct xgbe_ring_desc *rdesc; unsigned int cur; - int count; + int count = 0; uint8_t incomplete = 1, context_next = 0, running = 0; axgbe_printf(1, "--> %s: rxq %d idx %d budget %d cur %d dirty %d\n", __func__, qsidx, idx, budget, ring->cur, ring->dirty); + if (__predict_false(test_bit(XGBE_DOWN, &pdata->dev_state))) { + axgbe_printf(0, "%s: Polling when XGBE_DOWN\n", __func__); + return (count); + } + cur = ring->cur; for (count = 0; count <= budget; ) { @@ -607,13 +654,16 @@ static unsigned int xgbe_rx_buf1_len(struct xgbe_prv_data *pdata, struct xgbe_ring_data *rdata, - struct xgbe_packet_data *packet) + struct xgbe_packet_data *packet, unsigned int len) { + unsigned int ret = 0; - /* Always zero if not the first descriptor */ - if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, FIRST)) { - axgbe_printf(1, "%s: Not First\n", __func__); - return (0); + if (pdata->sph_enabled) { + /* Always zero if not the first descriptor */ + if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, FIRST)) { + axgbe_printf(1, "%s: Not First\n", __func__); + return (0); + } } /* First descriptor with split header, return header length */ @@ -623,21 +673,32 @@ } /* First descriptor but not the last descriptor and no split header, - * so the full buffer was used + * so the full buffer was used, 256 represents the hardcoded value of + * a max header split defined in the hardware */ if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, LAST)) { axgbe_printf(1, "%s: Not last %d\n", __func__, pdata->rx_buf_size); - return (256); + if (pdata->sph_enabled) { + return (256); + } else { + return (pdata->rx_buf_size); + } } /* First descriptor and last descriptor and no split header, so - * calculate how much of the buffer was used + * calculate how much of the buffer was used, we can return the + * segment length or the remaining bytes of the packet */ axgbe_printf(1, "%s: pkt_len %d buf_size %d\n", __func__, rdata->rx.len, pdata->rx_buf_size); - return (min_t(unsigned int, 256, rdata->rx.len)); + if (pdata->sph_enabled) + ret = min_t(unsigned int, 256, rdata->rx.len); + else + ret = rdata->rx.len - len; + + return (ret); } static unsigned int @@ -710,7 +771,7 @@ if (!context) { /* Get the data length in the descriptor buffers */ - buf1_len = xgbe_rx_buf1_len(pdata, rdata, packet); + buf1_len = xgbe_rx_buf1_len(pdata, rdata, packet, len); len += buf1_len; buf2_len = xgbe_rx_buf2_len(pdata, rdata, packet, len); len += buf2_len; @@ -722,10 +783,20 @@ "buf2 %d len %d frags %d error %d\n", __func__, last, context, context_next, buf1_len, buf2_len, len, i, packet->errors); - axgbe_add_frag(pdata, ri, prev_cur, buf1_len, i, 0); - i++; - axgbe_add_frag(pdata, ri, prev_cur, buf2_len, i, 1); - i++; + if (!pdata->single_fl) { + axgbe_add_frag(pdata, ri, prev_cur, buf1_len, i, 0); + i++; + axgbe_add_frag(pdata, ri, prev_cur, buf2_len, i, 1); + i++; + } else { + axgbe_add_frag(pdata, ri, rdata->fl_hdr_idx, buf1_len, i, 0); + i++; + axgbe_add_frag(pdata, ri, rdata->fl_data_idx, buf2_len, i, 0); + i++; + + rdata->fl_hdr_idx = -1; + rdata->fl_data_idx = -1; + } if (!last || context_next) goto read_again; @@ -758,7 +829,7 @@ } if (__predict_false(len == 0)) - axgbe_error("%s: Zero len packet\n", __func__); + axgbe_printf(1, "%s: Discarding Zero len packet\n", __func__); if (__predict_false(len > max_len)) axgbe_error("%s: Big packet %d/%d\n", __func__, len, max_len); Index: sys/dev/axgbe/xgbe.h =================================================================== --- sys/dev/axgbe/xgbe.h +++ sys/dev/axgbe/xgbe.h @@ -471,6 +471,9 @@ unsigned int error; } state; + /* Free list index associated with this descriptor */ + int fl_hdr_idx; + int fl_data_idx; }; struct xgbe_ring { @@ -1296,6 +1299,8 @@ uint64_t rx_coalesce_usecs; unsigned int debug_level; + unsigned int sph_enabled; /* toggles the split header feature, requires complete restart */ + unsigned int single_fl; /* rxq uses 1 freelist(default 2), requires complete restart */ }; struct axgbe_if_softc { Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -2623,7 +2623,8 @@ } } else { fl->ifl_sds.ifsd_m[cidx] = NULL; - *pf_rv = PFIL_PASS; + if (pf_rv != NULL) + *pf_rv = PFIL_PASS; } if (unload && irf->irf_len != 0)