diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c index 271c35015a9f..1127565282c1 100644 --- a/sys/dev/oce/oce_if.c +++ b/sys/dev/oce/oce_if.c @@ -1,2962 +1,2966 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2013 Emulex * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Emulex Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * Contact Information: * freebsd-drivers@emulex.com * * Emulex * 3333 Susan Street * Costa Mesa, CA 92626 */ /* $FreeBSD$ */ #include "opt_inet6.h" #include "opt_inet.h" #include "oce_if.h" #include "oce_user.h" #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) /* UE Status Low CSR */ static char *ue_status_low_desc[] = { "CEV", "CTX", "DBUF", "ERX", "Host", "MPU", "NDMA", "PTC ", "RDMA ", "RXF ", "RXIPS ", "RXULP0 ", "RXULP1 ", "RXULP2 ", "TIM ", "TPOST ", "TPRE ", "TXIPS ", "TXULP0 ", "TXULP1 ", "UC ", "WDMA ", "TXULP2 ", "HOST1 ", "P0_OB_LINK ", "P1_OB_LINK ", "HOST_GPIO ", "MBOX ", "AXGMAC0", "AXGMAC1", "JTAG", "MPU_INTPEND" }; /* UE Status High CSR */ static char *ue_status_hi_desc[] = { "LPCMEMHOST", "MGMT_MAC", "PCS0ONLINE", "MPU_IRAM", "PCS1ONLINE", "PCTL0", "PCTL1", "PMEM", "RR", "TXPB", "RXPP", "XAUI", "TXP", "ARM", "IPC", "HOST2", "HOST3", "HOST4", "HOST5", "HOST6", "HOST7", "HOST8", "HOST9", "NETC", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown" }; struct oce_common_cqe_info{ uint8_t vtp:1; uint8_t l4_cksum_pass:1; uint8_t ip_cksum_pass:1; uint8_t ipv6_frame:1; uint8_t qnq:1; uint8_t rsvd:3; uint8_t num_frags; uint16_t pkt_size; uint16_t vtag; }; /* Driver entry points prototypes */ static int oce_probe(device_t dev); static int oce_attach(device_t dev); static int oce_detach(device_t dev); static int oce_shutdown(device_t dev); static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); static void oce_init(void *xsc); static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); static void oce_multiq_flush(struct ifnet *ifp); /* Driver interrupt routines protypes */ static void oce_intr(void *arg, int pending); static int oce_setup_intr(POCE_SOFTC sc); static int oce_fast_isr(void *arg); static int oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)); /* Media callbacks prototypes */ static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); static int oce_media_change(struct ifnet *ifp); /* Transmit routines prototypes */ static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); static void oce_process_tx_completion(struct oce_wq *wq); static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq); /* Receive routines prototypes */ static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); static uint16_t oce_rq_handler_lro(void *arg); static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); /* Helper function prototypes in this file */ static int oce_attach_ifp(POCE_SOFTC sc); static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); static int oce_vid_config(POCE_SOFTC sc); static void oce_mac_addr_set(POCE_SOFTC sc); static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); static void oce_local_timer(void *arg); static void oce_if_deactivate(POCE_SOFTC sc); static void oce_if_activate(POCE_SOFTC sc); static void setup_max_queues_want(POCE_SOFTC sc); static void update_queues_got(POCE_SOFTC sc); static void process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe); static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); static void oce_get_config(POCE_SOFTC sc); static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); static void oce_read_env_variables(POCE_SOFTC sc); /* IP specific */ #if defined(INET6) || defined(INET) static int oce_init_lro(POCE_SOFTC sc); static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); #endif static device_method_t oce_dispatch[] = { DEVMETHOD(device_probe, oce_probe), DEVMETHOD(device_attach, oce_attach), DEVMETHOD(device_detach, oce_detach), DEVMETHOD(device_shutdown, oce_shutdown), DEVMETHOD_END }; static driver_t oce_driver = { "oce", oce_dispatch, sizeof(OCE_SOFTC) }; static devclass_t oce_devclass; /* global vars */ const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; /* Module capabilites and parameters */ uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; uint32_t oce_enable_rss = OCE_MODCAP_RSS; uint32_t oce_rq_buf_size = 2048; TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); /* Supported devices table */ static uint32_t supportedDevices[] = { (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH }; DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, nitems(supportedDevices)); MODULE_DEPEND(oce, pci, 1, 1, 1); MODULE_DEPEND(oce, ether, 1, 1, 1); MODULE_VERSION(oce, 1); POCE_SOFTC softc_head = NULL; POCE_SOFTC softc_tail = NULL; struct oce_rdma_if *oce_rdma_if = NULL; /***************************************************************************** * Driver entry points functions * *****************************************************************************/ static int oce_probe(device_t dev) { uint16_t vendor = 0; uint16_t device = 0; int i = 0; char str[256] = {0}; POCE_SOFTC sc; sc = device_get_softc(dev); bzero(sc, sizeof(OCE_SOFTC)); sc->dev = dev; vendor = pci_get_vendor(dev); device = pci_get_device(dev); for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { if (device == (supportedDevices[i] & 0xffff)) { sprintf(str, "%s:%s", "Emulex CNA NIC function", component_revision); device_set_desc_copy(dev, str); switch (device) { case PCI_PRODUCT_BE2: sc->flags |= OCE_FLAGS_BE2; break; case PCI_PRODUCT_BE3: sc->flags |= OCE_FLAGS_BE3; break; case PCI_PRODUCT_XE201: case PCI_PRODUCT_XE201_VF: sc->flags |= OCE_FLAGS_XE201; break; case PCI_PRODUCT_SH: sc->flags |= OCE_FLAGS_SH; break; default: return ENXIO; } return BUS_PROBE_DEFAULT; } } } return ENXIO; } static int oce_attach(device_t dev) { POCE_SOFTC sc; int rc = 0; sc = device_get_softc(dev); rc = oce_hw_pci_alloc(sc); if (rc) return rc; sc->tx_ring_size = OCE_TX_RING_SIZE; sc->rx_ring_size = OCE_RX_RING_SIZE; /* receive fragment size should be multiple of 2K */ sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; sc->promisc = OCE_DEFAULT_PROMISCUOUS; LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); LOCK_CREATE(&sc->dev_lock, "Device_lock"); /* initialise the hardware */ rc = oce_hw_init(sc); if (rc) goto pci_res_free; oce_read_env_variables(sc); oce_get_config(sc); setup_max_queues_want(sc); rc = oce_setup_intr(sc); if (rc) goto mbox_free; rc = oce_queue_init_all(sc); if (rc) goto intr_free; rc = oce_attach_ifp(sc); if (rc) goto queues_free; #if defined(INET6) || defined(INET) rc = oce_init_lro(sc); if (rc) goto ifp_free; #endif rc = oce_hw_start(sc); if (rc) goto lro_free; sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); rc = oce_stats_init(sc); if (rc) goto vlan_free; oce_add_sysctls(sc); callout_init(&sc->timer, CALLOUT_MPSAFE); rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); if (rc) goto stats_free; sc->next =NULL; if (softc_tail != NULL) { softc_tail->next = sc; } else { softc_head = sc; } softc_tail = sc; return 0; stats_free: callout_drain(&sc->timer); oce_stats_free(sc); vlan_free: if (sc->vlan_attach) EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); if (sc->vlan_detach) EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); oce_hw_intr_disable(sc); lro_free: #if defined(INET6) || defined(INET) oce_free_lro(sc); ifp_free: #endif ether_ifdetach(sc->ifp); if_free(sc->ifp); queues_free: oce_queue_release_all(sc); intr_free: oce_intr_free(sc); mbox_free: oce_dma_free(sc, &sc->bsmbx); pci_res_free: oce_hw_pci_free(sc); LOCK_DESTROY(&sc->dev_lock); LOCK_DESTROY(&sc->bmbx_lock); return rc; } static int oce_detach(device_t dev) { POCE_SOFTC sc = device_get_softc(dev); POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; poce_sc_tmp = softc_head; ppoce_sc_tmp1 = &softc_head; while (poce_sc_tmp != NULL) { if (poce_sc_tmp == sc) { *ppoce_sc_tmp1 = sc->next; if (sc->next == NULL) { softc_tail = poce_sc_tmp2; } break; } poce_sc_tmp2 = poce_sc_tmp; ppoce_sc_tmp1 = &poce_sc_tmp->next; poce_sc_tmp = poce_sc_tmp->next; } LOCK(&sc->dev_lock); oce_if_deactivate(sc); UNLOCK(&sc->dev_lock); callout_drain(&sc->timer); if (sc->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); if (sc->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); ether_ifdetach(sc->ifp); if_free(sc->ifp); oce_hw_shutdown(sc); bus_generic_detach(dev); return 0; } static int oce_shutdown(device_t dev) { int rc; rc = oce_detach(dev); return rc; } static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct ifreq *ifr = (struct ifreq *)data; POCE_SOFTC sc = ifp->if_softc; struct ifi2creq i2c; uint8_t offset = 0; int rc = 0; uint32_t u; switch (command) { case SIOCGIFMEDIA: rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); break; case SIOCSIFMTU: if (ifr->ifr_mtu > OCE_MAX_MTU) rc = EINVAL; else ifp->if_mtu = ifr->ifr_mtu; break; case SIOCSIFFLAGS: if (ifp->if_flags & IFF_UP) { if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; oce_init(sc); } device_printf(sc->dev, "Interface Up\n"); } else { LOCK(&sc->dev_lock); sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); oce_if_deactivate(sc); UNLOCK(&sc->dev_lock); device_printf(sc->dev, "Interface Down\n"); } if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) sc->promisc = TRUE; } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { if (!oce_rxf_set_promiscuous(sc, 0)) sc->promisc = FALSE; } break; case SIOCADDMULTI: case SIOCDELMULTI: rc = oce_hw_update_multicast(sc); if (rc) device_printf(sc->dev, "Update multicast address failed\n"); break; case SIOCSIFCAP: u = ifr->ifr_reqcap ^ ifp->if_capenable; if (u & IFCAP_TXCSUM) { ifp->if_capenable ^= IFCAP_TXCSUM; ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); if (IFCAP_TSO & ifp->if_capenable && !(IFCAP_TXCSUM & ifp->if_capenable)) { u &= ~IFCAP_TSO; ifp->if_capenable &= ~IFCAP_TSO; ifp->if_hwassist &= ~CSUM_TSO; if_printf(ifp, "TSO disabled due to -txcsum.\n"); } } if (u & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; if (u & IFCAP_TSO4) { ifp->if_capenable ^= IFCAP_TSO4; if (IFCAP_TSO & ifp->if_capenable) { if (IFCAP_TXCSUM & ifp->if_capenable) ifp->if_hwassist |= CSUM_TSO; else { ifp->if_capenable &= ~IFCAP_TSO; ifp->if_hwassist &= ~CSUM_TSO; if_printf(ifp, "Enable txcsum first.\n"); rc = EAGAIN; } } else ifp->if_hwassist &= ~CSUM_TSO; } if (u & IFCAP_VLAN_HWTAGGING) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (u & IFCAP_VLAN_HWFILTER) { ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; oce_vid_config(sc); } #if defined(INET6) || defined(INET) if (u & IFCAP_LRO) { ifp->if_capenable ^= IFCAP_LRO; if(sc->enable_hwlro) { if(ifp->if_capenable & IFCAP_LRO) { rc = oce_mbox_nic_set_iface_lro_config(sc, 1); }else { rc = oce_mbox_nic_set_iface_lro_config(sc, 0); } } } #endif break; case SIOCGI2C: rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); if (rc) break; if (i2c.dev_addr == PAGE_NUM_A0) { offset = i2c.offset; } else if (i2c.dev_addr == PAGE_NUM_A2) { offset = TRANSCEIVER_A0_SIZE + i2c.offset; } else { rc = EINVAL; break; } if (i2c.len > sizeof(i2c.data) || i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) { rc = EINVAL; break; } rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); if (rc) { rc = -rc; break; } memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); break; case SIOCGPRIVATE_0: rc = priv_check(curthread, PRIV_DRIVER); if (rc != 0) break; rc = oce_handle_passthrough(ifp, data); break; default: rc = ether_ioctl(ifp, command, data); break; } return rc; } static void oce_init(void *arg) { POCE_SOFTC sc = arg; LOCK(&sc->dev_lock); if (sc->ifp->if_flags & IFF_UP) { oce_if_deactivate(sc); oce_if_activate(sc); } UNLOCK(&sc->dev_lock); } static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m) { POCE_SOFTC sc = ifp->if_softc; struct oce_wq *wq = NULL; int queue_index = 0; int status = 0; if (!sc->link_status) return ENXIO; if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) queue_index = m->m_pkthdr.flowid % sc->nwqs; wq = sc->wq[queue_index]; LOCK(&wq->tx_lock); status = oce_multiq_transmit(ifp, m, wq); UNLOCK(&wq->tx_lock); return status; } static void oce_multiq_flush(struct ifnet *ifp) { POCE_SOFTC sc = ifp->if_softc; struct mbuf *m; int i = 0; for (i = 0; i < sc->nwqs; i++) { while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) m_freem(m); } if_qflush(ifp); } /***************************************************************************** * Driver interrupt routines functions * *****************************************************************************/ static void oce_intr(void *arg, int pending) { POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; POCE_SOFTC sc = ii->sc; struct oce_eq *eq = ii->eq; struct oce_eqe *eqe; struct oce_cq *cq = NULL; int i, num_eqes = 0; bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, BUS_DMASYNC_POSTWRITE); do { eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); if (eqe->evnt == 0) break; eqe->evnt = 0; bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, BUS_DMASYNC_POSTWRITE); RING_GET(eq->ring, 1); num_eqes++; } while (TRUE); if (!num_eqes) goto eq_arm; /* Spurious */ /* Clear EQ entries, but dont arm */ oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); /* Process TX, RX and MCC. But dont arm CQ*/ for (i = 0; i < eq->cq_valid; i++) { cq = eq->cq[i]; (*cq->cq_handler)(cq->cb_arg); } /* Arm all cqs connected to this EQ */ for (i = 0; i < eq->cq_valid; i++) { cq = eq->cq[i]; oce_arm_cq(sc, cq->cq_id, 0, TRUE); } eq_arm: oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); return; } static int oce_setup_intr(POCE_SOFTC sc) { int rc = 0, use_intx = 0; int vector = 0, req_vectors = 0; int tot_req_vectors, tot_vectors; if (is_rss_enabled(sc)) req_vectors = MAX((sc->nrqs - 1), sc->nwqs); else req_vectors = 1; tot_req_vectors = req_vectors; if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { if (req_vectors > 1) { tot_req_vectors += OCE_RDMA_VECTORS; sc->roce_intr_count = OCE_RDMA_VECTORS; } } if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { sc->intr_count = req_vectors; tot_vectors = tot_req_vectors; rc = pci_alloc_msix(sc->dev, &tot_vectors); if (rc != 0) { use_intx = 1; pci_release_msi(sc->dev); } else { if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { if (tot_vectors < tot_req_vectors) { if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { sc->roce_intr_count = (tot_vectors / 2); } sc->intr_count = tot_vectors - sc->roce_intr_count; } } else { sc->intr_count = tot_vectors; } sc->flags |= OCE_FLAGS_USING_MSIX; } } else use_intx = 1; if (use_intx) sc->intr_count = 1; /* Scale number of queues based on intr we got */ update_queues_got(sc); if (use_intx) { device_printf(sc->dev, "Using legacy interrupt\n"); rc = oce_alloc_intr(sc, vector, oce_intr); if (rc) goto error; } else { for (; vector < sc->intr_count; vector++) { rc = oce_alloc_intr(sc, vector, oce_intr); if (rc) goto error; } } return 0; error: oce_intr_free(sc); return rc; } static int oce_fast_isr(void *arg) { POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; POCE_SOFTC sc = ii->sc; if (ii->eq == NULL) return FILTER_STRAY; oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); taskqueue_enqueue(ii->tq, &ii->task); ii->eq->intr++; return FILTER_HANDLED; } static int oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) { POCE_INTR_INFO ii; int rc = 0, rr; if (vector >= OCE_MAX_EQ) return (EINVAL); ii = &sc->intrs[vector]; /* Set the resource id for the interrupt. * MSIx is vector + 1 for the resource id, * INTx is 0 for the resource id. */ if (sc->flags & OCE_FLAGS_USING_MSIX) rr = vector + 1; else rr = 0; ii->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rr, RF_ACTIVE|RF_SHAREABLE); ii->irq_rr = rr; if (ii->intr_res == NULL) { device_printf(sc->dev, "Could not allocate interrupt\n"); rc = ENXIO; return rc; } TASK_INIT(&ii->task, 0, isr, ii); ii->vector = vector; sprintf(ii->task_name, "oce_task[%d]", ii->vector); ii->tq = taskqueue_create_fast(ii->task_name, M_NOWAIT, taskqueue_thread_enqueue, &ii->tq); taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", device_get_nameunit(sc->dev)); ii->sc = sc; rc = bus_setup_intr(sc->dev, ii->intr_res, INTR_TYPE_NET, oce_fast_isr, NULL, ii, &ii->tag); return rc; } void oce_intr_free(POCE_SOFTC sc) { int i = 0; for (i = 0; i < sc->intr_count; i++) { if (sc->intrs[i].tag != NULL) bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, sc->intrs[i].tag); if (sc->intrs[i].tq != NULL) taskqueue_free(sc->intrs[i].tq); if (sc->intrs[i].intr_res != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, sc->intrs[i].irq_rr, sc->intrs[i].intr_res); sc->intrs[i].tag = NULL; sc->intrs[i].intr_res = NULL; } if (sc->flags & OCE_FLAGS_USING_MSIX) pci_release_msi(sc->dev); } /****************************************************************************** * Media callbacks functions * ******************************************************************************/ static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req) { POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; req->ifm_status = IFM_AVALID; req->ifm_active = IFM_ETHER; if (sc->link_status == 1) req->ifm_status |= IFM_ACTIVE; else return; switch (sc->link_speed) { case 1: /* 10 Mbps */ req->ifm_active |= IFM_10_T | IFM_FDX; sc->speed = 10; break; case 2: /* 100 Mbps */ req->ifm_active |= IFM_100_TX | IFM_FDX; sc->speed = 100; break; case 3: /* 1 Gbps */ req->ifm_active |= IFM_1000_T | IFM_FDX; sc->speed = 1000; break; case 4: /* 10 Gbps */ req->ifm_active |= IFM_10G_SR | IFM_FDX; sc->speed = 10000; break; case 5: /* 20 Gbps */ req->ifm_active |= IFM_10G_SR | IFM_FDX; sc->speed = 20000; break; case 6: /* 25 Gbps */ req->ifm_active |= IFM_10G_SR | IFM_FDX; sc->speed = 25000; break; case 7: /* 40 Gbps */ req->ifm_active |= IFM_40G_SR4 | IFM_FDX; sc->speed = 40000; break; default: sc->speed = 0; break; } return; } int oce_media_change(struct ifnet *ifp) { return 0; } static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, struct mbuf *m, boolean_t *os2bmc, struct mbuf **m_new) { struct ether_header *eh = NULL; eh = mtod(m, struct ether_header *); if (!is_os2bmc_enabled(sc) || *os2bmc) { *os2bmc = FALSE; goto done; } if (!ETHER_IS_MULTICAST(eh->ether_dhost)) goto done; if (is_mc_allowed_on_bmc(sc, eh) || is_bc_allowed_on_bmc(sc, eh) || is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { *os2bmc = TRUE; goto done; } if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); uint8_t nexthdr = ip6->ip6_nxt; if (nexthdr == IPPROTO_ICMPV6) { struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); switch (icmp6->icmp6_type) { case ND_ROUTER_ADVERT: *os2bmc = is_ipv6_ra_filt_enabled(sc); goto done; case ND_NEIGHBOR_ADVERT: *os2bmc = is_ipv6_na_filt_enabled(sc); goto done; default: break; } } } if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { struct ip *ip = mtod(m, struct ip *); int iphlen = ip->ip_hl << 2; struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); switch (uh->uh_dport) { case DHCP_CLIENT_PORT: *os2bmc = is_dhcp_client_filt_enabled(sc); goto done; case DHCP_SERVER_PORT: *os2bmc = is_dhcp_srvr_filt_enabled(sc); goto done; case NET_BIOS_PORT1: case NET_BIOS_PORT2: *os2bmc = is_nbios_filt_enabled(sc); goto done; case DHCPV6_RAS_PORT: *os2bmc = is_ipv6_ras_filt_enabled(sc); goto done; default: break; } } done: if (*os2bmc) { *m_new = m_dup(m, M_NOWAIT); if (!*m_new) { *os2bmc = FALSE; return; } *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); } } /***************************************************************************** * Transmit routines functions * *****************************************************************************/ static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) { int rc = 0, i, retry_cnt = 0; bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; struct mbuf *m, *m_temp, *m_new = NULL; struct oce_wq *wq = sc->wq[wq_index]; struct oce_packet_desc *pd; struct oce_nic_hdr_wqe *nichdr; struct oce_nic_frag_wqe *nicfrag; struct ether_header *eh = NULL; int num_wqes; uint32_t reg_value; boolean_t complete = TRUE; boolean_t os2bmc = FALSE; m = *mpp; if (!m) return EINVAL; if (!(m->m_flags & M_PKTHDR)) { rc = ENXIO; goto free_ret; } /* Don't allow non-TSO packets longer than MTU */ if (!is_tso_pkt(m)) { eh = mtod(m, struct ether_header *); if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) goto free_ret; } if(oce_tx_asic_stall_verify(sc, m)) { m = oce_insert_vlan_tag(sc, m, &complete); if(!m) { device_printf(sc->dev, "Insertion unsuccessful\n"); return 0; } } /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less * may cause a transmit stall on that port. So the work-around is to * pad short packets (<= 32 bytes) to a 36-byte length. */ if(IS_SH(sc) || IS_XE201(sc) ) { if(m->m_pkthdr.len <= 32) { char buf[36]; bzero((void *)buf, 36); m_append(m, (36 - m->m_pkthdr.len), buf); } } tx_start: if (m->m_pkthdr.csum_flags & CSUM_TSO) { /* consolidate packet buffers for TSO/LSO segment offload */ #if defined(INET6) || defined(INET) m = oce_tso_setup(sc, mpp); #else m = NULL; #endif if (m == NULL) { rc = ENXIO; goto free_ret; } } pd = &wq->pckts[wq->pkt_desc_head]; retry: rc = bus_dmamap_load_mbuf_sg(wq->tag, pd->map, m, segs, &pd->nsegs, BUS_DMA_NOWAIT); if (rc == 0) { num_wqes = pd->nsegs + 1; if (IS_BE(sc) || IS_SH(sc)) { /*Dummy required only for BE3.*/ if (num_wqes & 1) num_wqes++; } if (num_wqes >= RING_NUM_FREE(wq->ring)) { bus_dmamap_unload(wq->tag, pd->map); return EBUSY; } atomic_store_rel_int(&wq->pkt_desc_head, (wq->pkt_desc_head + 1) % \ OCE_WQ_PACKET_ARRAY_SIZE); bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); pd->mbuf = m; nichdr = RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); nichdr->u0.dw[0] = 0; nichdr->u0.dw[1] = 0; nichdr->u0.dw[2] = 0; nichdr->u0.dw[3] = 0; nichdr->u0.s.complete = complete; nichdr->u0.s.mgmt = os2bmc; nichdr->u0.s.event = 1; nichdr->u0.s.crc = 1; nichdr->u0.s.forward = 0; nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; nichdr->u0.s.udpcs = (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; nichdr->u0.s.tcpcs = (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; nichdr->u0.s.num_wqe = num_wqes; nichdr->u0.s.total_length = m->m_pkthdr.len; if (m->m_flags & M_VLANTAG) { nichdr->u0.s.vlan = 1; /*Vlan present*/ nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; } if (m->m_pkthdr.csum_flags & CSUM_TSO) { if (m->m_pkthdr.tso_segsz) { nichdr->u0.s.lso = 1; nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; } if (!IS_BE(sc) || !IS_SH(sc)) nichdr->u0.s.ipcs = 1; } RING_PUT(wq->ring, 1); atomic_add_int(&wq->ring->num_used, 1); for (i = 0; i < pd->nsegs; i++) { nicfrag = RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_frag_wqe); nicfrag->u0.s.rsvd0 = 0; nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); nicfrag->u0.s.frag_len = segs[i].ds_len; pd->wqe_idx = wq->ring->pidx; RING_PUT(wq->ring, 1); atomic_add_int(&wq->ring->num_used, 1); } if (num_wqes > (pd->nsegs + 1)) { nicfrag = RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_frag_wqe); nicfrag->u0.dw[0] = 0; nicfrag->u0.dw[1] = 0; nicfrag->u0.dw[2] = 0; nicfrag->u0.dw[3] = 0; pd->wqe_idx = wq->ring->pidx; RING_PUT(wq->ring, 1); atomic_add_int(&wq->ring->num_used, 1); pd->nsegs++; } if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); wq->tx_stats.tx_reqs++; wq->tx_stats.tx_wrbs += num_wqes; wq->tx_stats.tx_bytes += m->m_pkthdr.len; wq->tx_stats.tx_pkts++; bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); reg_value = (num_wqes << 16) | wq->wq_id; /* if os2bmc is not enabled or if the pkt is already tagged as bmc, do nothing */ oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len); if (m->m_flags & M_MCAST) if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); ETHER_BPF_MTAP(sc->ifp, m); OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); } else if (rc == EFBIG) { if (retry_cnt == 0) { m_temp = m_defrag(m, M_NOWAIT); if (m_temp == NULL) goto free_ret; m = m_temp; *mpp = m_temp; retry_cnt = retry_cnt + 1; goto retry; } else goto free_ret; } else if (rc == ENOMEM) return rc; else goto free_ret; if (os2bmc) { m = m_new; goto tx_start; } return 0; free_ret: m_freem(*mpp); *mpp = NULL; return rc; } static void oce_process_tx_completion(struct oce_wq *wq) { struct oce_packet_desc *pd; POCE_SOFTC sc = (POCE_SOFTC) wq->parent; struct mbuf *m; pd = &wq->pckts[wq->pkt_desc_tail]; atomic_store_rel_int(&wq->pkt_desc_tail, (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(wq->tag, pd->map); m = pd->mbuf; m_freem(m); pd->mbuf = NULL; if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { if (wq->ring->num_used < (wq->ring->num_items / 2)) { sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); oce_tx_restart(sc, wq); } } } static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) { if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) return; if (!drbr_empty(sc->ifp, wq->br)) taskqueue_enqueue(taskqueue_swi, &wq->txtask); } #if defined(INET6) || defined(INET) static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) { struct mbuf *m; #ifdef INET struct ip *ip; #endif #ifdef INET6 struct ip6_hdr *ip6; #endif struct ether_vlan_header *eh; struct tcphdr *th; uint16_t etype; int total_len = 0, ehdrlen = 0; m = *mpp; if (M_WRITABLE(m) == 0) { m = m_dup(*mpp, M_NOWAIT); if (!m) return NULL; m_freem(*mpp); *mpp = m; } eh = mtod(m, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } switch (etype) { #ifdef INET case ETHERTYPE_IP: ip = (struct ip *)(m->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) return NULL; th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); break; #endif #ifdef INET6 case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); if (ip6->ip6_nxt != IPPROTO_TCP) return NULL; th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); break; #endif default: return NULL; } m = m_pullup(m, total_len); *mpp = m; return m; } #endif /* INET6 || INET */ void oce_tx_task(void *arg, int npending) { struct oce_wq *wq = arg; POCE_SOFTC sc = wq->parent; struct ifnet *ifp = sc->ifp; int rc = 0; LOCK(&wq->tx_lock); rc = oce_multiq_transmit(ifp, NULL, wq); if (rc) { device_printf(sc->dev, "TX[%d] restart failed\n", wq->queue_index); } UNLOCK(&wq->tx_lock); } void oce_start(struct ifnet *ifp) { POCE_SOFTC sc = ifp->if_softc; struct mbuf *m; int rc = 0; int def_q = 0; /* Defualt tx queue is 0*/ if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!sc->link_status) return; while (true) { IF_DEQUEUE(&sc->ifp->if_snd, m); if (m == NULL) break; LOCK(&sc->wq[def_q]->tx_lock); rc = oce_tx(sc, &m, def_q); UNLOCK(&sc->wq[def_q]->tx_lock); if (rc) { if (m != NULL) { sc->wq[def_q]->tx_stats.tx_stops ++; ifp->if_drv_flags |= IFF_DRV_OACTIVE; IFQ_DRV_PREPEND(&ifp->if_snd, m); m = NULL; } break; } } } /* Handle the Completion Queue for transmit */ uint16_t oce_wq_handler(void *arg) { struct oce_wq *wq = (struct oce_wq *)arg; POCE_SOFTC sc = wq->parent; struct oce_cq *cq = wq->cq; struct oce_nic_tx_cqe *cqe; int num_cqes = 0; LOCK(&wq->tx_compl_lock); bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); while (cqe->u0.dw[3]) { DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); wq->ring->cidx = cqe->u0.s.wqe_index + 1; if (wq->ring->cidx >= wq->ring->num_items) wq->ring->cidx -= wq->ring->num_items; oce_process_tx_completion(wq); wq->tx_stats.tx_compl++; cqe->u0.dw[3] = 0; RING_GET(cq->ring, 1); bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); num_cqes++; } if (num_cqes) oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); UNLOCK(&wq->tx_compl_lock); return num_cqes; } static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) { POCE_SOFTC sc = ifp->if_softc; int status = 0, queue_index = 0; struct mbuf *next = NULL; struct buf_ring *br = NULL; br = wq->br; queue_index = wq->queue_index; if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) { if (m != NULL) status = drbr_enqueue(ifp, br, m); return status; } if (m != NULL) { if ((status = drbr_enqueue(ifp, br, m)) != 0) return status; } while ((next = drbr_peek(ifp, br)) != NULL) { if (oce_tx(sc, &next, queue_index)) { if (next == NULL) { drbr_advance(ifp, br); } else { drbr_putback(ifp, br, next); wq->tx_stats.tx_stops ++; ifp->if_drv_flags |= IFF_DRV_OACTIVE; } break; } drbr_advance(ifp, br); } return 0; } /***************************************************************************** * Receive routines functions * *****************************************************************************/ static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) { uint32_t *p; struct ether_header *eh = NULL; struct tcphdr *tcp_hdr = NULL; struct ip *ip4_hdr = NULL; struct ip6_hdr *ip6 = NULL; uint32_t payload_len = 0; eh = mtod(m, struct ether_header *); /* correct IP header */ if(!cqe2->ipv6_frame) { ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); ip4_hdr->ip_ttl = cqe2->frame_lifespan; ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); }else { ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; payload_len = cqe2->coalesced_size - sizeof(struct ether_header) - sizeof(struct ip6_hdr); ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); } /* correct tcp header */ tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); if(cqe2->push) { tcp_hdr->th_flags |= TH_PUSH; } tcp_hdr->th_win = htons(cqe2->tcp_window); tcp_hdr->th_sum = 0xffff; if(cqe2->ts_opt) { p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); *p = cqe1->tcp_timestamp_val; *(p+1) = cqe1->tcp_timestamp_ecr; } return; } static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; uint32_t i = 0, frag_len = 0; uint32_t len = cqe_info->pkt_size; struct oce_packet_desc *pd; struct mbuf *tail = NULL; for (i = 0; i < cqe_info->num_frags; i++) { if (rq->ring->cidx == rq->ring->pidx) { device_printf(sc->dev, "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); return; } pd = &rq->pckts[rq->ring->cidx]; bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(rq->tag, pd->map); RING_GET(rq->ring, 1); rq->pending--; frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; pd->mbuf->m_len = frag_len; if (tail != NULL) { /* additional fragments */ pd->mbuf->m_flags &= ~M_PKTHDR; tail->m_next = pd->mbuf; if(rq->islro) tail->m_nextpkt = NULL; tail = pd->mbuf; } else { /* first fragment, fill out much of the packet header */ pd->mbuf->m_pkthdr.len = len; if(rq->islro) pd->mbuf->m_nextpkt = NULL; pd->mbuf->m_pkthdr.csum_flags = 0; if (IF_CSUM_ENABLED(sc)) { if (cqe_info->l4_cksum_pass) { if(!cqe_info->ipv6_frame) { /* IPV4 */ pd->mbuf->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); }else { /* IPV6 frame */ if(rq->islro) { pd->mbuf->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); } } pd->mbuf->m_pkthdr.csum_data = 0xffff; } if (cqe_info->ip_cksum_pass) { pd->mbuf->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED|CSUM_IP_VALID); } } *m = tail = pd->mbuf; } pd->mbuf = NULL; len -= frag_len; } return; } static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; struct nic_hwlro_cqe_part1 *cqe1 = NULL; struct mbuf *m = NULL; struct oce_common_cqe_info cq_info; /* parse cqe */ if(cqe2 == NULL) { cq_info.pkt_size = cqe->pkt_size; cq_info.vtag = cqe->vlan_tag; cq_info.l4_cksum_pass = cqe->l4_cksum_pass; cq_info.ip_cksum_pass = cqe->ip_cksum_pass; cq_info.ipv6_frame = cqe->ipv6_frame; cq_info.vtp = cqe->vtp; cq_info.qnq = cqe->qnq; }else { cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; cq_info.pkt_size = cqe2->coalesced_size; cq_info.vtag = cqe2->vlan_tag; cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; cq_info.ipv6_frame = cqe2->ipv6_frame; cq_info.vtp = cqe2->vtp; cq_info.qnq = cqe1->qnq; } cq_info.vtag = BSWAP_16(cq_info.vtag); cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; if(cq_info.pkt_size % rq->cfg.frag_size) cq_info.num_frags++; oce_rx_mbuf_chain(rq, &cq_info, &m); if (m) { if(cqe2) { //assert(cqe2->valid != 0); //assert(cqe2->cqe_type != 2); oce_correct_header(m, cqe1, cqe2); } m->m_pkthdr.rcvif = sc->ifp; if (rq->queue_index) m->m_pkthdr.flowid = (rq->queue_index - 1); else m->m_pkthdr.flowid = rq->queue_index; M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); /* This deternies if vlan tag is Valid */ if (cq_info.vtp) { if (sc->function_mode & FNM_FLEX10_MODE) { /* FLEX10. If QnQ is not set, neglect VLAN */ if (cq_info.qnq) { m->m_pkthdr.ether_vtag = cq_info.vtag; m->m_flags |= M_VLANTAG; } } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { /* In UMC mode generally pvid will be striped by hw. But in some cases we have seen it comes with pvid. So if pvid == vlan, neglect vlan. */ m->m_pkthdr.ether_vtag = cq_info.vtag; m->m_flags |= M_VLANTAG; } } if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); (*sc->ifp->if_input) (sc->ifp, m); /* Update rx stats per queue */ rq->rx_stats.rx_pkts++; rq->rx_stats.rx_bytes += cq_info.pkt_size; rq->rx_stats.rx_frags += cq_info.num_frags; rq->rx_stats.rx_ucast_pkts++; } return; } static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; int len; struct mbuf *m = NULL; struct oce_common_cqe_info cq_info; uint16_t vtag = 0; /* Is it a flush compl that has no data */ if(!cqe->u0.s.num_fragments) goto exit; len = cqe->u0.s.pkt_size; if (!len) { /*partial DMA workaround for Lancer*/ oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); goto exit; } if (!oce_cqe_portid_valid(sc, cqe)) { oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); goto exit; } /* Get vlan_tag value */ if(IS_BE(sc) || IS_SH(sc)) vtag = BSWAP_16(cqe->u0.s.vlan_tag); else vtag = cqe->u0.s.vlan_tag; cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; cq_info.ipv6_frame = cqe->u0.s.ip_ver; cq_info.num_frags = cqe->u0.s.num_fragments; cq_info.pkt_size = cqe->u0.s.pkt_size; oce_rx_mbuf_chain(rq, &cq_info, &m); if (m) { m->m_pkthdr.rcvif = sc->ifp; if (rq->queue_index) m->m_pkthdr.flowid = (rq->queue_index - 1); else m->m_pkthdr.flowid = rq->queue_index; M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); /* This deternies if vlan tag is Valid */ if (oce_cqe_vtp_valid(sc, cqe)) { if (sc->function_mode & FNM_FLEX10_MODE) { /* FLEX10. If QnQ is not set, neglect VLAN */ if (cqe->u0.s.qnq) { m->m_pkthdr.ether_vtag = vtag; m->m_flags |= M_VLANTAG; } } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { /* In UMC mode generally pvid will be striped by hw. But in some cases we have seen it comes with pvid. So if pvid == vlan, neglect vlan. */ m->m_pkthdr.ether_vtag = vtag; m->m_flags |= M_VLANTAG; } } if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); #if defined(INET6) || defined(INET) /* Try to queue to LRO */ if (IF_LRO_ENABLED(sc) && (cqe->u0.s.ip_cksum_pass) && (cqe->u0.s.l4_cksum_pass) && (!cqe->u0.s.ip_ver) && (rq->lro.lro_cnt != 0)) { if (tcp_lro_rx(&rq->lro, m, 0) == 0) { rq->lro_pkts_queued ++; goto post_done; } /* If LRO posting fails then try to post to STACK */ } #endif (*sc->ifp->if_input) (sc->ifp, m); #if defined(INET6) || defined(INET) post_done: #endif /* Update rx stats per queue */ rq->rx_stats.rx_pkts++; rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) rq->rx_stats.rx_mcast_pkts++; if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) rq->rx_stats.rx_ucast_pkts++; } exit: return; } void oce_discard_rx_comp(struct oce_rq *rq, int num_frags) { uint32_t i = 0; struct oce_packet_desc *pd; POCE_SOFTC sc = (POCE_SOFTC) rq->parent; for (i = 0; i < num_frags; i++) { if (rq->ring->cidx == rq->ring->pidx) { device_printf(sc->dev, "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); return; } pd = &rq->pckts[rq->ring->cidx]; bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(rq->tag, pd->map); if (pd->mbuf != NULL) { m_freem(pd->mbuf); pd->mbuf = NULL; } RING_GET(rq->ring, 1); rq->pending--; } } static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) { struct oce_nic_rx_cqe_v1 *cqe_v1; int vtp = 0; if (sc->be3_native) { cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; vtp = cqe_v1->u0.s.vlan_tag_present; } else vtp = cqe->u0.s.vlan_tag_present; return vtp; } static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) { struct oce_nic_rx_cqe_v1 *cqe_v1; int port_id = 0; if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; port_id = cqe_v1->u0.s.port; if (sc->port_id != port_id) return 0; } else ;/* For BE3 legacy and Lancer this is dummy */ return 1; } #if defined(INET6) || defined(INET) void oce_rx_flush_lro(struct oce_rq *rq) { struct lro_ctrl *lro = &rq->lro; POCE_SOFTC sc = (POCE_SOFTC) rq->parent; if (!IF_LRO_ENABLED(sc)) return; tcp_lro_flush_all(lro); rq->lro_pkts_queued = 0; return; } static int oce_init_lro(POCE_SOFTC sc) { struct lro_ctrl *lro = NULL; int i = 0, rc = 0; for (i = 0; i < sc->nrqs; i++) { lro = &sc->rq[i]->lro; rc = tcp_lro_init(lro); if (rc != 0) { device_printf(sc->dev, "LRO init failed\n"); return rc; } lro->ifp = sc->ifp; } return rc; } void oce_free_lro(POCE_SOFTC sc) { struct lro_ctrl *lro = NULL; int i = 0; for (i = 0; i < sc->nrqs; i++) { lro = &sc->rq[i]->lro; if (lro) tcp_lro_free(lro); } } #endif int oce_alloc_rx_bufs(struct oce_rq *rq, int count) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; int i, in, rc; struct oce_packet_desc *pd; bus_dma_segment_t segs[6]; int nsegs, added = 0; struct oce_nic_rqe *rqe; pd_rxulp_db_t rxdb_reg; uint32_t val = 0; uint32_t oce_max_rq_posts = 64; bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); for (i = 0; i < count; i++) { in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; pd = &rq->pckts[rq->ring->pidx]; pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); if (pd->mbuf == NULL) { device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); break; } pd->mbuf->m_nextpkt = NULL; pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; rc = bus_dmamap_load_mbuf_sg(rq->tag, pd->map, pd->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); if (rc) { m_free(pd->mbuf); device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); break; } if (nsegs != 1) { i--; continue; } bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); RING_PUT(rq->ring, 1); added++; rq->pending++; } oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; if (added != 0) { for (i = added / oce_max_rq_posts; i > 0; i--) { rxdb_reg.bits.num_posted = oce_max_rq_posts; rxdb_reg.bits.qid = rq->rq_id; if(rq->islro) { val |= rq->rq_id & DB_LRO_RQ_ID_MASK; val |= oce_max_rq_posts << 16; OCE_WRITE_REG32(sc, db, DB_OFFSET, val); }else { OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); } added -= oce_max_rq_posts; } if (added > 0) { rxdb_reg.bits.qid = rq->rq_id; rxdb_reg.bits.num_posted = added; if(rq->islro) { val |= rq->rq_id & DB_LRO_RQ_ID_MASK; val |= added << 16; OCE_WRITE_REG32(sc, db, DB_OFFSET, val); }else { OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); } } } return 0; } static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) { if (num_cqes) { oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); if(!sc->enable_hwlro) { if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); }else { if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) oce_alloc_rx_bufs(rq, 64); } } return; } uint16_t oce_rq_handler_lro(void *arg) { struct oce_rq *rq = (struct oce_rq *)arg; struct oce_cq *cq = rq->cq; POCE_SOFTC sc = rq->parent; struct nic_hwlro_singleton_cqe *cqe; struct nic_hwlro_cqe_part2 *cqe2; int num_cqes = 0; LOCK(&rq->rx_lock); bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); while (cqe->valid) { if(cqe->cqe_type == 0) { /* singleton cqe */ /* we should not get singleton cqe after cqe1 on same rq */ if(rq->cqe_firstpart != NULL) { device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); goto exit_rq_handler_lro; } if(cqe->error != 0) { rq->rx_stats.rxcp_err++; if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); } oce_rx_lro(rq, cqe, NULL); rq->rx_stats.rx_compl++; cqe->valid = 0; RING_GET(cq->ring, 1); num_cqes++; if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) break; }else if(cqe->cqe_type == 0x1) { /* first part */ /* we should not get cqe1 after cqe1 on same rq */ if(rq->cqe_firstpart != NULL) { device_printf(sc->dev, "Got cqe1 after cqe1 \n"); goto exit_rq_handler_lro; } rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; RING_GET(cq->ring, 1); }else if(cqe->cqe_type == 0x2) { /* second part */ cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; if(cqe2->error != 0) { rq->rx_stats.rxcp_err++; if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); } /* We should not get cqe2 without cqe1 */ if(rq->cqe_firstpart == NULL) { device_printf(sc->dev, "Got cqe2 without cqe1 \n"); goto exit_rq_handler_lro; } oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); rq->rx_stats.rx_compl++; rq->cqe_firstpart->valid = 0; cqe2->valid = 0; rq->cqe_firstpart = NULL; RING_GET(cq->ring, 1); num_cqes += 2; if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) break; } bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); } oce_check_rx_bufs(sc, num_cqes, rq); exit_rq_handler_lro: UNLOCK(&rq->rx_lock); return 0; } /* Handle the Completion Queue for receive */ uint16_t oce_rq_handler(void *arg) { + struct epoch_tracker et; struct oce_rq *rq = (struct oce_rq *)arg; struct oce_cq *cq = rq->cq; POCE_SOFTC sc = rq->parent; struct oce_nic_rx_cqe *cqe; int num_cqes = 0; + NET_EPOCH_ENTER(et); if(rq->islro) { oce_rq_handler_lro(arg); + NET_EPOCH_EXIT(et); return 0; } LOCK(&rq->rx_lock); bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); while (cqe->u0.dw[2]) { DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); if (cqe->u0.s.error == 0) { oce_rx(rq, cqe); } else { rq->rx_stats.rxcp_err++; if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); /* Post L3/L4 errors to stack.*/ oce_rx(rq, cqe); } rq->rx_stats.rx_compl++; cqe->u0.dw[2] = 0; #if defined(INET6) || defined(INET) if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { oce_rx_flush_lro(rq); } #endif RING_GET(cq->ring, 1); bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); num_cqes++; if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) break; } #if defined(INET6) || defined(INET) if (IF_LRO_ENABLED(sc)) oce_rx_flush_lro(rq); #endif oce_check_rx_bufs(sc, num_cqes, rq); UNLOCK(&rq->rx_lock); + NET_EPOCH_EXIT(et); return 0; } /***************************************************************************** * Helper function prototypes in this file * *****************************************************************************/ static int oce_attach_ifp(POCE_SOFTC sc) { sc->ifp = if_alloc(IFT_ETHER); if (!sc->ifp) return ENOMEM; ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); - sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; + sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_KNOWSEPOCH; sc->ifp->if_ioctl = oce_ioctl; sc->ifp->if_start = oce_start; sc->ifp->if_init = oce_init; sc->ifp->if_mtu = ETHERMTU; sc->ifp->if_softc = sc; sc->ifp->if_transmit = oce_multiq_start; sc->ifp->if_qflush = oce_multiq_flush; if_initname(sc->ifp, device_get_name(sc->dev), device_get_unit(sc->dev)); sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); IFQ_SET_READY(&sc->ifp->if_snd); sc->ifp->if_hwassist = OCE_IF_HWASSIST; sc->ifp->if_hwassist |= CSUM_TSO; sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; sc->ifp->if_capabilities |= IFCAP_HWCSUM; sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; #if defined(INET6) || defined(INET) sc->ifp->if_capabilities |= IFCAP_TSO; sc->ifp->if_capabilities |= IFCAP_LRO; sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; #endif sc->ifp->if_capenable = sc->ifp->if_capabilities; sc->ifp->if_baudrate = IF_Gbps(10); sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; sc->ifp->if_hw_tsomaxsegsize = 4096; ether_ifattach(sc->ifp, sc->macaddr.mac_addr); return 0; } static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) { POCE_SOFTC sc = ifp->if_softc; if (ifp->if_softc != arg) return; if ((vtag == 0) || (vtag > 4095)) return; sc->vlan_tag[vtag] = 1; sc->vlans_added++; if (sc->vlans_added <= (sc->max_vlans + 1)) oce_vid_config(sc); } static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) { POCE_SOFTC sc = ifp->if_softc; if (ifp->if_softc != arg) return; if ((vtag == 0) || (vtag > 4095)) return; sc->vlan_tag[vtag] = 0; sc->vlans_added--; oce_vid_config(sc); } /* * A max of 64 vlans can be configured in BE. If the user configures * more, place the card in vlan promiscuous mode. */ static int oce_vid_config(POCE_SOFTC sc) { struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; uint16_t ntags = 0, i; int status = 0; if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { for (i = 0; i < MAX_VLANS; i++) { if (sc->vlan_tag[i]) { vtags[ntags].vtag = i; ntags++; } } if (ntags) status = oce_config_vlan(sc, (uint8_t) sc->if_id, vtags, ntags, 1, 0); } else status = oce_config_vlan(sc, (uint8_t) sc->if_id, NULL, 0, 1, 1); return status; } static void oce_mac_addr_set(POCE_SOFTC sc) { uint32_t old_pmac_id = sc->pmac_id; int status = 0; status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, sc->macaddr.size_of_struct); if (!status) return; status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), sc->if_id, &sc->pmac_id); if (!status) { status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, sc->macaddr.size_of_struct); } if (status) device_printf(sc->dev, "Failed update macaddress\n"); } static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data) { POCE_SOFTC sc = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; int rc = ENXIO; char cookie[32] = {0}; void *priv_data = ifr_data_get_ptr(ifr); void *ioctl_ptr; uint32_t req_size; struct mbx_hdr req; OCE_DMA_MEM dma_mem; struct mbx_common_get_cntl_attr *fw_cmd; if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) return EFAULT; if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) return EINVAL; ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) return EFAULT; req_size = le32toh(req.u0.req.request_length); if (req_size > 65536) return EINVAL; req_size += sizeof(struct mbx_hdr); rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); if (rc) return ENOMEM; if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { rc = EFAULT; goto dma_free; } rc = oce_pass_through_mbox(sc, &dma_mem, req_size); if (rc) { rc = EIO; goto dma_free; } if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) rc = EFAULT; /* firmware is filling all the attributes for this ioctl except the driver version..so fill it */ if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, COMPONENT_REVISION, strlen(COMPONENT_REVISION)); } dma_free: oce_dma_free(sc, &dma_mem); return rc; } static void oce_eqd_set_periodic(POCE_SOFTC sc) { struct oce_set_eqd set_eqd[OCE_MAX_EQ]; struct oce_aic_obj *aic; struct oce_eq *eqo; uint64_t now = 0, delta; int eqd, i, num = 0; uint32_t tx_reqs = 0, rxpkts = 0, pps; struct oce_wq *wq; struct oce_rq *rq; #define ticks_to_msecs(t) (1000 * (t) / hz) for (i = 0 ; i < sc->neqs; i++) { eqo = sc->eq[i]; aic = &sc->aic_obj[i]; /* When setting the static eq delay from the user space */ if (!aic->enable) { if (aic->ticks) aic->ticks = 0; eqd = aic->et_eqd; goto modify_eqd; } if (i == 0) { rq = sc->rq[0]; rxpkts = rq->rx_stats.rx_pkts; } else rxpkts = 0; if (i + 1 < sc->nrqs) { rq = sc->rq[i + 1]; rxpkts += rq->rx_stats.rx_pkts; } if (i < sc->nwqs) { wq = sc->wq[i]; tx_reqs = wq->tx_stats.tx_reqs; } else tx_reqs = 0; now = ticks; if (!aic->ticks || now < aic->ticks || rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { aic->prev_rxpkts = rxpkts; aic->prev_txreqs = tx_reqs; aic->ticks = now; continue; } delta = ticks_to_msecs(now - aic->ticks); pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); eqd = (pps / 15000) << 2; if (eqd < 8) eqd = 0; /* Make sure that the eq delay is in the known range */ eqd = min(eqd, aic->max_eqd); eqd = max(eqd, aic->min_eqd); aic->prev_rxpkts = rxpkts; aic->prev_txreqs = tx_reqs; aic->ticks = now; modify_eqd: if (eqd != aic->cur_eqd) { set_eqd[num].delay_multiplier = (eqd * 65)/100; set_eqd[num].eq_id = eqo->eq_id; aic->cur_eqd = eqd; num++; } } /* Is there atleast one eq that needs to be modified? */ for(i = 0; i < num; i += 8) { if((num - i) >=8 ) oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); else oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); } } static void oce_detect_hw_error(POCE_SOFTC sc) { uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; uint32_t i; if (sc->hw_error) return; if (IS_XE201(sc)) { sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); if (sliport_status & SLIPORT_STATUS_ERR_MASK) { sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); } } else { ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); ue_low = (ue_low & ~ue_low_mask); ue_high = (ue_high & ~ue_high_mask); } /* On certain platforms BE hardware can indicate spurious UEs. * Allow the h/w to stop working completely in case of a real UE. * Hence not setting the hw_error for UE detection. */ if (sliport_status & SLIPORT_STATUS_ERR_MASK) { sc->hw_error = TRUE; device_printf(sc->dev, "Error detected in the card\n"); } if (sliport_status & SLIPORT_STATUS_ERR_MASK) { device_printf(sc->dev, "ERR: sliport status 0x%x\n", sliport_status); device_printf(sc->dev, "ERR: sliport error1 0x%x\n", sliport_err1); device_printf(sc->dev, "ERR: sliport error2 0x%x\n", sliport_err2); } if (ue_low) { for (i = 0; ue_low; ue_low >>= 1, i++) { if (ue_low & 1) device_printf(sc->dev, "UE: %s bit set\n", ue_status_low_desc[i]); } } if (ue_high) { for (i = 0; ue_high; ue_high >>= 1, i++) { if (ue_high & 1) device_printf(sc->dev, "UE: %s bit set\n", ue_status_hi_desc[i]); } } } static void oce_local_timer(void *arg) { POCE_SOFTC sc = arg; int i = 0; oce_detect_hw_error(sc); oce_refresh_nic_stats(sc); oce_refresh_queue_stats(sc); oce_mac_addr_set(sc); /* TX Watch Dog*/ for (i = 0; i < sc->nwqs; i++) oce_tx_restart(sc, sc->wq[i]); /* calculate and set the eq delay for optimal interrupt rate */ if (IS_BE(sc) || IS_SH(sc)) oce_eqd_set_periodic(sc); callout_reset(&sc->timer, hz, oce_local_timer, sc); } static void oce_tx_compl_clean(POCE_SOFTC sc) { struct oce_wq *wq; int i = 0, timeo = 0, num_wqes = 0; int pending_txqs = sc->nwqs; /* Stop polling for compls when HW has been silent for 10ms or * hw_error or no outstanding completions expected */ do { pending_txqs = sc->nwqs; for_all_wq_queues(sc, wq, i) { num_wqes = oce_wq_handler(wq); if(num_wqes) timeo = 0; if(!wq->ring->num_used) pending_txqs--; } if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) break; DELAY(1000); } while (TRUE); for_all_wq_queues(sc, wq, i) { while(wq->ring->num_used) { LOCK(&wq->tx_compl_lock); oce_process_tx_completion(wq); UNLOCK(&wq->tx_compl_lock); } } } /* NOTE : This should only be called holding * DEVICE_LOCK. */ static void oce_if_deactivate(POCE_SOFTC sc) { int i; struct oce_rq *rq; struct oce_wq *wq; struct oce_eq *eq; sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); oce_tx_compl_clean(sc); /* Stop intrs and finish any bottom halves pending */ oce_hw_intr_disable(sc); /* Since taskqueue_drain takes a Gaint Lock, We should not acquire any other lock. So unlock device lock and require after completing taskqueue_drain. */ UNLOCK(&sc->dev_lock); for (i = 0; i < sc->intr_count; i++) { if (sc->intrs[i].tq != NULL) { taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); } } LOCK(&sc->dev_lock); /* Delete RX queue in card with flush param */ oce_stop_rx(sc); /* Invalidate any pending cq and eq entries*/ for_all_evnt_queues(sc, eq, i) oce_drain_eq(eq); for_all_rq_queues(sc, rq, i) oce_drain_rq_cq(rq); for_all_wq_queues(sc, wq, i) oce_drain_wq_cq(wq); /* But still we need to get MCC aync events. So enable intrs and also arm first EQ */ oce_hw_intr_enable(sc); oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); DELAY(10); } static void oce_if_activate(POCE_SOFTC sc) { struct oce_eq *eq; struct oce_rq *rq; struct oce_wq *wq; int i, rc = 0; sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; oce_hw_intr_disable(sc); oce_start_rx(sc); for_all_rq_queues(sc, rq, i) { rc = oce_start_rq(rq); if (rc) device_printf(sc->dev, "Unable to start RX\n"); } for_all_wq_queues(sc, wq, i) { rc = oce_start_wq(wq); if (rc) device_printf(sc->dev, "Unable to start TX\n"); } for_all_evnt_queues(sc, eq, i) oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); oce_hw_intr_enable(sc); } static void process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) { /* Update Link status */ if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == ASYNC_EVENT_LINK_UP) { sc->link_status = ASYNC_EVENT_LINK_UP; if_link_state_change(sc->ifp, LINK_STATE_UP); } else { sc->link_status = ASYNC_EVENT_LINK_DOWN; if_link_state_change(sc->ifp, LINK_STATE_DOWN); } } static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, struct oce_async_evt_grp5_os2bmc *evt) { DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); if (evt->u.s.mgmt_enable) sc->flags |= OCE_FLAGS_OS2BMC; else return; sc->bmc_filt_mask = evt->u.s.arp_filter; sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); } static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) { struct oce_async_event_grp5_pvid_state *gcqe; struct oce_async_evt_grp5_os2bmc *bmccqe; switch (cqe->u0.s.async_type) { case ASYNC_EVENT_PVID_STATE: /* GRP5 PVID */ gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; if (gcqe->enabled) sc->pvid = gcqe->tag & VLAN_VID_MASK; else sc->pvid = 0; break; case ASYNC_EVENT_OS2BMC: bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; oce_async_grp5_osbmc_process(sc, bmccqe); break; default: break; } } /* Handle the Completion Queue for the Mailbox/Async notifications */ uint16_t oce_mq_handler(void *arg) { struct oce_mq *mq = (struct oce_mq *)arg; POCE_SOFTC sc = mq->parent; struct oce_cq *cq = mq->cq; int num_cqes = 0, evt_type = 0, optype = 0; struct oce_mq_cqe *cqe; struct oce_async_cqe_link_state *acqe; struct oce_async_event_qnq *dbgcqe; bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); while (cqe->u0.dw[3]) { DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); if (cqe->u0.s.async_event) { evt_type = cqe->u0.s.event_type; optype = cqe->u0.s.async_type; if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { /* Link status evt */ acqe = (struct oce_async_cqe_link_state *)cqe; process_link_state(sc, acqe); } else if (evt_type == ASYNC_EVENT_GRP5) { oce_process_grp5_events(sc, cqe); } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && optype == ASYNC_EVENT_DEBUG_QNQ) { dbgcqe = (struct oce_async_event_qnq *)cqe; if(dbgcqe->valid) sc->qnqid = dbgcqe->vlan_tag; sc->qnq_debug_event = TRUE; } } cqe->u0.dw[3] = 0; RING_GET(cq->ring, 1); bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); num_cqes++; } if (num_cqes) oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); return 0; } static void setup_max_queues_want(POCE_SOFTC sc) { /* Check if it is FLEX machine. Is so dont use RSS */ if ((sc->function_mode & FNM_FLEX10_MODE) || (sc->function_mode & FNM_UMC_MODE) || (sc->function_mode & FNM_VNIC_MODE) || (!is_rss_enabled(sc)) || IS_BE2(sc)) { sc->nrqs = 1; sc->nwqs = 1; } else { sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); } if (IS_BE2(sc) && is_rss_enabled(sc)) sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; } static void update_queues_got(POCE_SOFTC sc) { if (is_rss_enabled(sc)) { sc->nrqs = sc->intr_count + 1; sc->nwqs = sc->intr_count; } else { sc->nrqs = 1; sc->nwqs = 1; } if (IS_BE2(sc)) sc->nwqs = 1; } static int oce_check_ipv6_ext_hdr(struct mbuf *m) { struct ether_header *eh = mtod(m, struct ether_header *); caddr_t m_datatemp = m->m_data; if (eh->ether_type == htons(ETHERTYPE_IPV6)) { m->m_data += sizeof(struct ether_header); struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); if((ip6->ip6_nxt != IPPROTO_TCP) && \ (ip6->ip6_nxt != IPPROTO_UDP)){ struct ip6_ext *ip6e = NULL; m->m_data += sizeof(struct ip6_hdr); ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); if(ip6e->ip6e_len == 0xff) { m->m_data = m_datatemp; return TRUE; } } m->m_data = m_datatemp; } return FALSE; } static int is_be3_a1(POCE_SOFTC sc) { if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { return TRUE; } return FALSE; } static struct mbuf * oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) { uint16_t vlan_tag = 0; if(!M_WRITABLE(m)) return NULL; /* Embed vlan tag in the packet if it is not part of it */ if(m->m_flags & M_VLANTAG) { vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); m->m_flags &= ~M_VLANTAG; } /* if UMC, ignore vlan tag insertion and instead insert pvid */ if(sc->pvid) { if(!vlan_tag) vlan_tag = sc->pvid; if (complete) *complete = FALSE; } if(vlan_tag) { m = ether_vlanencap(m, vlan_tag); } if(sc->qnqid) { m = ether_vlanencap(m, sc->qnqid); if (complete) *complete = FALSE; } return m; } static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) { if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ oce_check_ipv6_ext_hdr(m)) { return TRUE; } return FALSE; } static void oce_get_config(POCE_SOFTC sc) { int rc = 0; uint32_t max_rss = 0; if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) max_rss = OCE_LEGACY_MODE_RSS; else max_rss = OCE_MAX_RSS; if (!IS_BE(sc)) { rc = oce_get_profile_config(sc, max_rss); if (rc) { sc->nwqs = OCE_MAX_WQ; sc->nrssqs = max_rss; sc->nrqs = sc->nrssqs + 1; } } else { /* For BE3 don't rely on fw for determining the resources */ sc->nrssqs = max_rss; sc->nrqs = sc->nrssqs + 1; sc->nwqs = OCE_MAX_WQ; sc->max_vlans = MAX_VLANFILTER_SIZE; } } static void oce_rdma_close(void) { if (oce_rdma_if != NULL) { oce_rdma_if = NULL; } } static void oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) { memcpy(macaddr, sc->macaddr.mac_addr, 6); } int oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) { POCE_SOFTC sc; struct oce_dev_info di; int i; if ((rdma_info == NULL) || (rdma_if == NULL)) { return -EINVAL; } if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || (rdma_if->size != OCE_RDMA_IF_SIZE)) { return -ENXIO; } rdma_info->close = oce_rdma_close; rdma_info->mbox_post = oce_mbox_post; rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; rdma_info->get_mac_addr = oce_get_mac_addr; oce_rdma_if = rdma_if; sc = softc_head; while (sc != NULL) { if (oce_rdma_if->announce != NULL) { memset(&di, 0, sizeof(di)); di.dev = sc->dev; di.softc = sc; di.ifp = sc->ifp; di.db_bhandle = sc->db_bhandle; di.db_btag = sc->db_btag; di.db_page_size = 4096; if (sc->flags & OCE_FLAGS_USING_MSIX) { di.intr_mode = OCE_INTERRUPT_MODE_MSIX; } else if (sc->flags & OCE_FLAGS_USING_MSI) { di.intr_mode = OCE_INTERRUPT_MODE_MSI; } else { di.intr_mode = OCE_INTERRUPT_MODE_INTX; } di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; di.msix.start_vector = sc->intr_count; for (i=0; iintrs[i].vector; } } else { } memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); di.vendor_id = pci_get_vendor(sc->dev); di.dev_id = pci_get_device(sc->dev); if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; } rdma_if->announce(&di); sc = sc->next; } } return 0; } static void oce_read_env_variables( POCE_SOFTC sc ) { char *value = NULL; int rc = 0; /* read if user wants to enable hwlro or swlro */ //value = getenv("oce_enable_hwlro"); if(value && IS_SH(sc)) { sc->enable_hwlro = strtol(value, NULL, 10); if(sc->enable_hwlro) { rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); if(rc) { device_printf(sc->dev, "no hardware lro support\n"); device_printf(sc->dev, "software lro enabled\n"); sc->enable_hwlro = 0; }else { device_printf(sc->dev, "hardware lro enabled\n"); oce_max_rsp_handled = 32; } }else { device_printf(sc->dev, "software lro enabled\n"); } }else { sc->enable_hwlro = 0; } /* read mbuf size */ //value = getenv("oce_rq_buf_size"); if(value && IS_SH(sc)) { oce_rq_buf_size = strtol(value, NULL, 10); switch(oce_rq_buf_size) { case 2048: case 4096: case 9216: case 16384: break; default: device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); oce_rq_buf_size = 2048; } } return; } diff --git a/sys/dev/oce/oce_if.h b/sys/dev/oce/oce_if.h index 8938dd86ecce..c7386aab68f8 100644 --- a/sys/dev/oce/oce_if.h +++ b/sys/dev/oce/oce_if.h @@ -1,1251 +1,1252 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2013 Emulex * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Emulex Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * Contact Information: * freebsd-drivers@emulex.com * * Emulex * 3333 Susan Street * Costa Mesa, CA 92626 */ /* $FreeBSD$ */ #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "oce_hw.h" /* OCE device driver module component revision informaiton */ #define COMPONENT_REVISION "11.0.50.0" /* OCE devices supported by this driver */ #define PCI_VENDOR_EMULEX 0x10df /* Emulex */ #define PCI_VENDOR_SERVERENGINES 0x19a2 /* ServerEngines (BE) */ #define PCI_PRODUCT_BE2 0x0700 /* BE2 network adapter */ #define PCI_PRODUCT_BE3 0x0710 /* BE3 network adapter */ #define PCI_PRODUCT_XE201 0xe220 /* XE201 network adapter */ #define PCI_PRODUCT_XE201_VF 0xe228 /* XE201 with VF in Lancer */ #define PCI_PRODUCT_SH 0x0720 /* Skyhawk network adapter */ #define IS_BE(sc) (((sc->flags & OCE_FLAGS_BE3) | \ (sc->flags & OCE_FLAGS_BE2))? 1:0) #define IS_BE3(sc) (sc->flags & OCE_FLAGS_BE3) #define IS_BE2(sc) (sc->flags & OCE_FLAGS_BE2) #define IS_XE201(sc) ((sc->flags & OCE_FLAGS_XE201) ? 1:0) #define HAS_A0_CHIP(sc) ((sc->flags & OCE_FLAGS_HAS_A0_CHIP) ? 1:0) #define IS_SH(sc) ((sc->flags & OCE_FLAGS_SH) ? 1 : 0) #define is_be_mode_mc(sc) ((sc->function_mode & FNM_FLEX10_MODE) || \ (sc->function_mode & FNM_UMC_MODE) || \ (sc->function_mode & FNM_VNIC_MODE)) #define OCE_FUNCTION_CAPS_SUPER_NIC 0x40 #define IS_PROFILE_SUPER_NIC(sc) (sc->function_caps & OCE_FUNCTION_CAPS_SUPER_NIC) /* proportion Service Level Interface queues */ #define OCE_MAX_UNITS 2 #define OCE_MAX_PPORT OCE_MAX_UNITS #define OCE_MAX_VPORT OCE_MAX_UNITS extern int mp_ncpus; /* system's total active cpu cores */ #define OCE_NCPUS mp_ncpus /* This should be powers of 2. Like 2,4,8 & 16 */ #define OCE_MAX_RSS 8 #define OCE_LEGACY_MODE_RSS 4 /* For BE3 Legacy mode*/ #define is_rss_enabled(sc) ((sc->function_caps & FNC_RSS) && !is_be_mode_mc(sc)) #define OCE_MIN_RQ 1 #define OCE_MIN_WQ 1 #define OCE_MAX_RQ OCE_MAX_RSS + 1 /* one default queue */ #define OCE_MAX_WQ 8 #define OCE_MAX_EQ 32 #define OCE_MAX_CQ OCE_MAX_RQ + OCE_MAX_WQ + 1 /* one MCC queue */ #define OCE_MAX_CQ_EQ 8 /* Max CQ that can attached to an EQ */ #define OCE_DEFAULT_WQ_EQD 16 #define OCE_MAX_PACKET_Q 16 #define OCE_LSO_MAX_SIZE (64 * 1024) #define LONG_TIMEOUT 30 #define OCE_MAX_JUMBO_FRAME_SIZE 9018 #define OCE_MAX_MTU (OCE_MAX_JUMBO_FRAME_SIZE - \ ETHER_VLAN_ENCAP_LEN - \ ETHER_HDR_LEN) #define OCE_RDMA_VECTORS 2 #define OCE_MAX_TX_ELEMENTS 29 #define OCE_MAX_TX_DESC 1024 #define OCE_MAX_TX_SIZE 65535 #define OCE_MAX_TSO_SIZE (65535 - ETHER_HDR_LEN) #define OCE_MAX_RX_SIZE 4096 #define OCE_MAX_RQ_POSTS 255 #define OCE_HWLRO_MAX_RQ_POSTS 64 #define OCE_DEFAULT_PROMISCUOUS 0 #define RSS_ENABLE_IPV4 0x1 #define RSS_ENABLE_TCP_IPV4 0x2 #define RSS_ENABLE_IPV6 0x4 #define RSS_ENABLE_TCP_IPV6 0x8 #define INDIRECTION_TABLE_ENTRIES 128 /* flow control definitions */ #define OCE_FC_NONE 0x00000000 #define OCE_FC_TX 0x00000001 #define OCE_FC_RX 0x00000002 #define OCE_DEFAULT_FLOW_CONTROL (OCE_FC_TX | OCE_FC_RX) /* Interface capabilities to give device when creating interface */ #define OCE_CAPAB_FLAGS (MBX_RX_IFACE_FLAGS_BROADCAST | \ MBX_RX_IFACE_FLAGS_UNTAGGED | \ MBX_RX_IFACE_FLAGS_PROMISCUOUS | \ MBX_RX_IFACE_FLAGS_VLAN_PROMISCUOUS | \ MBX_RX_IFACE_FLAGS_MCAST_PROMISCUOUS | \ MBX_RX_IFACE_FLAGS_RSS | \ MBX_RX_IFACE_FLAGS_PASS_L3L4_ERR) /* Interface capabilities to enable by default (others set dynamically) */ #define OCE_CAPAB_ENABLE (MBX_RX_IFACE_FLAGS_BROADCAST | \ MBX_RX_IFACE_FLAGS_UNTAGGED | \ MBX_RX_IFACE_FLAGS_PASS_L3L4_ERR) #define OCE_IF_HWASSIST (CSUM_IP | CSUM_TCP | CSUM_UDP) #define OCE_IF_CAPABILITIES (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | \ IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU) #define OCE_IF_HWASSIST_NONE 0 #define OCE_IF_CAPABILITIES_NONE 0 #define MAX_VLANFILTER_SIZE 64 #define MAX_VLANS 4096 #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) #define BSWAP_8(x) ((x) & 0xff) #define BSWAP_16(x) ((BSWAP_8(x) << 8) | BSWAP_8((x) >> 8)) #define BSWAP_32(x) ((BSWAP_16(x) << 16) | \ BSWAP_16((x) >> 16)) #define BSWAP_64(x) ((BSWAP_32(x) << 32) | \ BSWAP_32((x) >> 32)) #define for_all_wq_queues(sc, wq, i) \ for (i = 0, wq = sc->wq[0]; i < sc->nwqs; i++, wq = sc->wq[i]) #define for_all_rq_queues(sc, rq, i) \ for (i = 0, rq = sc->rq[0]; i < sc->nrqs; i++, rq = sc->rq[i]) #define for_all_rss_queues(sc, rq, i) \ for (i = 0, rq = sc->rq[i + 1]; i < (sc->nrqs - 1); \ i++, rq = sc->rq[i + 1]) #define for_all_evnt_queues(sc, eq, i) \ for (i = 0, eq = sc->eq[0]; i < sc->neqs; i++, eq = sc->eq[i]) #define for_all_cq_queues(sc, cq, i) \ for (i = 0, cq = sc->cq[0]; i < sc->ncqs; i++, cq = sc->cq[i]) /* Flash specific */ #define IOCTL_COOKIE "SERVERENGINES CORP" #define MAX_FLASH_COMP 32 #define IMG_ISCSI 160 #define IMG_REDBOOT 224 #define IMG_BIOS 34 #define IMG_PXEBIOS 32 #define IMG_FCOEBIOS 33 #define IMG_ISCSI_BAK 176 #define IMG_FCOE 162 #define IMG_FCOE_BAK 178 #define IMG_NCSI 16 #define IMG_PHY 192 #define FLASHROM_OPER_FLASH 1 #define FLASHROM_OPER_SAVE 2 #define FLASHROM_OPER_REPORT 4 #define FLASHROM_OPER_FLASH_PHY 9 #define FLASHROM_OPER_SAVE_PHY 10 #define TN_8022 13 enum { PHY_TYPE_CX4_10GB = 0, PHY_TYPE_XFP_10GB, PHY_TYPE_SFP_1GB, PHY_TYPE_SFP_PLUS_10GB, PHY_TYPE_KR_10GB, PHY_TYPE_KX4_10GB, PHY_TYPE_BASET_10GB, PHY_TYPE_BASET_1GB, PHY_TYPE_BASEX_1GB, PHY_TYPE_SGMII, PHY_TYPE_DISABLED = 255 }; /** * @brief Define and hold all necessary info for a single interrupt */ #define OCE_MAX_MSI 32 /* Message Signaled Interrupts */ #define OCE_MAX_MSIX 2048 /* PCI Express MSI Interrrupts */ typedef struct oce_intr_info { void *tag; /* cookie returned by bus_setup_intr */ struct resource *intr_res; /* PCI resource container */ int irq_rr; /* resource id for the interrupt */ struct oce_softc *sc; /* pointer to the parent soft c */ struct oce_eq *eq; /* pointer to the connected EQ */ struct taskqueue *tq; /* Associated task queue */ struct task task; /* task queue task */ char task_name[32]; /* task name */ int vector; /* interrupt vector number */ } OCE_INTR_INFO, *POCE_INTR_INFO; /* Ring related */ #define GET_Q_NEXT(_START, _STEP, _END) \ (((_START) + (_STEP)) < (_END) ? ((_START) + (_STEP)) \ : (((_START) + (_STEP)) - (_END))) #define DBUF_PA(obj) ((obj)->addr) #define DBUF_VA(obj) ((obj)->ptr) #define DBUF_TAG(obj) ((obj)->tag) #define DBUF_MAP(obj) ((obj)->map) #define DBUF_SYNC(obj, flags) \ (void) bus_dmamap_sync(DBUF_TAG(obj), DBUF_MAP(obj), (flags)) #define RING_NUM_PENDING(ring) ring->num_used #define RING_FULL(ring) (ring->num_used == ring->num_items) #define RING_EMPTY(ring) (ring->num_used == 0) #define RING_NUM_FREE(ring) \ (uint32_t)(ring->num_items - ring->num_used) #define RING_GET(ring, n) \ ring->cidx = GET_Q_NEXT(ring->cidx, n, ring->num_items) #define RING_PUT(ring, n) \ ring->pidx = GET_Q_NEXT(ring->pidx, n, ring->num_items) #define RING_GET_CONSUMER_ITEM_VA(ring, type) \ (void*)((type *)DBUF_VA(&ring->dma) + ring->cidx) #define RING_GET_CONSUMER_ITEM_PA(ring, type) \ (uint64_t)(((type *)DBUF_PA(ring->dbuf)) + ring->cidx) #define RING_GET_PRODUCER_ITEM_VA(ring, type) \ (void *)(((type *)DBUF_VA(&ring->dma)) + ring->pidx) #define RING_GET_PRODUCER_ITEM_PA(ring, type) \ (uint64_t)(((type *)DBUF_PA(ring->dbuf)) + ring->pidx) #define OCE_DMAPTR(o, c) ((c *)(o)->ptr) struct oce_packet_desc { struct mbuf *mbuf; bus_dmamap_t map; int nsegs; uint32_t wqe_idx; }; typedef struct oce_dma_mem { bus_dma_tag_t tag; bus_dmamap_t map; void *ptr; bus_addr_t paddr; } OCE_DMA_MEM, *POCE_DMA_MEM; typedef struct oce_ring_buffer_s { uint16_t cidx; /* Get ptr */ uint16_t pidx; /* Put Ptr */ size_t item_size; size_t num_items; uint32_t num_used; OCE_DMA_MEM dma; } oce_ring_buffer_t; /* Stats */ #define OCE_UNICAST_PACKET 0 #define OCE_MULTICAST_PACKET 1 #define OCE_BROADCAST_PACKET 2 #define OCE_RSVD_PACKET 3 struct oce_rx_stats { /* Total Receive Stats*/ uint64_t t_rx_pkts; uint64_t t_rx_bytes; uint32_t t_rx_frags; uint32_t t_rx_mcast_pkts; uint32_t t_rx_ucast_pkts; uint32_t t_rxcp_errs; }; struct oce_tx_stats { /*Total Transmit Stats */ uint64_t t_tx_pkts; uint64_t t_tx_bytes; uint32_t t_tx_reqs; uint32_t t_tx_stops; uint32_t t_tx_wrbs; uint32_t t_tx_compl; uint32_t t_ipv6_ext_hdr_tx_drop; }; struct oce_be_stats { uint8_t be_on_die_temperature; uint32_t be_tx_events; uint32_t eth_red_drops; uint32_t rx_drops_no_pbuf; uint32_t rx_drops_no_txpb; uint32_t rx_drops_no_erx_descr; uint32_t rx_drops_no_tpre_descr; uint32_t rx_drops_too_many_frags; uint32_t rx_drops_invalid_ring; uint32_t forwarded_packets; uint32_t rx_drops_mtu; uint32_t rx_crc_errors; uint32_t rx_alignment_symbol_errors; uint32_t rx_pause_frames; uint32_t rx_priority_pause_frames; uint32_t rx_control_frames; uint32_t rx_in_range_errors; uint32_t rx_out_range_errors; uint32_t rx_frame_too_long; uint32_t rx_address_match_errors; uint32_t rx_dropped_too_small; uint32_t rx_dropped_too_short; uint32_t rx_dropped_header_too_small; uint32_t rx_dropped_tcp_length; uint32_t rx_dropped_runt; uint32_t rx_ip_checksum_errs; uint32_t rx_tcp_checksum_errs; uint32_t rx_udp_checksum_errs; uint32_t rx_switched_unicast_packets; uint32_t rx_switched_multicast_packets; uint32_t rx_switched_broadcast_packets; uint32_t tx_pauseframes; uint32_t tx_priority_pauseframes; uint32_t tx_controlframes; uint32_t rxpp_fifo_overflow_drop; uint32_t rx_input_fifo_overflow_drop; uint32_t pmem_fifo_overflow_drop; uint32_t jabber_events; }; struct oce_xe201_stats { uint64_t tx_pkts; uint64_t tx_unicast_pkts; uint64_t tx_multicast_pkts; uint64_t tx_broadcast_pkts; uint64_t tx_bytes; uint64_t tx_unicast_bytes; uint64_t tx_multicast_bytes; uint64_t tx_broadcast_bytes; uint64_t tx_discards; uint64_t tx_errors; uint64_t tx_pause_frames; uint64_t tx_pause_on_frames; uint64_t tx_pause_off_frames; uint64_t tx_internal_mac_errors; uint64_t tx_control_frames; uint64_t tx_pkts_64_bytes; uint64_t tx_pkts_65_to_127_bytes; uint64_t tx_pkts_128_to_255_bytes; uint64_t tx_pkts_256_to_511_bytes; uint64_t tx_pkts_512_to_1023_bytes; uint64_t tx_pkts_1024_to_1518_bytes; uint64_t tx_pkts_1519_to_2047_bytes; uint64_t tx_pkts_2048_to_4095_bytes; uint64_t tx_pkts_4096_to_8191_bytes; uint64_t tx_pkts_8192_to_9216_bytes; uint64_t tx_lso_pkts; uint64_t rx_pkts; uint64_t rx_unicast_pkts; uint64_t rx_multicast_pkts; uint64_t rx_broadcast_pkts; uint64_t rx_bytes; uint64_t rx_unicast_bytes; uint64_t rx_multicast_bytes; uint64_t rx_broadcast_bytes; uint32_t rx_unknown_protos; uint64_t rx_discards; uint64_t rx_errors; uint64_t rx_crc_errors; uint64_t rx_alignment_errors; uint64_t rx_symbol_errors; uint64_t rx_pause_frames; uint64_t rx_pause_on_frames; uint64_t rx_pause_off_frames; uint64_t rx_frames_too_long; uint64_t rx_internal_mac_errors; uint32_t rx_undersize_pkts; uint32_t rx_oversize_pkts; uint32_t rx_fragment_pkts; uint32_t rx_jabbers; uint64_t rx_control_frames; uint64_t rx_control_frames_unknown_opcode; uint32_t rx_in_range_errors; uint32_t rx_out_of_range_errors; uint32_t rx_address_match_errors; uint32_t rx_vlan_mismatch_errors; uint32_t rx_dropped_too_small; uint32_t rx_dropped_too_short; uint32_t rx_dropped_header_too_small; uint32_t rx_dropped_invalid_tcp_length; uint32_t rx_dropped_runt; uint32_t rx_ip_checksum_errors; uint32_t rx_tcp_checksum_errors; uint32_t rx_udp_checksum_errors; uint32_t rx_non_rss_pkts; uint64_t rx_ipv4_pkts; uint64_t rx_ipv6_pkts; uint64_t rx_ipv4_bytes; uint64_t rx_ipv6_bytes; uint64_t rx_nic_pkts; uint64_t rx_tcp_pkts; uint64_t rx_iscsi_pkts; uint64_t rx_management_pkts; uint64_t rx_switched_unicast_pkts; uint64_t rx_switched_multicast_pkts; uint64_t rx_switched_broadcast_pkts; uint64_t num_forwards; uint32_t rx_fifo_overflow; uint32_t rx_input_fifo_overflow; uint64_t rx_drops_too_many_frags; uint32_t rx_drops_invalid_queue; uint64_t rx_drops_mtu; uint64_t rx_pkts_64_bytes; uint64_t rx_pkts_65_to_127_bytes; uint64_t rx_pkts_128_to_255_bytes; uint64_t rx_pkts_256_to_511_bytes; uint64_t rx_pkts_512_to_1023_bytes; uint64_t rx_pkts_1024_to_1518_bytes; uint64_t rx_pkts_1519_to_2047_bytes; uint64_t rx_pkts_2048_to_4095_bytes; uint64_t rx_pkts_4096_to_8191_bytes; uint64_t rx_pkts_8192_to_9216_bytes; }; struct oce_drv_stats { struct oce_rx_stats rx; struct oce_tx_stats tx; union { struct oce_be_stats be; struct oce_xe201_stats xe201; } u0; }; #define INTR_RATE_HWM 15000 #define INTR_RATE_LWM 10000 #define OCE_MAX_EQD 128u #define OCE_MIN_EQD 0u struct oce_set_eqd { uint32_t eq_id; uint32_t phase; uint32_t delay_multiplier; }; struct oce_aic_obj { /* Adaptive interrupt coalescing (AIC) info */ boolean_t enable; uint32_t min_eqd; /* in usecs */ uint32_t max_eqd; /* in usecs */ uint32_t cur_eqd; /* in usecs */ uint32_t et_eqd; /* configured value when aic is off */ uint64_t ticks; uint64_t prev_rxpkts; uint64_t prev_txreqs; }; #define MAX_LOCK_DESC_LEN 32 struct oce_lock { struct mtx mutex; char name[MAX_LOCK_DESC_LEN+1]; }; #define OCE_LOCK struct oce_lock #define LOCK_CREATE(lock, desc) { \ strncpy((lock)->name, (desc), MAX_LOCK_DESC_LEN); \ (lock)->name[MAX_LOCK_DESC_LEN] = '\0'; \ mtx_init(&(lock)->mutex, (lock)->name, NULL, MTX_DEF); \ } #define LOCK_DESTROY(lock) \ if (mtx_initialized(&(lock)->mutex))\ mtx_destroy(&(lock)->mutex) #define TRY_LOCK(lock) mtx_trylock(&(lock)->mutex) #define LOCK(lock) mtx_lock(&(lock)->mutex) #define LOCKED(lock) mtx_owned(&(lock)->mutex) #define UNLOCK(lock) mtx_unlock(&(lock)->mutex) #define DEFAULT_MQ_MBOX_TIMEOUT (5 * 1000 * 1000) #define MBX_READY_TIMEOUT (1 * 1000 * 1000) #define DEFAULT_DRAIN_TIME 200 #define MBX_TIMEOUT_SEC 5 #define STAT_TIMEOUT 2000000 /* size of the packet descriptor array in a transmit queue */ #define OCE_TX_RING_SIZE 2048 #define OCE_RX_RING_SIZE 1024 #define OCE_WQ_PACKET_ARRAY_SIZE (OCE_TX_RING_SIZE/2) #define OCE_RQ_PACKET_ARRAY_SIZE (OCE_RX_RING_SIZE) struct oce_dev; enum eq_len { EQ_LEN_256 = 256, EQ_LEN_512 = 512, EQ_LEN_1024 = 1024, EQ_LEN_2048 = 2048, EQ_LEN_4096 = 4096 }; enum eqe_size { EQE_SIZE_4 = 4, EQE_SIZE_16 = 16 }; enum qtype { QTYPE_EQ, QTYPE_MQ, QTYPE_WQ, QTYPE_RQ, QTYPE_CQ, QTYPE_RSS }; typedef enum qstate_e { QDELETED = 0x0, QCREATED = 0x1 } qstate_t; struct eq_config { enum eq_len q_len; enum eqe_size item_size; uint32_t q_vector_num; uint8_t min_eqd; uint8_t max_eqd; uint8_t cur_eqd; uint8_t pad; }; struct oce_eq { uint32_t eq_id; void *parent; void *cb_context; oce_ring_buffer_t *ring; uint32_t ref_count; qstate_t qstate; struct oce_cq *cq[OCE_MAX_CQ_EQ]; int cq_valid; struct eq_config eq_cfg; int vector; uint64_t intr; }; enum cq_len { CQ_LEN_256 = 256, CQ_LEN_512 = 512, CQ_LEN_1024 = 1024, CQ_LEN_2048 = 2048 }; struct cq_config { enum cq_len q_len; uint32_t item_size; boolean_t is_eventable; boolean_t sol_eventable; boolean_t nodelay; uint16_t dma_coalescing; }; typedef uint16_t(*cq_handler_t) (void *arg1); struct oce_cq { uint32_t cq_id; void *parent; struct oce_eq *eq; cq_handler_t cq_handler; void *cb_arg; oce_ring_buffer_t *ring; qstate_t qstate; struct cq_config cq_cfg; uint32_t ref_count; }; struct mq_config { uint32_t eqd; uint8_t q_len; uint8_t pad[3]; }; struct oce_mq { void *parent; oce_ring_buffer_t *ring; uint32_t mq_id; struct oce_cq *cq; struct oce_cq *async_cq; uint32_t mq_free; qstate_t qstate; struct mq_config cfg; }; struct oce_mbx_ctx { struct oce_mbx *mbx; void (*cb) (void *ctx); void *cb_ctx; }; struct wq_config { uint8_t wq_type; uint16_t buf_size; uint8_t pad[1]; uint32_t q_len; uint16_t pd_id; uint16_t pci_fn_num; uint32_t eqd; /* interrupt delay */ uint32_t nbufs; uint32_t nhdl; }; struct oce_tx_queue_stats { uint64_t tx_pkts; uint64_t tx_bytes; uint32_t tx_reqs; uint32_t tx_stops; /* number of times TX Q was stopped */ uint32_t tx_wrbs; uint32_t tx_compl; uint32_t tx_rate; uint32_t ipv6_ext_hdr_tx_drop; }; struct oce_wq { OCE_LOCK tx_lock; OCE_LOCK tx_compl_lock; void *parent; oce_ring_buffer_t *ring; struct oce_cq *cq; bus_dma_tag_t tag; struct oce_packet_desc pckts[OCE_WQ_PACKET_ARRAY_SIZE]; uint32_t pkt_desc_tail; uint32_t pkt_desc_head; uint32_t wqm_used; boolean_t resched; uint32_t wq_free; uint32_t tx_deferd; uint32_t pkt_drops; qstate_t qstate; uint16_t wq_id; struct wq_config cfg; int queue_index; struct oce_tx_queue_stats tx_stats; struct buf_ring *br; struct task txtask; uint32_t db_offset; }; struct rq_config { uint32_t q_len; uint32_t frag_size; uint32_t mtu; uint32_t if_id; uint32_t is_rss_queue; uint32_t eqd; uint32_t nbufs; }; struct oce_rx_queue_stats { uint32_t rx_post_fail; uint32_t rx_ucast_pkts; uint32_t rx_compl; uint64_t rx_bytes; uint64_t rx_bytes_prev; uint64_t rx_pkts; uint32_t rx_rate; uint32_t rx_mcast_pkts; uint32_t rxcp_err; uint32_t rx_frags; uint32_t prev_rx_frags; uint32_t rx_fps; uint32_t rx_drops_no_frags; /* HW has no fetched frags */ }; struct oce_rq { struct rq_config cfg; uint32_t rq_id; int queue_index; uint32_t rss_cpuid; void *parent; oce_ring_buffer_t *ring; struct oce_cq *cq; void *pad1; bus_dma_tag_t tag; struct oce_packet_desc pckts[OCE_RQ_PACKET_ARRAY_SIZE]; uint32_t pending; #ifdef notdef struct mbuf *head; struct mbuf *tail; int fragsleft; #endif qstate_t qstate; OCE_LOCK rx_lock; struct oce_rx_queue_stats rx_stats; struct lro_ctrl lro; int lro_pkts_queued; int islro; struct nic_hwlro_cqe_part1 *cqe_firstpart; }; struct link_status { uint8_t phys_port_speed; uint8_t logical_link_status; uint16_t qos_link_speed; }; #define OCE_FLAGS_PCIX 0x00000001 #define OCE_FLAGS_PCIE 0x00000002 #define OCE_FLAGS_MSI_CAPABLE 0x00000004 #define OCE_FLAGS_MSIX_CAPABLE 0x00000008 #define OCE_FLAGS_USING_MSI 0x00000010 #define OCE_FLAGS_USING_MSIX 0x00000020 #define OCE_FLAGS_FUNCRESET_RQD 0x00000040 #define OCE_FLAGS_VIRTUAL_PORT 0x00000080 #define OCE_FLAGS_MBOX_ENDIAN_RQD 0x00000100 #define OCE_FLAGS_BE3 0x00000200 #define OCE_FLAGS_XE201 0x00000400 #define OCE_FLAGS_BE2 0x00000800 #define OCE_FLAGS_SH 0x00001000 #define OCE_FLAGS_OS2BMC 0x00002000 #define OCE_DEV_BE2_CFG_BAR 1 #define OCE_DEV_CFG_BAR 0 #define OCE_PCI_CSR_BAR 2 #define OCE_PCI_DB_BAR 4 typedef struct oce_softc { device_t dev; OCE_LOCK dev_lock; uint32_t flags; uint32_t pcie_link_speed; uint32_t pcie_link_width; uint8_t fn; /* PCI function number */ struct resource *devcfg_res; bus_space_tag_t devcfg_btag; bus_space_handle_t devcfg_bhandle; void *devcfg_vhandle; struct resource *csr_res; bus_space_tag_t csr_btag; bus_space_handle_t csr_bhandle; void *csr_vhandle; struct resource *db_res; bus_space_tag_t db_btag; bus_space_handle_t db_bhandle; void *db_vhandle; OCE_INTR_INFO intrs[OCE_MAX_EQ]; int intr_count; int roce_intr_count; struct ifnet *ifp; struct ifmedia media; uint8_t link_status; uint8_t link_speed; uint8_t duplex; uint32_t qos_link_speed; uint32_t speed; uint32_t enable_hwlro; char fw_version[32]; struct mac_address_format macaddr; OCE_DMA_MEM bsmbx; OCE_LOCK bmbx_lock; uint32_t config_number; uint32_t asic_revision; uint32_t port_id; uint32_t function_mode; uint32_t function_caps; uint32_t max_tx_rings; uint32_t max_rx_rings; struct oce_wq *wq[OCE_MAX_WQ]; /* TX work queues */ struct oce_rq *rq[OCE_MAX_RQ]; /* RX work queues */ struct oce_cq *cq[OCE_MAX_CQ]; /* Completion queues */ struct oce_eq *eq[OCE_MAX_EQ]; /* Event queues */ struct oce_mq *mq; /* Mailbox queue */ uint32_t neqs; uint32_t ncqs; uint32_t nrqs; uint32_t nwqs; uint32_t nrssqs; uint32_t tx_ring_size; uint32_t rx_ring_size; uint32_t rq_frag_size; uint32_t if_id; /* interface ID */ uint32_t nifs; /* number of adapter interfaces, 0 or 1 */ uint32_t pmac_id; /* PMAC id */ uint32_t if_cap_flags; uint32_t flow_control; uint8_t promisc; struct oce_aic_obj aic_obj[OCE_MAX_EQ]; /*Vlan Filtering related */ eventhandler_tag vlan_attach; eventhandler_tag vlan_detach; uint16_t vlans_added; uint8_t vlan_tag[MAX_VLANS]; /*stats */ OCE_DMA_MEM stats_mem; struct oce_drv_stats oce_stats_info; struct callout timer; int8_t be3_native; uint8_t hw_error; uint16_t qnq_debug_event; uint16_t qnqid; uint32_t pvid; uint32_t max_vlans; uint32_t bmc_filt_mask; void *rdma_context; uint32_t rdma_flags; struct oce_softc *next; } OCE_SOFTC, *POCE_SOFTC; #define OCE_RDMA_FLAG_SUPPORTED 0x00000001 /************************************************** * BUS memory read/write macros * BE3: accesses three BAR spaces (CFG, CSR, DB) * Lancer: accesses one BAR space (CFG) **************************************************/ #define OCE_READ_CSR_MPU(sc, space, o) \ ((IS_BE(sc)) ? (bus_space_read_4((sc)->space##_btag, \ (sc)->space##_bhandle,o)) \ : (bus_space_read_4((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o))) #define OCE_READ_REG32(sc, space, o) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_read_4((sc)->space##_btag, \ (sc)->space##_bhandle,o)) \ : (bus_space_read_4((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o))) #define OCE_READ_REG16(sc, space, o) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_read_2((sc)->space##_btag, \ (sc)->space##_bhandle,o)) \ : (bus_space_read_2((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o))) #define OCE_READ_REG8(sc, space, o) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_read_1((sc)->space##_btag, \ (sc)->space##_bhandle,o)) \ : (bus_space_read_1((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o))) #define OCE_WRITE_CSR_MPU(sc, space, o, v) \ ((IS_BE(sc)) ? (bus_space_write_4((sc)->space##_btag, \ (sc)->space##_bhandle,o,v)) \ : (bus_space_write_4((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o,v))) #define OCE_WRITE_REG32(sc, space, o, v) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_write_4((sc)->space##_btag, \ (sc)->space##_bhandle,o,v)) \ : (bus_space_write_4((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o,v))) #define OCE_WRITE_REG16(sc, space, o, v) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_write_2((sc)->space##_btag, \ (sc)->space##_bhandle,o,v)) \ : (bus_space_write_2((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o,v))) #define OCE_WRITE_REG8(sc, space, o, v) \ ((IS_BE(sc) || IS_SH(sc)) ? (bus_space_write_1((sc)->space##_btag, \ (sc)->space##_bhandle,o,v)) \ : (bus_space_write_1((sc)->devcfg_btag, \ (sc)->devcfg_bhandle,o,v))) void oce_rx_flush_lro(struct oce_rq *rq); /*********************************************************** * DMA memory functions ***********************************************************/ #define oce_dma_sync(d, f) bus_dmamap_sync((d)->tag, (d)->map, f) int oce_dma_alloc(POCE_SOFTC sc, bus_size_t size, POCE_DMA_MEM dma, int flags); void oce_dma_free(POCE_SOFTC sc, POCE_DMA_MEM dma); void oce_dma_map_addr(void *arg, bus_dma_segment_t * segs, int nseg, int error); void oce_destroy_ring_buffer(POCE_SOFTC sc, oce_ring_buffer_t *ring); oce_ring_buffer_t *oce_create_ring_buffer(POCE_SOFTC sc, uint32_t q_len, uint32_t num_entries); /************************************************************ * oce_hw_xxx functions ************************************************************/ int oce_clear_rx_buf(struct oce_rq *rq); int oce_hw_pci_alloc(POCE_SOFTC sc); int oce_hw_init(POCE_SOFTC sc); int oce_hw_start(POCE_SOFTC sc); int oce_create_nw_interface(POCE_SOFTC sc); int oce_pci_soft_reset(POCE_SOFTC sc); int oce_hw_update_multicast(POCE_SOFTC sc); void oce_delete_nw_interface(POCE_SOFTC sc); void oce_hw_shutdown(POCE_SOFTC sc); void oce_hw_intr_enable(POCE_SOFTC sc); void oce_hw_intr_disable(POCE_SOFTC sc); void oce_hw_pci_free(POCE_SOFTC sc); /*********************************************************** * oce_queue_xxx functions ***********************************************************/ int oce_queue_init_all(POCE_SOFTC sc); int oce_start_rq(struct oce_rq *rq); int oce_start_wq(struct oce_wq *wq); int oce_start_mq(struct oce_mq *mq); int oce_start_rx(POCE_SOFTC sc); void oce_arm_eq(POCE_SOFTC sc, int16_t qid, int npopped, uint32_t rearm, uint32_t clearint); void oce_queue_release_all(POCE_SOFTC sc); void oce_arm_cq(POCE_SOFTC sc, int16_t qid, int npopped, uint32_t rearm); void oce_drain_eq(struct oce_eq *eq); void oce_drain_mq_cq(void *arg); void oce_drain_rq_cq(struct oce_rq *rq); void oce_drain_wq_cq(struct oce_wq *wq); uint32_t oce_page_list(oce_ring_buffer_t *ring, struct phys_addr *pa_list); /*********************************************************** * cleanup functions ***********************************************************/ void oce_stop_rx(POCE_SOFTC sc); void oce_discard_rx_comp(struct oce_rq *rq, int num_frags); void oce_rx_cq_clean(struct oce_rq *rq); void oce_rx_cq_clean_hwlro(struct oce_rq *rq); void oce_intr_free(POCE_SOFTC sc); void oce_free_posted_rxbuf(struct oce_rq *rq); #if defined(INET6) || defined(INET) void oce_free_lro(POCE_SOFTC sc); #endif /************************************************************ * Mailbox functions ************************************************************/ int oce_fw_clean(POCE_SOFTC sc); int oce_wait_ready(POCE_SOFTC sc); int oce_reset_fun(POCE_SOFTC sc); int oce_mbox_init(POCE_SOFTC sc); int oce_mbox_dispatch(POCE_SOFTC sc, uint32_t tmo_sec); int oce_get_fw_version(POCE_SOFTC sc); int oce_first_mcc_cmd(POCE_SOFTC sc); int oce_read_mac_addr(POCE_SOFTC sc, uint32_t if_id, uint8_t perm, uint8_t type, struct mac_address_format *mac); int oce_get_fw_config(POCE_SOFTC sc); int oce_if_create(POCE_SOFTC sc, uint32_t cap_flags, uint32_t en_flags, uint16_t vlan_tag, uint8_t *mac_addr, uint32_t *if_id); int oce_if_del(POCE_SOFTC sc, uint32_t if_id); int oce_config_vlan(POCE_SOFTC sc, uint32_t if_id, struct normal_vlan *vtag_arr, uint8_t vtag_cnt, uint32_t untagged, uint32_t enable_promisc); int oce_set_flow_control(POCE_SOFTC sc, uint32_t flow_control); int oce_config_nic_rss(POCE_SOFTC sc, uint32_t if_id, uint16_t enable_rss); int oce_rxf_set_promiscuous(POCE_SOFTC sc, uint8_t enable); int oce_set_common_iface_rx_filter(POCE_SOFTC sc, POCE_DMA_MEM sgl); int oce_get_link_status(POCE_SOFTC sc, struct link_status *link); int oce_mbox_get_nic_stats_v0(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem); int oce_mbox_get_nic_stats_v1(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem); int oce_mbox_get_nic_stats_v2(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem); int oce_mbox_get_pport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, uint32_t reset_stats); int oce_mbox_get_vport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, uint32_t req_size, uint32_t reset_stats); int oce_update_multicast(POCE_SOFTC sc, POCE_DMA_MEM pdma_mem); int oce_pass_through_mbox(POCE_SOFTC sc, POCE_DMA_MEM dma_mem, uint32_t req_size); int oce_mbox_macaddr_del(POCE_SOFTC sc, uint32_t if_id, uint32_t pmac_id); int oce_mbox_macaddr_add(POCE_SOFTC sc, uint8_t *mac_addr, uint32_t if_id, uint32_t *pmac_id); int oce_mbox_cmd_test_loopback(POCE_SOFTC sc, uint32_t port_num, uint32_t loopback_type, uint32_t pkt_size, uint32_t num_pkts, uint64_t pattern); int oce_mbox_cmd_set_loopback(POCE_SOFTC sc, uint8_t port_num, uint8_t loopback_type, uint8_t enable); int oce_mbox_check_native_mode(POCE_SOFTC sc); int oce_mbox_post(POCE_SOFTC sc, struct oce_mbx *mbx, struct oce_mbx_ctx *mbxctx); int oce_mbox_write_flashrom(POCE_SOFTC sc, uint32_t optype,uint32_t opcode, POCE_DMA_MEM pdma_mem, uint32_t num_bytes); int oce_mbox_lancer_write_flashrom(POCE_SOFTC sc, uint32_t data_size, uint32_t data_offset,POCE_DMA_MEM pdma_mem, uint32_t *written_data, uint32_t *additional_status); int oce_mbox_get_flashrom_crc(POCE_SOFTC sc, uint8_t *flash_crc, uint32_t offset, uint32_t optype); int oce_mbox_get_phy_info(POCE_SOFTC sc, struct oce_phy_info *phy_info); int oce_mbox_create_rq(struct oce_rq *rq); int oce_mbox_create_wq(struct oce_wq *wq); int oce_mbox_create_eq(struct oce_eq *eq); int oce_mbox_cq_create(struct oce_cq *cq, uint32_t ncoalesce, uint32_t is_eventable); int oce_mbox_read_transrecv_data(POCE_SOFTC sc, uint32_t page_num); void oce_mbox_eqd_modify_periodic(POCE_SOFTC sc, struct oce_set_eqd *set_eqd, int num); int oce_get_profile_config(POCE_SOFTC sc, uint32_t max_rss); int oce_get_func_config(POCE_SOFTC sc); void mbx_common_req_hdr_init(struct mbx_hdr *hdr, uint8_t dom, uint8_t port, uint8_t subsys, uint8_t opcode, uint32_t timeout, uint32_t pyld_len, uint8_t version); uint16_t oce_mq_handler(void *arg); /************************************************************ * Transmit functions ************************************************************/ uint16_t oce_wq_handler(void *arg); void oce_start(struct ifnet *ifp); void oce_tx_task(void *arg, int npending); /************************************************************ * Receive functions ************************************************************/ int oce_alloc_rx_bufs(struct oce_rq *rq, int count); uint16_t oce_rq_handler(void *arg); /* Sysctl functions */ void oce_add_sysctls(POCE_SOFTC sc); void oce_refresh_queue_stats(POCE_SOFTC sc); int oce_refresh_nic_stats(POCE_SOFTC sc); int oce_stats_init(POCE_SOFTC sc); void oce_stats_free(POCE_SOFTC sc); /* hw lro functions */ int oce_mbox_nic_query_lro_capabilities(POCE_SOFTC sc, uint32_t *lro_rq_cnt, uint32_t *lro_flags); int oce_mbox_nic_set_iface_lro_config(POCE_SOFTC sc, int enable); int oce_mbox_create_rq_v2(struct oce_rq *rq); /* Capabilities */ #define OCE_MODCAP_RSS 1 #define OCE_MAX_RSP_HANDLED 64 extern uint32_t oce_max_rsp_handled; /* max responses */ extern uint32_t oce_rq_buf_size; #define OCE_MAC_LOOPBACK 0x0 #define OCE_PHY_LOOPBACK 0x1 #define OCE_ONE_PORT_EXT_LOOPBACK 0x2 #define OCE_NO_LOOPBACK 0xff #undef IFM_40G_SR4 #define IFM_40G_SR4 28 #define atomic_inc_32(x) atomic_add_32(x, 1) #define atomic_dec_32(x) atomic_subtract_32(x, 1) #define LE_64(x) htole64(x) #define LE_32(x) htole32(x) #define LE_16(x) htole16(x) #define HOST_64(x) le64toh(x) #define HOST_32(x) le32toh(x) #define HOST_16(x) le16toh(x) #define DW_SWAP(x, l) #define IS_ALIGNED(x,a) ((x % a) == 0) #define ADDR_HI(x) ((uint32_t)((uint64_t)(x) >> 32)) #define ADDR_LO(x) ((uint32_t)((uint64_t)(x) & 0xffffffff)); #define IF_LRO_ENABLED(sc) (((sc)->ifp->if_capenable & IFCAP_LRO) ? 1:0) #define IF_LSO_ENABLED(sc) (((sc)->ifp->if_capenable & IFCAP_TSO4) ? 1:0) #define IF_CSUM_ENABLED(sc) (((sc)->ifp->if_capenable & IFCAP_HWCSUM) ? 1:0) #define OCE_LOG2(x) (oce_highbit(x)) static inline uint32_t oce_highbit(uint32_t x) { int i; int c; int b; c = 0; b = 0; for (i = 0; i < 32; i++) { if ((1 << i) & x) { c++; b = i; } } if (c == 1) return b; return 0; } static inline int MPU_EP_SEMAPHORE(POCE_SOFTC sc) { if (IS_BE(sc)) return MPU_EP_SEMAPHORE_BE3; else if (IS_SH(sc)) return MPU_EP_SEMAPHORE_SH; else return MPU_EP_SEMAPHORE_XE201; } #define TRANSCEIVER_DATA_NUM_ELE 64 #define TRANSCEIVER_DATA_SIZE 256 #define TRANSCEIVER_A0_SIZE 128 #define TRANSCEIVER_A2_SIZE 128 #define PAGE_NUM_A0 0xa0 #define PAGE_NUM_A2 0xa2 #define IS_QNQ_OR_UMC(sc) ((sc->pvid && (sc->function_mode & FNM_UMC_MODE ))\ || (sc->qnqid && (sc->function_mode & FNM_FLEX10_MODE))) extern uint8_t sfp_vpd_dump_buffer[TRANSCEIVER_DATA_SIZE]; struct oce_rdma_info; extern struct oce_rdma_if *oce_rdma_if; /* OS2BMC related */ #define DHCP_CLIENT_PORT 68 #define DHCP_SERVER_PORT 67 #define NET_BIOS_PORT1 137 #define NET_BIOS_PORT2 138 #define DHCPV6_RAS_PORT 547 #define BMC_FILT_BROADCAST_ARP ((uint32_t)(1)) #define BMC_FILT_BROADCAST_DHCP_CLIENT ((uint32_t)(1 << 1)) #define BMC_FILT_BROADCAST_DHCP_SERVER ((uint32_t)(1 << 2)) #define BMC_FILT_BROADCAST_NET_BIOS ((uint32_t)(1 << 3)) #define BMC_FILT_BROADCAST ((uint32_t)(1 << 4)) #define BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER ((uint32_t)(1 << 5)) #define BMC_FILT_MULTICAST_IPV6_RA ((uint32_t)(1 << 6)) #define BMC_FILT_MULTICAST_IPV6_RAS ((uint32_t)(1 << 7)) #define BMC_FILT_MULTICAST ((uint32_t)(1 << 8)) #define ND_ROUTER_ADVERT 134 #define ND_NEIGHBOR_ADVERT 136 #define is_mc_allowed_on_bmc(sc, eh) \ (!is_multicast_filt_enabled(sc) && \ ETHER_IS_MULTICAST(eh->ether_dhost) && \ !ETHER_IS_BROADCAST(eh->ether_dhost)) #define is_bc_allowed_on_bmc(sc, eh) \ (!is_broadcast_filt_enabled(sc) && \ ETHER_IS_BROADCAST(eh->ether_dhost)) #define is_arp_allowed_on_bmc(sc, et) \ (is_arp(et) && is_arp_filt_enabled(sc)) #define is_arp(et) (et == ETHERTYPE_ARP) #define is_arp_filt_enabled(sc) \ (sc->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP)) #define is_dhcp_client_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT) #define is_dhcp_srvr_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER) #define is_nbios_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS) #define is_ipv6_na_filt_enabled(sc) \ (sc->bmc_filt_mask & \ BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER) #define is_ipv6_ra_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA) #define is_ipv6_ras_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS) #define is_broadcast_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_BROADCAST) #define is_multicast_filt_enabled(sc) \ (sc->bmc_filt_mask & BMC_FILT_MULTICAST) #define is_os2bmc_enabled(sc) (sc->flags & OCE_FLAGS_OS2BMC) #define LRO_FLAGS_HASH_MODE 0x00000001 #define LRO_FLAGS_RSS_MODE 0x00000004 #define LRO_FLAGS_CLSC_IPV4 0x00000010 #define LRO_FLAGS_CLSC_IPV6 0x00000020 #define NIC_RQ_FLAGS_RSS 0x0001 #define NIC_RQ_FLAGS_LRO 0x0020 diff --git a/sys/dev/oce/oce_queue.c b/sys/dev/oce/oce_queue.c index 04c92a5f397b..da4d33065d40 100644 --- a/sys/dev/oce/oce_queue.c +++ b/sys/dev/oce/oce_queue.c @@ -1,1327 +1,1330 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (C) 2013 Emulex * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Emulex Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * Contact Information: * freebsd-drivers@emulex.com * * Emulex * 3333 Susan Street * Costa Mesa, CA 92626 */ /* $FreeBSD$ */ #include "oce_if.h" /***************************************************** * local queue functions *****************************************************/ static struct oce_wq *oce_wq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t wq_type); static int oce_wq_create(struct oce_wq *wq, struct oce_eq *eq); static void oce_wq_free(struct oce_wq *wq); static void oce_wq_del(struct oce_wq *wq); static struct oce_rq *oce_rq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t frag_size, uint32_t mtu, uint32_t rss); static int oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq); static void oce_rq_free(struct oce_rq *rq); static void oce_rq_del(struct oce_rq *rq); static struct oce_eq *oce_eq_create(POCE_SOFTC sc, uint32_t q_len, uint32_t item_size, uint32_t eq_delay, uint32_t vector); static void oce_eq_del(struct oce_eq *eq); static struct oce_mq *oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len); static void oce_mq_free(struct oce_mq *mq); static int oce_destroy_q(POCE_SOFTC sc, struct oce_mbx *mbx, size_t req_size, enum qtype qtype, int version); struct oce_cq *oce_cq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len, uint32_t item_size, uint32_t sol_event, uint32_t is_eventable, uint32_t nodelay, uint32_t ncoalesce); static void oce_cq_del(POCE_SOFTC sc, struct oce_cq *cq); /** * @brief Create and initialize all the queues on the board * @param sc software handle to the device * @returns 0 if successful, or error **/ int oce_queue_init_all(POCE_SOFTC sc) { int rc = 0, i, vector; struct oce_wq *wq; struct oce_rq *rq; struct oce_aic_obj *aic; /* alloc TX/RX queues */ for_all_wq_queues(sc, wq, i) { sc->wq[i] = oce_wq_init(sc, sc->tx_ring_size, NIC_WQ_TYPE_STANDARD); if (!sc->wq[i]) goto error; } for_all_rq_queues(sc, rq, i) { sc->rq[i] = oce_rq_init(sc, sc->rx_ring_size, sc->rq_frag_size, OCE_MAX_JUMBO_FRAME_SIZE, (i == 0) ? 0 : is_rss_enabled(sc)); if (!sc->rq[i]) goto error; } /* Create network interface on card */ if (oce_create_nw_interface(sc)) goto error; /* create all of the event queues */ for (vector = 0; vector < sc->intr_count; vector++) { /* setup aic defaults for each event queue */ aic = &sc->aic_obj[vector]; aic->max_eqd = OCE_MAX_EQD; aic->min_eqd = OCE_MIN_EQD; aic->et_eqd = OCE_MIN_EQD; aic->enable = TRUE; sc->eq[vector] = oce_eq_create(sc, sc->enable_hwlro ? EQ_LEN_2048 : EQ_LEN_1024, EQE_SIZE_4,0, vector); if (!sc->eq[vector]) goto error; } /* create Tx, Rx and mcc queues */ for_all_wq_queues(sc, wq, i) { rc = oce_wq_create(wq, sc->eq[i]); if (rc) goto error; wq->queue_index = i; TASK_INIT(&wq->txtask, 1, oce_tx_task, wq); } for_all_rq_queues(sc, rq, i) { rc = oce_rq_create(rq, sc->if_id, sc->eq[(i == 0) ? 0:(i-1)]); if (rc) goto error; rq->queue_index = i; } sc->mq = oce_mq_create(sc, sc->eq[0], 64); if (!sc->mq) goto error; return rc; error: oce_queue_release_all(sc); return 1; } /** * @brief Releases all mailbox queues created * @param sc software handle to the device */ void oce_queue_release_all(POCE_SOFTC sc) { int i = 0; struct oce_wq *wq; struct oce_rq *rq; struct oce_eq *eq; /* before deleting lro queues, we have to disable hwlro */ if(sc->enable_hwlro) oce_mbox_nic_set_iface_lro_config(sc, 0); for_all_rq_queues(sc, rq, i) { if (rq) { oce_rq_del(sc->rq[i]); oce_rq_free(sc->rq[i]); } } for_all_wq_queues(sc, wq, i) { if (wq) { oce_wq_del(sc->wq[i]); oce_wq_free(sc->wq[i]); } } if (sc->mq) oce_mq_free(sc->mq); for_all_evnt_queues(sc, eq, i) { if (eq) oce_eq_del(sc->eq[i]); } } /** * @brief Function to create a WQ for NIC Tx * @param sc software handle to the device * @param qlen number of entries in the queue * @param wq_type work queue type * @returns the pointer to the WQ created or NULL on failure */ static struct oce_wq *oce_wq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t wq_type) { struct oce_wq *wq; int rc = 0, i; /* q_len must be min 256 and max 2k */ if (q_len < 256 || q_len > 2048) { device_printf(sc->dev, "Invalid q length. Must be " "[256, 2000]: 0x%x\n", q_len); return NULL; } /* allocate wq */ wq = malloc(sizeof(struct oce_wq), M_DEVBUF, M_NOWAIT | M_ZERO); if (!wq) return NULL; /* Set the wq config */ wq->cfg.q_len = q_len; wq->cfg.wq_type = (uint8_t) wq_type; wq->cfg.eqd = OCE_DEFAULT_WQ_EQD; wq->cfg.nbufs = 2 * wq->cfg.q_len; wq->cfg.nhdl = 2 * wq->cfg.q_len; wq->parent = (void *)sc; rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, OCE_MAX_TX_SIZE, OCE_MAX_TX_ELEMENTS, PAGE_SIZE, 0, NULL, NULL, &wq->tag); if (rc) goto free_wq; for (i = 0; i < OCE_WQ_PACKET_ARRAY_SIZE; i++) { rc = bus_dmamap_create(wq->tag, 0, &wq->pckts[i].map); if (rc) goto free_wq; } wq->ring = oce_create_ring_buffer(sc, q_len, NIC_WQE_SIZE); if (!wq->ring) goto free_wq; LOCK_CREATE(&wq->tx_lock, "TX_lock"); LOCK_CREATE(&wq->tx_compl_lock, "WQ_HANDLER_LOCK"); /* Allocate buf ring for multiqueue*/ wq->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &wq->tx_lock.mutex); if (!wq->br) goto free_wq; return wq; free_wq: device_printf(sc->dev, "Create WQ failed\n"); oce_wq_free(wq); return NULL; } /** * @brief Frees the work queue * @param wq pointer to work queue to free */ static void oce_wq_free(struct oce_wq *wq) { POCE_SOFTC sc = (POCE_SOFTC) wq->parent; int i; taskqueue_drain(taskqueue_swi, &wq->txtask); if (wq->ring != NULL) { oce_destroy_ring_buffer(sc, wq->ring); wq->ring = NULL; } for (i = 0; i < OCE_WQ_PACKET_ARRAY_SIZE; i++) { if (wq->pckts[i].map != NULL) { bus_dmamap_unload(wq->tag, wq->pckts[i].map); bus_dmamap_destroy(wq->tag, wq->pckts[i].map); wq->pckts[i].map = NULL; } } if (wq->tag != NULL) bus_dma_tag_destroy(wq->tag); if (wq->br != NULL) buf_ring_free(wq->br, M_DEVBUF); LOCK_DESTROY(&wq->tx_lock); LOCK_DESTROY(&wq->tx_compl_lock); free(wq, M_DEVBUF); } /** * @brief Create a work queue * @param wq pointer to work queue * @param eq pointer to associated event queue */ static int oce_wq_create(struct oce_wq *wq, struct oce_eq *eq) { POCE_SOFTC sc = wq->parent; struct oce_cq *cq; int rc = 0; /* create the CQ */ cq = oce_cq_create(sc, eq, CQ_LEN_1024, sizeof(struct oce_nic_tx_cqe), 0, 1, 0, 3); if (!cq) return ENXIO; wq->cq = cq; rc = oce_mbox_create_wq(wq); if (rc) goto error; wq->qstate = QCREATED; wq->wq_free = wq->cfg.q_len; wq->ring->cidx = 0; wq->ring->pidx = 0; eq->cq[eq->cq_valid] = cq; eq->cq_valid++; cq->cb_arg = wq; cq->cq_handler = oce_wq_handler; return 0; error: device_printf(sc->dev, "WQ create failed\n"); oce_wq_del(wq); return rc; } /** * @brief Delete a work queue * @param wq pointer to work queue */ static void oce_wq_del(struct oce_wq *wq) { struct oce_mbx mbx; struct mbx_delete_nic_wq *fwcmd; POCE_SOFTC sc = (POCE_SOFTC) wq->parent; if (wq->qstate == QCREATED) { bzero(&mbx, sizeof(struct oce_mbx)); /* now fill the command */ fwcmd = (struct mbx_delete_nic_wq *)&mbx.payload; fwcmd->params.req.wq_id = wq->wq_id; (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_wq), QTYPE_WQ, 0); wq->qstate = QDELETED; } if (wq->cq != NULL) { oce_cq_del(sc, wq->cq); wq->cq = NULL; } } /** * @brief function to allocate receive queue resources * @param sc software handle to the device * @param q_len length of receive queue * @param frag_size size of an receive queue fragment * @param mtu maximum transmission unit * @param rss is-rss-queue flag * @returns the pointer to the RQ created or NULL on failure */ static struct oce_rq *oce_rq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t frag_size, uint32_t mtu, uint32_t rss) { struct oce_rq *rq; int rc = 0, i; if (OCE_LOG2(frag_size) <= 0) return NULL; if ((q_len == 0) || (q_len > 1024)) return NULL; /* allocate the rq */ rq = malloc(sizeof(struct oce_rq), M_DEVBUF, M_NOWAIT | M_ZERO); if (!rq) return NULL; rq->cfg.q_len = q_len; rq->cfg.frag_size = frag_size; rq->cfg.mtu = mtu; rq->cfg.eqd = 0; rq->lro_pkts_queued = 0; rq->cfg.is_rss_queue = rss; rq->pending = 0; rq->parent = (void *)sc; rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, oce_rq_buf_size, 1, oce_rq_buf_size, 0, NULL, NULL, &rq->tag); if (rc) goto free_rq; for (i = 0; i < OCE_RQ_PACKET_ARRAY_SIZE; i++) { rc = bus_dmamap_create(rq->tag, 0, &rq->pckts[i].map); if (rc) goto free_rq; } /* create the ring buffer */ rq->ring = oce_create_ring_buffer(sc, q_len, sizeof(struct oce_nic_rqe)); if (!rq->ring) goto free_rq; LOCK_CREATE(&rq->rx_lock, "RX_lock"); return rq; free_rq: device_printf(sc->dev, "Create RQ failed\n"); oce_rq_free(rq); return NULL; } /** * @brief Free a receive queue * @param rq pointer to receive queue */ static void oce_rq_free(struct oce_rq *rq) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; int i = 0 ; if (rq->ring != NULL) { oce_destroy_ring_buffer(sc, rq->ring); rq->ring = NULL; } for (i = 0; i < OCE_RQ_PACKET_ARRAY_SIZE; i++) { if (rq->pckts[i].map != NULL) { bus_dmamap_unload(rq->tag, rq->pckts[i].map); bus_dmamap_destroy(rq->tag, rq->pckts[i].map); rq->pckts[i].map = NULL; } if (rq->pckts[i].mbuf) { m_free(rq->pckts[i].mbuf); rq->pckts[i].mbuf = NULL; } } if (rq->tag != NULL) bus_dma_tag_destroy(rq->tag); LOCK_DESTROY(&rq->rx_lock); free(rq, M_DEVBUF); } /** * @brief Create a receive queue * @param rq receive queue * @param if_id interface identifier index` * @param eq pointer to event queue */ static int oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq) { POCE_SOFTC sc = rq->parent; struct oce_cq *cq; cq = oce_cq_create(sc, eq, sc->enable_hwlro ? CQ_LEN_2048 : CQ_LEN_1024, sizeof(struct oce_nic_rx_cqe), 0, 1, 0, 3); if (!cq) return ENXIO; rq->cq = cq; rq->cfg.if_id = if_id; /* Dont create RQ here. Create in if_activate */ rq->qstate = 0; rq->ring->cidx = 0; rq->ring->pidx = 0; eq->cq[eq->cq_valid] = cq; eq->cq_valid++; cq->cb_arg = rq; cq->cq_handler = oce_rq_handler; return 0; } /** * @brief Delete a receive queue * @param rq receive queue */ static void oce_rq_del(struct oce_rq *rq) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; struct oce_mbx mbx; struct mbx_delete_nic_rq *fwcmd; struct mbx_delete_nic_rq_v1 *fwcmd1; if (rq->qstate == QCREATED) { bzero(&mbx, sizeof(mbx)); if(!rq->islro) { fwcmd = (struct mbx_delete_nic_rq *)&mbx.payload; fwcmd->params.req.rq_id = rq->rq_id; (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq), QTYPE_RQ, 0); }else { fwcmd1 = (struct mbx_delete_nic_rq_v1 *)&mbx.payload; fwcmd1->params.req.rq_id = rq->rq_id; fwcmd1->params.req.rq_flags = (NIC_RQ_FLAGS_RSS | NIC_RQ_FLAGS_LRO); (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq_v1), QTYPE_RQ, 1); } rq->qstate = QDELETED; } if (rq->cq != NULL) { oce_cq_del(sc, rq->cq); rq->cq = NULL; } } /** * @brief function to create an event queue * @param sc software handle to the device * @param q_len length of event queue * @param item_size size of an event queue item * @param eq_delay event queue delay * @retval eq success, pointer to event queue * @retval NULL failure */ static struct oce_eq *oce_eq_create(POCE_SOFTC sc, uint32_t q_len, uint32_t item_size, uint32_t eq_delay, uint32_t vector) { struct oce_eq *eq; int rc = 0; /* allocate an eq */ eq = malloc(sizeof(struct oce_eq), M_DEVBUF, M_NOWAIT | M_ZERO); if (eq == NULL) return NULL; eq->parent = (void *)sc; eq->eq_id = 0xffff; eq->ring = oce_create_ring_buffer(sc, q_len, item_size); if (!eq->ring) goto free_eq; eq->eq_cfg.q_len = q_len; eq->eq_cfg.item_size = item_size; eq->eq_cfg.cur_eqd = (uint8_t) eq_delay; rc = oce_mbox_create_eq(eq); if (rc) goto free_eq; sc->intrs[sc->neqs++].eq = eq; return eq; free_eq: oce_eq_del(eq); return NULL; } /** * @brief Function to delete an event queue * @param eq pointer to an event queue */ static void oce_eq_del(struct oce_eq *eq) { struct oce_mbx mbx; struct mbx_destroy_common_eq *fwcmd; POCE_SOFTC sc = (POCE_SOFTC) eq->parent; if (eq->eq_id != 0xffff) { bzero(&mbx, sizeof(mbx)); fwcmd = (struct mbx_destroy_common_eq *)&mbx.payload; fwcmd->params.req.id = eq->eq_id; (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_destroy_common_eq), QTYPE_EQ, 0); } if (eq->ring != NULL) { oce_destroy_ring_buffer(sc, eq->ring); eq->ring = NULL; } free(eq, M_DEVBUF); } /** * @brief Function to create an MQ * @param sc software handle to the device * @param eq the EQ to associate with the MQ for event notification * @param q_len the number of entries to create in the MQ * @returns pointer to the created MQ, failure otherwise */ static struct oce_mq * oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len) { struct oce_mbx mbx; struct mbx_create_common_mq_ex *fwcmd = NULL; struct oce_mq *mq = NULL; int rc = 0; struct oce_cq *cq; oce_mq_ext_ctx_t *ctx; uint32_t num_pages; uint32_t page_size; int version; cq = oce_cq_create(sc, eq, CQ_LEN_256, sizeof(struct oce_mq_cqe), 1, 1, 0, 0); if (!cq) return NULL; /* allocate the mq */ mq = malloc(sizeof(struct oce_mq), M_DEVBUF, M_NOWAIT | M_ZERO); if (!mq) { oce_cq_del(sc, cq); goto error; } mq->parent = sc; mq->ring = oce_create_ring_buffer(sc, q_len, sizeof(struct oce_mbx)); if (!mq->ring) goto error; bzero(&mbx, sizeof(struct oce_mbx)); IS_XE201(sc) ? (version = OCE_MBX_VER_V1) : (version = OCE_MBX_VER_V0); fwcmd = (struct mbx_create_common_mq_ex *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CREATE_MQ_EXT, MBX_TIMEOUT_SEC, sizeof(struct mbx_create_common_mq_ex), version); num_pages = oce_page_list(mq->ring, &fwcmd->params.req.pages[0]); page_size = mq->ring->num_items * mq->ring->item_size; ctx = &fwcmd->params.req.context; if (IS_XE201(sc)) { ctx->v1.num_pages = num_pages; ctx->v1.ring_size = OCE_LOG2(q_len) + 1; ctx->v1.cq_id = cq->cq_id; ctx->v1.valid = 1; ctx->v1.async_cq_id = cq->cq_id; ctx->v1.async_cq_valid = 1; /* Subscribe to Link State and Group 5 Events(bits 1 & 5 set) */ ctx->v1.async_evt_bitmap |= LE_32(0x00000022); ctx->v1.async_evt_bitmap |= LE_32(1 << ASYNC_EVENT_CODE_DEBUG); ctx->v1.async_evt_bitmap |= LE_32(1 << ASYNC_EVENT_CODE_SLIPORT); } else { ctx->v0.num_pages = num_pages; ctx->v0.cq_id = cq->cq_id; ctx->v0.ring_size = OCE_LOG2(q_len) + 1; ctx->v0.valid = 1; /* Subscribe to Link State and Group5 Events(bits 1 & 5 set) */ ctx->v0.async_evt_bitmap = 0xffffffff; } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_common_mq_ex); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); if (!rc) rc = fwcmd->hdr.u0.rsp.status; if (rc) { device_printf(sc->dev,"%s failed - cmd status: %d\n", __FUNCTION__, rc); goto error; } mq->mq_id = LE_16(fwcmd->params.rsp.mq_id); mq->cq = cq; eq->cq[eq->cq_valid] = cq; eq->cq_valid++; mq->cq->eq = eq; mq->cfg.q_len = (uint8_t) q_len; mq->cfg.eqd = 0; mq->qstate = QCREATED; mq->cq->cb_arg = mq; mq->cq->cq_handler = oce_mq_handler; return mq; error: device_printf(sc->dev, "MQ create failed\n"); oce_mq_free(mq); mq = NULL; return mq; } /** * @brief Function to free a mailbox queue * @param mq pointer to a mailbox queue */ static void oce_mq_free(struct oce_mq *mq) { POCE_SOFTC sc = (POCE_SOFTC) mq->parent; struct oce_mbx mbx; struct mbx_destroy_common_mq *fwcmd; if (!mq) return; if (mq->ring != NULL) { oce_destroy_ring_buffer(sc, mq->ring); mq->ring = NULL; if (mq->qstate == QCREATED) { bzero(&mbx, sizeof (struct oce_mbx)); fwcmd = (struct mbx_destroy_common_mq *)&mbx.payload; fwcmd->params.req.id = mq->mq_id; (void) oce_destroy_q(sc, &mbx, sizeof (struct mbx_destroy_common_mq), QTYPE_MQ, 0); } mq->qstate = QDELETED; } if (mq->cq != NULL) { oce_cq_del(sc, mq->cq); mq->cq = NULL; } free(mq, M_DEVBUF); mq = NULL; } /** * @brief Function to delete a EQ, CQ, MQ, WQ or RQ * @param sc sofware handle to the device * @param mbx mailbox command to send to the fw to delete the queue * (mbx contains the queue information to delete) * @param req_size the size of the mbx payload dependent on the qtype * @param qtype the type of queue i.e. EQ, CQ, MQ, WQ or RQ * @returns 0 on success, failure otherwise */ static int oce_destroy_q(POCE_SOFTC sc, struct oce_mbx *mbx, size_t req_size, enum qtype qtype, int version) { struct mbx_hdr *hdr = (struct mbx_hdr *)&mbx->payload; int opcode; int subsys; int rc = 0; switch (qtype) { case QTYPE_EQ: opcode = OPCODE_COMMON_DESTROY_EQ; subsys = MBX_SUBSYSTEM_COMMON; break; case QTYPE_CQ: opcode = OPCODE_COMMON_DESTROY_CQ; subsys = MBX_SUBSYSTEM_COMMON; break; case QTYPE_MQ: opcode = OPCODE_COMMON_DESTROY_MQ; subsys = MBX_SUBSYSTEM_COMMON; break; case QTYPE_WQ: opcode = NIC_DELETE_WQ; subsys = MBX_SUBSYSTEM_NIC; break; case QTYPE_RQ: opcode = NIC_DELETE_RQ; subsys = MBX_SUBSYSTEM_NIC; break; default: return EINVAL; } mbx_common_req_hdr_init(hdr, 0, 0, subsys, opcode, MBX_TIMEOUT_SEC, req_size, version); mbx->u0.s.embedded = 1; mbx->payload_length = (uint32_t) req_size; DW_SWAP(u32ptr(mbx), mbx->payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, mbx, NULL); if (!rc) rc = hdr->u0.rsp.status; if (rc) device_printf(sc->dev,"%s failed - cmd status: %d\n", __FUNCTION__, rc); return rc; } /** * @brief Function to create a completion queue * @param sc software handle to the device * @param eq optional eq to be associated with to the cq * @param q_len length of completion queue * @param item_size size of completion queue items * @param sol_event command context event * @param is_eventable event table * @param nodelay no delay flag * @param ncoalesce no coalescence flag * @returns pointer to the cq created, NULL on failure */ struct oce_cq * oce_cq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len, uint32_t item_size, uint32_t sol_event, uint32_t is_eventable, uint32_t nodelay, uint32_t ncoalesce) { struct oce_cq *cq = NULL; int rc = 0; cq = malloc(sizeof(struct oce_cq), M_DEVBUF, M_NOWAIT | M_ZERO); if (!cq) return NULL; cq->ring = oce_create_ring_buffer(sc, q_len, item_size); if (!cq->ring) goto error; cq->parent = sc; cq->eq = eq; cq->cq_cfg.q_len = q_len; cq->cq_cfg.item_size = item_size; cq->cq_cfg.nodelay = (uint8_t) nodelay; rc = oce_mbox_cq_create(cq, ncoalesce, is_eventable); if (rc) goto error; sc->cq[sc->ncqs++] = cq; return cq; error: device_printf(sc->dev, "CQ create failed\n"); oce_cq_del(sc, cq); return NULL; } /** * @brief Deletes the completion queue * @param sc software handle to the device * @param cq pointer to a completion queue */ static void oce_cq_del(POCE_SOFTC sc, struct oce_cq *cq) { struct oce_mbx mbx; struct mbx_destroy_common_cq *fwcmd; if (cq->ring != NULL) { bzero(&mbx, sizeof(struct oce_mbx)); /* now fill the command */ fwcmd = (struct mbx_destroy_common_cq *)&mbx.payload; fwcmd->params.req.id = cq->cq_id; (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_destroy_common_cq), QTYPE_CQ, 0); /*NOW destroy the ring */ oce_destroy_ring_buffer(sc, cq->ring); cq->ring = NULL; } free(cq, M_DEVBUF); cq = NULL; } /** * @brief Start a receive queue * @param rq pointer to a receive queue */ int oce_start_rq(struct oce_rq *rq) { POCE_SOFTC sc = (POCE_SOFTC) rq->parent; int rc; if(sc->enable_hwlro) rc = oce_alloc_rx_bufs(rq, 960); else rc = oce_alloc_rx_bufs(rq, rq->cfg.q_len - 1); if (rc == 0) oce_arm_cq(rq->parent, rq->cq->cq_id, 0, TRUE); return rc; } /** * @brief Start a work queue * @param wq pointer to a work queue */ int oce_start_wq(struct oce_wq *wq) { oce_arm_cq(wq->parent, wq->cq->cq_id, 0, TRUE); return 0; } /** * @brief Start a mailbox queue * @param mq pointer to a mailbox queue */ int oce_start_mq(struct oce_mq *mq) { oce_arm_cq(mq->parent, mq->cq->cq_id, 0, TRUE); return 0; } /** * @brief Function to arm an EQ so that it can generate events * @param sc software handle to the device * @param qid id of the EQ returned by the fw at the time of creation * @param npopped number of EQEs to arm * @param rearm rearm bit enable/disable * @param clearint bit to clear the interrupt condition because of which * EQEs are generated */ void oce_arm_eq(POCE_SOFTC sc, int16_t qid, int npopped, uint32_t rearm, uint32_t clearint) { eq_db_t eq_db = { 0 }; eq_db.bits.rearm = rearm; eq_db.bits.event = 1; eq_db.bits.num_popped = npopped; eq_db.bits.clrint = clearint; eq_db.bits.qid = qid; OCE_WRITE_REG32(sc, db, PD_EQ_DB, eq_db.dw0); } /** * @brief Function to arm a CQ with CQEs * @param sc software handle to the device * @param qid id of the CQ returned by the fw at the time of creation * @param npopped number of CQEs to arm * @param rearm rearm bit enable/disable */ void oce_arm_cq(POCE_SOFTC sc, int16_t qid, int npopped, uint32_t rearm) { cq_db_t cq_db = { 0 }; cq_db.bits.rearm = rearm; cq_db.bits.num_popped = npopped; cq_db.bits.event = 0; cq_db.bits.qid = qid; OCE_WRITE_REG32(sc, db, PD_CQ_DB, cq_db.dw0); } /* * @brief function to cleanup the eqs used during stop * @param eq pointer to event queue structure * @returns the number of EQs processed */ void oce_drain_eq(struct oce_eq *eq) { struct oce_eqe *eqe; uint16_t num_eqe = 0; POCE_SOFTC sc = eq->parent; do { eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); if (eqe->evnt == 0) break; eqe->evnt = 0; bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, BUS_DMASYNC_POSTWRITE); num_eqe++; RING_GET(eq->ring, 1); } while (TRUE); oce_arm_eq(sc, eq->eq_id, num_eqe, FALSE, TRUE); } void oce_drain_wq_cq(struct oce_wq *wq) { POCE_SOFTC sc = wq->parent; struct oce_cq *cq = wq->cq; struct oce_nic_tx_cqe *cqe; int num_cqes = 0; bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); do { cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); if (cqe->u0.dw[3] == 0) break; cqe->u0.dw[3] = 0; bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); RING_GET(cq->ring, 1); num_cqes++; } while (TRUE); oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); } /* * @brief function to drain a MCQ and process its CQEs * @param dev software handle to the device * @param cq pointer to the cq to drain * @returns the number of CQEs processed */ void oce_drain_mq_cq(void *arg) { /* TODO: additional code. */ return; } /** * @brief function to process a Recieve queue * @param arg pointer to the RQ to charge * @return number of cqes processed */ void oce_drain_rq_cq(struct oce_rq *rq) { struct oce_nic_rx_cqe *cqe; uint16_t num_cqe = 0; struct oce_cq *cq; POCE_SOFTC sc; sc = rq->parent; cq = rq->cq; cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); /* dequeue till you reach an invalid cqe */ while (RQ_CQE_VALID(cqe)) { RQ_CQE_INVALIDATE(cqe); RING_GET(cq->ring, 1); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); num_cqe++; } oce_arm_cq(sc, cq->cq_id, num_cqe, FALSE); return; } void oce_free_posted_rxbuf(struct oce_rq *rq) { struct oce_packet_desc *pd; while (rq->pending) { pd = &rq->pckts[rq->ring->cidx]; bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(rq->tag, pd->map); if (pd->mbuf != NULL) { m_freem(pd->mbuf); pd->mbuf = NULL; } RING_GET(rq->ring,1); rq->pending--; } } void oce_rx_cq_clean_hwlro(struct oce_rq *rq) { struct oce_cq *cq = rq->cq; POCE_SOFTC sc = rq->parent; struct nic_hwlro_singleton_cqe *cqe; struct nic_hwlro_cqe_part2 *cqe2; int flush_wait = 0; int flush_compl = 0; int num_frags = 0; for (;;) { bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); if(cqe->valid) { if(cqe->cqe_type == 0) { /* singleton cqe */ /* we should not get singleton cqe after cqe1 on same rq */ if(rq->cqe_firstpart != NULL) { device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); goto exit_rx_cq_clean_hwlro; } num_frags = cqe->pkt_size / rq->cfg.frag_size; if(cqe->pkt_size % rq->cfg.frag_size) num_frags++; oce_discard_rx_comp(rq, num_frags); /* Check if CQE is flush completion */ if(!cqe->pkt_size) flush_compl = 1; cqe->valid = 0; RING_GET(cq->ring, 1); }else if(cqe->cqe_type == 0x1) { /* first part */ /* we should not get cqe1 after cqe1 on same rq */ if(rq->cqe_firstpart != NULL) { device_printf(sc->dev, "Got cqe1 after cqe1 \n"); goto exit_rx_cq_clean_hwlro; } rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; RING_GET(cq->ring, 1); }else if(cqe->cqe_type == 0x2) { /* second part */ cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; /* We should not get cqe2 without cqe1 */ if(rq->cqe_firstpart == NULL) { device_printf(sc->dev, "Got cqe2 without cqe1 \n"); goto exit_rx_cq_clean_hwlro; } num_frags = cqe2->coalesced_size / rq->cfg.frag_size; if(cqe2->coalesced_size % rq->cfg.frag_size) num_frags++; /* Flush completion will always come in singleton CQE */ oce_discard_rx_comp(rq, num_frags); rq->cqe_firstpart->valid = 0; cqe2->valid = 0; rq->cqe_firstpart = NULL; RING_GET(cq->ring, 1); } oce_arm_cq(sc, cq->cq_id, 1, FALSE); if(flush_compl) break; }else { if (flush_wait++ > 100) { device_printf(sc->dev, "did not receive hwlro flush compl\n"); break; } oce_arm_cq(sc, cq->cq_id, 0, TRUE); DELAY(1000); } } /* After cleanup, leave the CQ in unarmed state */ oce_arm_cq(sc, cq->cq_id, 0, FALSE); exit_rx_cq_clean_hwlro: return; } void oce_rx_cq_clean(struct oce_rq *rq) { struct oce_nic_rx_cqe *cqe; struct oce_cq *cq; POCE_SOFTC sc; int flush_wait = 0; int flush_compl = 0; sc = rq->parent; cq = rq->cq; for (;;) { bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); if(RQ_CQE_VALID(cqe)) { DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); /* Check if CQE is flush completion */ if((cqe->u0.s.num_fragments==0)&&(cqe->u0.s.pkt_size == 0)&&(cqe->u0.s.error == 0)) flush_compl = 1; RQ_CQE_INVALIDATE(cqe); RING_GET(cq->ring, 1); #if defined(INET6) || defined(INET) if (IF_LRO_ENABLED(sc)) oce_rx_flush_lro(rq); #endif oce_arm_cq(sc, cq->cq_id, 1, FALSE); if(flush_compl) break; }else { if (flush_wait++ > 100) { device_printf(sc->dev, "did not receive flush compl\n"); break; } oce_arm_cq(sc, cq->cq_id, 0, TRUE); DELAY(1000); } } /* After cleanup, leave the CQ in unarmed state */ oce_arm_cq(sc, cq->cq_id, 0, FALSE); } void oce_stop_rx(POCE_SOFTC sc) { + struct epoch_tracker et; struct oce_mbx mbx; struct mbx_delete_nic_rq *fwcmd; struct mbx_delete_nic_rq_v1 *fwcmd1; struct oce_rq *rq; int i = 0; - /* before deleting disable hwlro */ + NET_EPOCH_ENTER(et); + /* before deleting disable hwlro */ if(sc->enable_hwlro) oce_mbox_nic_set_iface_lro_config(sc, 0); for_all_rq_queues(sc, rq, i) { if (rq->qstate == QCREATED) { /* Delete rxq in firmware */ LOCK(&rq->rx_lock); bzero(&mbx, sizeof(mbx)); if(!rq->islro) { fwcmd = (struct mbx_delete_nic_rq *)&mbx.payload; fwcmd->params.req.rq_id = rq->rq_id; (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq), QTYPE_RQ, 0); }else { fwcmd1 = (struct mbx_delete_nic_rq_v1 *)&mbx.payload; fwcmd1->params.req.rq_id = rq->rq_id; fwcmd1->params.req.rq_flags = (NIC_RQ_FLAGS_RSS | NIC_RQ_FLAGS_LRO); (void)oce_destroy_q(sc,&mbx,sizeof(struct mbx_delete_nic_rq_v1),QTYPE_RQ,1); } rq->qstate = QDELETED; DELAY(1000); if(!rq->islro) oce_rx_cq_clean(rq); else oce_rx_cq_clean_hwlro(rq); /* Free posted RX buffers that are not used */ oce_free_posted_rxbuf(rq); UNLOCK(&rq->rx_lock); } } + NET_EPOCH_EXIT(et); } int oce_start_rx(POCE_SOFTC sc) { struct oce_rq *rq; int rc = 0, i; for_all_rq_queues(sc, rq, i) { if (rq->qstate == QCREATED) continue; if((i == 0) || (!sc->enable_hwlro)) { rc = oce_mbox_create_rq(rq); if (rc) goto error; rq->islro = 0; }else { rc = oce_mbox_create_rq_v2(rq); if (rc) goto error; rq->islro = 1; } /* reset queue pointers */ rq->qstate = QCREATED; rq->pending = 0; rq->ring->cidx = 0; rq->ring->pidx = 0; } if(sc->enable_hwlro) { rc = oce_mbox_nic_set_iface_lro_config(sc, 1); if (rc) goto error; } DELAY(1); /* RSS config */ if (is_rss_enabled(sc)) { rc = oce_config_nic_rss(sc, (uint8_t) sc->if_id, RSS_ENABLE); if (rc) goto error; } DELAY(1); return rc; error: device_printf(sc->dev, "Start RX failed\n"); return rc; }