Index: head/sys/dev/sfxge/sfxge.h =================================================================== --- head/sys/dev/sfxge/sfxge.h (revision 341781) +++ head/sys/dev/sfxge/sfxge.h (revision 341782) @@ -1,483 +1,490 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2016 Solarflare Communications Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. * * $FreeBSD$ */ #ifndef _SFXGE_H #define _SFXGE_H #include #include #include #include #include #include #include #include #include #include #include #include "sfxge_ioc.h" /* * Debugging */ #if 0 #define DBGPRINT(dev, fmt, args...) \ device_printf(dev, "%s: " fmt "\n", __func__, ## args) #else #define DBGPRINT(dev, fmt, args...) #endif /* * Backward-compatibility */ #ifndef CACHE_LINE_SIZE /* This should be right on most machines the driver will be used on, and * we needn't care too much about wasting a few KB per interface. */ #define CACHE_LINE_SIZE 128 #endif #ifndef IFCAP_LINKSTATE #define IFCAP_LINKSTATE 0 #endif #ifndef IFCAP_VLAN_HWTSO #define IFCAP_VLAN_HWTSO 0 #endif #ifndef IFM_10G_T #define IFM_10G_T IFM_UNKNOWN #endif #ifndef IFM_10G_KX4 #define IFM_10G_KX4 IFM_10G_CX4 #endif #ifndef IFM_40G_CR4 #define IFM_40G_CR4 IFM_UNKNOWN #endif #if (__FreeBSD_version >= 800501 && __FreeBSD_version < 900000) || \ __FreeBSD_version >= 900003 #define SFXGE_HAVE_DESCRIBE_INTR #endif #ifdef IFM_ETH_RXPAUSE #define SFXGE_HAVE_PAUSE_MEDIAOPTS #endif #ifndef CTLTYPE_U64 #define CTLTYPE_U64 CTLTYPE_QUAD #endif #include "sfxge_rx.h" #include "sfxge_tx.h" #define ROUNDUP_POW_OF_TWO(_n) (1ULL << flsl((_n) - 1)) #define SFXGE_IP_ALIGN 2 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */ #define SFXGE_MAGIC_RESERVED 0x8000 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH 6 #define SFXGE_MAGIC_DMAQ_LABEL_MASK \ ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1) enum sfxge_sw_ev { SFXGE_SW_EV_RX_QFLUSH_DONE = 1, SFXGE_SW_EV_RX_QFLUSH_FAILED, SFXGE_SW_EV_RX_QREFILL, SFXGE_SW_EV_TX_QFLUSH_DONE, }; #define SFXGE_SW_EV_MAGIC(_sw_ev) \ (SFXGE_MAGIC_RESERVED | ((_sw_ev) << SFXGE_MAGIC_DMAQ_LABEL_WIDTH)) static inline uint16_t sfxge_sw_ev_mk_magic(enum sfxge_sw_ev sw_ev, unsigned int label) { KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); return SFXGE_SW_EV_MAGIC(sw_ev) | label; } static inline uint16_t sfxge_sw_ev_rxq_magic(enum sfxge_sw_ev sw_ev, struct sfxge_rxq *rxq) { return sfxge_sw_ev_mk_magic(sw_ev, 0); } static inline uint16_t sfxge_sw_ev_txq_magic(enum sfxge_sw_ev sw_ev, struct sfxge_txq *txq) { return sfxge_sw_ev_mk_magic(sw_ev, txq->type); } enum sfxge_evq_state { SFXGE_EVQ_UNINITIALIZED = 0, SFXGE_EVQ_INITIALIZED, SFXGE_EVQ_STARTING, SFXGE_EVQ_STARTED }; #define SFXGE_EV_BATCH 16384 #define SFXGE_STATS_UPDATE_PERIOD_MS 1000 struct sfxge_evq { /* Structure members below are sorted by usage order */ struct sfxge_softc *sc; struct mtx lock; unsigned int index; enum sfxge_evq_state init_state; efsys_mem_t mem; efx_evq_t *common; unsigned int read_ptr; boolean_t exception; unsigned int rx_done; unsigned int tx_done; /* Linked list of TX queues with completions to process */ struct sfxge_txq *txq; struct sfxge_txq **txqs; /* Structure members not used on event processing path */ unsigned int buf_base_id; unsigned int entries; char lock_name[SFXGE_LOCK_NAME_MAX]; +#if EFSYS_OPT_QSTATS + clock_t stats_update_time; + uint64_t stats[EV_NQSTATS]; +#endif } __aligned(CACHE_LINE_SIZE); #define SFXGE_NDESCS 1024 #define SFXGE_MODERATION 30 enum sfxge_intr_state { SFXGE_INTR_UNINITIALIZED = 0, SFXGE_INTR_INITIALIZED, SFXGE_INTR_TESTING, SFXGE_INTR_STARTED }; struct sfxge_intr_hdl { int eih_rid; void *eih_tag; struct resource *eih_res; }; struct sfxge_intr { enum sfxge_intr_state state; struct resource *msix_res; struct sfxge_intr_hdl *table; int n_alloc; int type; efsys_mem_t status; uint32_t zero_count; }; enum sfxge_mcdi_state { SFXGE_MCDI_UNINITIALIZED = 0, SFXGE_MCDI_INITIALIZED, SFXGE_MCDI_BUSY, SFXGE_MCDI_COMPLETED }; struct sfxge_mcdi { struct mtx lock; efsys_mem_t mem; enum sfxge_mcdi_state state; efx_mcdi_transport_t transport; /* Only used in debugging output */ char lock_name[SFXGE_LOCK_NAME_MAX]; }; struct sfxge_hw_stats { clock_t update_time; efsys_mem_t dma_buf; void *decode_buf; }; enum sfxge_port_state { SFXGE_PORT_UNINITIALIZED = 0, SFXGE_PORT_INITIALIZED, SFXGE_PORT_STARTED }; struct sfxge_port { struct sfxge_softc *sc; struct mtx lock; enum sfxge_port_state init_state; #ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS unsigned int wanted_fc; #endif struct sfxge_hw_stats phy_stats; struct sfxge_hw_stats mac_stats; uint16_t stats_update_period_ms; efx_link_mode_t link_mode; uint8_t mcast_addrs[EFX_MAC_MULTICAST_LIST_MAX * EFX_MAC_ADDR_LEN]; unsigned int mcast_count; /* Only used in debugging output */ char lock_name[SFXGE_LOCK_NAME_MAX]; }; enum sfxge_softc_state { SFXGE_UNINITIALIZED = 0, SFXGE_INITIALIZED, SFXGE_REGISTERED, SFXGE_STARTED }; struct sfxge_softc { device_t dev; struct sx softc_lock; char softc_lock_name[SFXGE_LOCK_NAME_MAX]; enum sfxge_softc_state init_state; struct ifnet *ifnet; unsigned int if_flags; struct sysctl_oid *stats_node; +#if EFSYS_OPT_QSTATS + struct sysctl_oid *evqs_stats_node; +#endif struct sysctl_oid *txqs_node; struct task task_reset; efx_family_t family; unsigned int mem_bar; caddr_t vpd_data; size_t vpd_size; efx_nic_t *enp; efsys_lock_t enp_lock; unsigned int rxq_entries; unsigned int txq_entries; bus_dma_tag_t parent_dma_tag; efsys_bar_t bar; struct sfxge_intr intr; struct sfxge_mcdi mcdi; struct sfxge_port port; uint32_t buffer_table_next; struct sfxge_evq *evq[SFXGE_RX_SCALE_MAX]; unsigned int ev_moderation; #if EFSYS_OPT_QSTATS clock_t ev_stats_update_time; uint64_t ev_stats[EV_NQSTATS]; #endif unsigned int max_rss_channels; struct sfxge_rxq *rxq[SFXGE_RX_SCALE_MAX]; unsigned int rx_indir_table[EFX_RSS_TBL_SIZE]; struct sfxge_txq *txq[SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX]; struct ifmedia media; size_t rx_prefix_size; size_t rx_buffer_size; size_t rx_buffer_align; int rx_cluster_size; unsigned int evq_max; unsigned int evq_count; unsigned int rxq_count; unsigned int txq_count; unsigned int tso_fw_assisted; #define SFXGE_FATSOV1 (1 << 0) #define SFXGE_FATSOV2 (1 << 1) #if EFSYS_OPT_MCDI_LOGGING int mcdi_logging; #endif }; #define SFXGE_LINK_UP(sc) \ ((sc)->port.link_mode != EFX_LINK_DOWN && \ (sc)->port.link_mode != EFX_LINK_UNKNOWN) #define SFXGE_RUNNING(sc) ((sc)->ifnet->if_drv_flags & IFF_DRV_RUNNING) #define SFXGE_PARAM(_name) "hw.sfxge." #_name SYSCTL_DECL(_hw_sfxge); /* * From sfxge.c. */ extern void sfxge_schedule_reset(struct sfxge_softc *sc); extern void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp); /* * From sfxge_dma.c. */ extern int sfxge_dma_init(struct sfxge_softc *sc); extern void sfxge_dma_fini(struct sfxge_softc *sc); extern int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len, efsys_mem_t *esmp); extern void sfxge_dma_free(efsys_mem_t *esmp); extern int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs); /* * From sfxge_ev.c. */ extern int sfxge_ev_init(struct sfxge_softc *sc); extern void sfxge_ev_fini(struct sfxge_softc *sc); extern int sfxge_ev_start(struct sfxge_softc *sc); extern void sfxge_ev_stop(struct sfxge_softc *sc); extern int sfxge_ev_qpoll(struct sfxge_evq *evq); /* * From sfxge_intr.c. */ extern int sfxge_intr_init(struct sfxge_softc *sc); extern void sfxge_intr_fini(struct sfxge_softc *sc); extern int sfxge_intr_start(struct sfxge_softc *sc); extern void sfxge_intr_stop(struct sfxge_softc *sc); /* * From sfxge_mcdi.c. */ extern int sfxge_mcdi_init(struct sfxge_softc *sc); extern void sfxge_mcdi_fini(struct sfxge_softc *sc); extern int sfxge_mcdi_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip); /* * From sfxge_nvram.c. */ extern int sfxge_nvram_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip); /* * From sfxge_port.c. */ extern int sfxge_port_init(struct sfxge_softc *sc); extern void sfxge_port_fini(struct sfxge_softc *sc); extern int sfxge_port_start(struct sfxge_softc *sc); extern void sfxge_port_stop(struct sfxge_softc *sc); extern void sfxge_mac_link_update(struct sfxge_softc *sc, efx_link_mode_t mode); extern int sfxge_mac_filter_set(struct sfxge_softc *sc); extern int sfxge_port_ifmedia_init(struct sfxge_softc *sc); extern uint64_t sfxge_get_counter(struct ifnet *ifp, ift_counter c); #define SFXGE_MAX_MTU (9 * 1024) #define SFXGE_ADAPTER_LOCK_INIT(_sc, _ifname) \ do { \ struct sfxge_softc *__sc = (_sc); \ \ snprintf((__sc)->softc_lock_name, \ sizeof((__sc)->softc_lock_name), \ "%s:softc", (_ifname)); \ sx_init(&(__sc)->softc_lock, (__sc)->softc_lock_name); \ } while (B_FALSE) #define SFXGE_ADAPTER_LOCK_DESTROY(_sc) \ sx_destroy(&(_sc)->softc_lock) #define SFXGE_ADAPTER_LOCK(_sc) \ sx_xlock(&(_sc)->softc_lock) #define SFXGE_ADAPTER_UNLOCK(_sc) \ sx_xunlock(&(_sc)->softc_lock) #define SFXGE_ADAPTER_LOCK_ASSERT_OWNED(_sc) \ sx_assert(&(_sc)->softc_lock, LA_XLOCKED) #define SFXGE_PORT_LOCK_INIT(_port, _ifname) \ do { \ struct sfxge_port *__port = (_port); \ \ snprintf((__port)->lock_name, \ sizeof((__port)->lock_name), \ "%s:port", (_ifname)); \ mtx_init(&(__port)->lock, (__port)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_PORT_LOCK_DESTROY(_port) \ mtx_destroy(&(_port)->lock) #define SFXGE_PORT_LOCK(_port) \ mtx_lock(&(_port)->lock) #define SFXGE_PORT_UNLOCK(_port) \ mtx_unlock(&(_port)->lock) #define SFXGE_PORT_LOCK_ASSERT_OWNED(_port) \ mtx_assert(&(_port)->lock, MA_OWNED) #define SFXGE_MCDI_LOCK_INIT(_mcdi, _ifname) \ do { \ struct sfxge_mcdi *__mcdi = (_mcdi); \ \ snprintf((__mcdi)->lock_name, \ sizeof((__mcdi)->lock_name), \ "%s:mcdi", (_ifname)); \ mtx_init(&(__mcdi)->lock, (__mcdi)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_MCDI_LOCK_DESTROY(_mcdi) \ mtx_destroy(&(_mcdi)->lock) #define SFXGE_MCDI_LOCK(_mcdi) \ mtx_lock(&(_mcdi)->lock) #define SFXGE_MCDI_UNLOCK(_mcdi) \ mtx_unlock(&(_mcdi)->lock) #define SFXGE_MCDI_LOCK_ASSERT_OWNED(_mcdi) \ mtx_assert(&(_mcdi)->lock, MA_OWNED) #define SFXGE_EVQ_LOCK_INIT(_evq, _ifname, _evq_index) \ do { \ struct sfxge_evq *__evq = (_evq); \ \ snprintf((__evq)->lock_name, \ sizeof((__evq)->lock_name), \ "%s:evq%u", (_ifname), (_evq_index)); \ mtx_init(&(__evq)->lock, (__evq)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_EVQ_LOCK_DESTROY(_evq) \ mtx_destroy(&(_evq)->lock) #define SFXGE_EVQ_LOCK(_evq) \ mtx_lock(&(_evq)->lock) #define SFXGE_EVQ_UNLOCK(_evq) \ mtx_unlock(&(_evq)->lock) #define SFXGE_EVQ_LOCK_ASSERT_OWNED(_evq) \ mtx_assert(&(_evq)->lock, MA_OWNED) #endif /* _SFXGE_H */ Index: head/sys/dev/sfxge/sfxge_ev.c =================================================================== --- head/sys/dev/sfxge/sfxge_ev.c (revision 341781) +++ head/sys/dev/sfxge/sfxge_ev.c (revision 341782) @@ -1,945 +1,1040 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2016 Solarflare Communications Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" static void sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) { struct sfxge_softc *sc; unsigned int index; struct sfxge_rxq *rxq; struct sfxge_txq *txq; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; index = evq->index; rxq = sc->rxq[index]; if ((txq = evq->txq) != NULL) { evq->txq = NULL; evq->txqs = &(evq->txq); do { struct sfxge_txq *next; next = txq->next; txq->next = NULL; KASSERT(txq->evq_index == index, ("txq->evq_index != index")); if (txq->pending != txq->completed) sfxge_tx_qcomplete(txq, evq); txq = next; } while (txq != NULL); } if (rxq->pending != rxq->completed) sfxge_rx_qcomplete(rxq, eop); } static struct sfxge_rxq * sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) { struct sfxge_rxq *rxq; KASSERT(label == 0, ("unexpected rxq label != 0")); rxq = evq->sc->rxq[evq->index]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); return (rxq); } static boolean_t sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int stop; unsigned int delta; struct sfxge_rx_sw_desc *rx_desc; evq = arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; if (evq->exception) goto done; rxq = sfxge_get_rxq_by_label(evq, label); if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) goto done; stop = (id + 1) & rxq->ptr_mask; id = rxq->pending & rxq->ptr_mask; delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); rxq->pending += delta; if (delta != 1) { if ((delta <= 0) || (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { evq->exception = B_TRUE; device_printf(sc->dev, "RX completion out of order" " (id=%#x delta=%u flags=%#x); resetting\n", id, delta, flags); sfxge_schedule_reset(sc); goto done; } } rx_desc = &rxq->queue[id]; prefetch_read_many(rx_desc->mbuf); for (; id != stop; id = (id + 1) & rxq->ptr_mask) { rx_desc = &rxq->queue[id]; KASSERT(rx_desc->flags == EFX_DISCARD, ("rx_desc->flags != EFX_DISCARD")); rx_desc->flags = flags; KASSERT(size < (1 << 16), ("size > (1 << 16)")); rx_desc->size = (uint16_t)size; } evq->rx_done++; if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) sfxge_ev_qcomplete(evq, B_FALSE); done: return (evq->rx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; DBGPRINT(sc->dev, "[%d] %s", evq->index, (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : "UNKNOWN"); evq->exception = B_TRUE; if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { device_printf(sc->dev, "hardware exception (code=%u); resetting\n", code); sfxge_schedule_reset(sc); } return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; uint16_t magic; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; if (index == evq->index) { sfxge_rx_qflush_done(rxq); return (B_FALSE); } evq = sc->evq[index]; magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; uint16_t magic; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; evq = sc->evq[index]; magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static struct sfxge_txq * sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) { unsigned int index; KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); return (evq->sc->txq[index]); } static boolean_t sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) { struct sfxge_evq *evq; struct sfxge_txq *txq; unsigned int stop; unsigned int delta; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) goto done; stop = (id + 1) & txq->ptr_mask; id = txq->pending & txq->ptr_mask; delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); txq->pending += delta; evq->tx_done++; if (txq->next == NULL && evq->txqs != &(txq->next)) { *(evq->txqs) = txq; evq->txqs = &(txq->next); } if (txq->pending - txq->completed >= SFXGE_TX_BATCH) sfxge_tx_qcomplete(txq, evq); done: return (evq->tx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_txq *txq; uint16_t magic; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; txq = sc->txq[txq_index]; KASSERT(txq != NULL, ("txq == NULL")); KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, ("txq not initialized")); if (txq->evq_index == evq->index) { sfxge_tx_qflush_done(txq); return (B_FALSE); } /* Resend a software event on the correct queue */ evq = sc->evq[txq->evq_index]; magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_software(void *arg, uint16_t magic) { struct sfxge_evq *evq; struct sfxge_softc *sc; unsigned int label; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; switch (magic) { case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); break; case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); break; case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); break; case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); sfxge_tx_qflush_done(txq); break; } default: break; } return (B_FALSE); } static boolean_t sfxge_ev_sram(void *arg, uint32_t code) { (void)arg; (void)code; switch (code) { case EFX_SRAM_UPDATE: EFSYS_PROBE(sram_update); break; case EFX_SRAM_CLEAR: EFSYS_PROBE(sram_clear); break; case EFX_SRAM_ILLEGAL_CLEAR: EFSYS_PROBE(sram_illegal_clear); break; default: KASSERT(B_FALSE, ("Impossible SRAM event")); break; } return (B_FALSE); } static boolean_t sfxge_ev_timer(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } static boolean_t sfxge_ev_wake_up(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } #if EFSYS_OPT_QSTATS static void +sfxge_evq_stat_update(struct sfxge_evq *evq) +{ + clock_t now; + + SFXGE_EVQ_LOCK(evq); + + if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) + goto out; + + now = ticks; + if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) + goto out; + + evq->stats_update_time = now; + efx_ev_qstats_update(evq->common, evq->stats); + +out: + SFXGE_EVQ_UNLOCK(evq); +} + +static int +sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) +{ + struct sfxge_evq *evq = arg1; + struct sfxge_softc *sc = evq->sc; + unsigned int id = arg2; + + SFXGE_ADAPTER_LOCK(sc); + + sfxge_evq_stat_update(evq); + + SFXGE_ADAPTER_UNLOCK(sc); + + return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); +} + +static int +sfxge_evq_stat_init(struct sfxge_evq *evq) +{ + struct sfxge_softc *sc = evq->sc; + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); + char name[16]; + struct sysctl_oid *evq_stats_node; + unsigned int id; + + snprintf(name, sizeof(name), "%u", evq->index); + evq_stats_node = SYSCTL_ADD_NODE(ctx, + SYSCTL_CHILDREN(sc->evqs_stats_node), + OID_AUTO, name, CTLFLAG_RD, NULL, ""); + if (evq_stats_node == NULL) + return (ENOMEM); + + for (id = 0; id < EV_NQSTATS; id++) { + SYSCTL_ADD_PROC( + ctx, SYSCTL_CHILDREN(evq_stats_node), + OID_AUTO, efx_ev_qstat_name(sc->enp, id), + CTLTYPE_U64|CTLFLAG_RD, + evq, id, sfxge_evq_stat_handler, "Q", + ""); + } + + return (0); +} + +static void sfxge_ev_stat_update(struct sfxge_softc *sc) { struct sfxge_evq *evq; unsigned int index; clock_t now; + unsigned int id; SFXGE_ADAPTER_LOCK(sc); - if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) - goto out; - now = ticks; if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) goto out; sc->ev_stats_update_time = now; - /* Add event counts from each event queue in turn */ + memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); + + /* Update and add event counts from each event queue in turn */ for (index = 0; index < sc->evq_count; index++) { evq = sc->evq[index]; - SFXGE_EVQ_LOCK(evq); - efx_ev_qstats_update(evq->common, sc->ev_stats); - SFXGE_EVQ_UNLOCK(evq); + sfxge_evq_stat_update(evq); + for (id = 0; id < EV_NQSTATS; id++) + sc->ev_stats[id] += evq->stats[id]; } out: SFXGE_ADAPTER_UNLOCK(sc); } static int sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; sfxge_ev_stat_update(sc); return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); } static void sfxge_ev_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; char name[40]; stat_list = SYSCTL_CHILDREN(sc->stats_node); for (id = 0; id < EV_NQSTATS; id++) { snprintf(name, sizeof(name), "ev_%s", efx_ev_qstat_name(sc->enp, id)); SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, sc, id, sfxge_ev_stat_handler, "Q", ""); } } #endif /* EFSYS_OPT_QSTATS */ static void sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) { struct sfxge_evq *evq; efx_evq_t *eep; evq = sc->evq[idx]; eep = evq->common; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); (void)efx_ev_qmoderate(eep, us); } static int sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; struct sfxge_intr *intr = &sc->intr; unsigned int moderation; int error; unsigned int index; SFXGE_ADAPTER_LOCK(sc); if (req->newptr != NULL) { if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) != 0) goto out; /* We may not be calling efx_ev_qmoderate() now, * so we have to range-check the value ourselves. */ if (moderation > efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { error = EINVAL; goto out; } sc->ev_moderation = moderation; if (intr->state == SFXGE_INTR_STARTED) { for (index = 0; index < sc->evq_count; index++) sfxge_ev_qmoderate(sc, index, moderation); } } else { error = SYSCTL_OUT(req, &sc->ev_moderation, sizeof(sc->ev_moderation)); } out: SFXGE_ADAPTER_UNLOCK(sc); return (error); } static boolean_t sfxge_ev_initialized(void *arg) { struct sfxge_evq *evq; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); /* Init done events may be duplicated on 7xxx */ KASSERT(evq->init_state == SFXGE_EVQ_STARTING || evq->init_state == SFXGE_EVQ_STARTED, ("evq not starting")); evq->init_state = SFXGE_EVQ_STARTED; return (0); } static boolean_t sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; sfxge_mac_link_update(sc, link_mode); return (0); } static const efx_ev_callbacks_t sfxge_ev_callbacks = { .eec_initialized = sfxge_ev_initialized, .eec_rx = sfxge_ev_rx, .eec_tx = sfxge_ev_tx, .eec_exception = sfxge_ev_exception, .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, .eec_txq_flush_done = sfxge_ev_txq_flush_done, .eec_software = sfxge_ev_software, .eec_sram = sfxge_ev_sram, .eec_wake_up = sfxge_ev_wake_up, .eec_timer = sfxge_ev_timer, .eec_link_change = sfxge_ev_link_change, }; int sfxge_ev_qpoll(struct sfxge_evq *evq) { int rc; SFXGE_EVQ_LOCK(evq); if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && evq->init_state != SFXGE_EVQ_STARTED)) { rc = EINVAL; goto fail; } /* Synchronize the DMA memory for reading */ bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, BUS_DMASYNC_POSTREAD); KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); KASSERT(evq->txq == NULL, ("evq->txq != NULL")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); /* Poll the queue */ efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); evq->rx_done = 0; evq->tx_done = 0; /* Perform any pending completion processing */ sfxge_ev_qcomplete(evq, B_TRUE); /* Re-prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail; SFXGE_EVQ_UNLOCK(evq); return (0); fail: SFXGE_EVQ_UNLOCK(evq); return (rc); } static void sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; evq->read_ptr = 0; evq->exception = B_FALSE; #if EFSYS_OPT_QSTATS /* Add event counts before discarding the common evq state */ - efx_ev_qstats_update(evq->common, sc->ev_stats); + efx_ev_qstats_update(evq->common, evq->stats); #endif efx_ev_qdestroy(evq->common); efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); SFXGE_EVQ_UNLOCK(evq); } static int sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int count; int rc; evq = sc->evq[index]; esmp = &evq->mem; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); /* Clear all events. */ (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); /* Program the buffer table. */ if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, EFX_EVQ_NBUFS(evq->entries))) != 0) return (rc); /* Create the common code event queue. */ if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, &evq->common)) != 0) goto fail; SFXGE_EVQ_LOCK(evq); /* Prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail2; evq->init_state = SFXGE_EVQ_STARTING; SFXGE_EVQ_UNLOCK(evq); /* Wait for the initialization event */ count = 0; do { /* Pause for 100 ms */ pause("sfxge evq init", hz / 10); /* Check to see if the test event has been processed */ if (evq->init_state == SFXGE_EVQ_STARTED) goto done; } while (++count < 20); rc = ETIMEDOUT; goto fail3; done: return (0); fail3: SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; fail2: SFXGE_EVQ_UNLOCK(evq); efx_ev_qdestroy(evq->common); fail: efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); return (rc); } void sfxge_ev_stop(struct sfxge_softc *sc) { struct sfxge_intr *intr; efx_nic_t *enp; int index; intr = &sc->intr; enp = sc->enp; KASSERT(intr->state == SFXGE_INTR_STARTED, ("Interrupts not started")); /* Stop the event queue(s) */ index = sc->evq_count; while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(enp); } int sfxge_ev_start(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_STARTED, ("intr->state != SFXGE_INTR_STARTED")); /* Initialize the event module */ if ((rc = efx_ev_init(sc->enp)) != 0) return (rc); /* Start the event queues */ for (index = 0; index < sc->evq_count; index++) { if ((rc = sfxge_ev_qstart(sc, index)) != 0) goto fail; } return (0); fail: /* Stop the event queue(s) */ while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(sc->enp); return (rc); } static void sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); sfxge_dma_free(&evq->mem); sc->evq[index] = NULL; SFXGE_EVQ_LOCK_DESTROY(evq); free(evq, M_SFXGE); } static int sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int rc; KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); evq->sc = sc; evq->index = index; sc->evq[index] = evq; esmp = &evq->mem; /* Build an event queue with room for one event per tx and rx buffer, * plus some extra for link state events and MCDI completions. * There are three tx queues in the first event queue and one in * other. */ if (index == 0) evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + 3 * sc->txq_entries + 128); else evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + sc->txq_entries + 128); /* Initialise TX completion list */ evq->txqs = &evq->txq; /* Allocate DMA space. */ if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) return (rc); /* Allocate buffer table entries. */ sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), &evq->buf_base_id); SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); evq->init_state = SFXGE_EVQ_INITIALIZED; +#if EFSYS_OPT_QSTATS + rc = sfxge_evq_stat_init(evq); + if (rc != 0) + goto fail_evq_stat_init; +#endif + return (0); + +#if EFSYS_OPT_QSTATS +fail_evq_stat_init: + evq->init_state = SFXGE_EVQ_UNINITIALIZED; + SFXGE_EVQ_LOCK_DESTROY(evq); + sfxge_dma_free(esmp); + sc->evq[index] = NULL; + free(evq, M_SFXGE); + + return (rc); +#endif } void sfxge_ev_fini(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); sc->ev_moderation = 0; /* Tear down the event queue(s). */ index = sc->evq_count; while (--index >= 0) sfxge_ev_qfini(sc, index); sc->evq_count = 0; } int sfxge_ev_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; sc->evq_count = intr->n_alloc; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); /* Set default interrupt moderation; add a sysctl to * read and change it. */ sc->ev_moderation = SFXGE_MODERATION; SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, sc, 0, sfxge_int_mod_handler, "IU", "sfxge interrupt moderation (us)"); +#if EFSYS_OPT_QSTATS + sc->evqs_stats_node = SYSCTL_ADD_NODE( + device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), + OID_AUTO, "evq", CTLFLAG_RD, NULL, "Event queues stats"); + if (sc->evqs_stats_node == NULL) { + rc = ENOMEM; + goto fail_evqs_stats_node; + } +#endif + /* * Initialize the event queue(s) - one per interrupt. */ for (index = 0; index < sc->evq_count; index++) { if ((rc = sfxge_ev_qinit(sc, index)) != 0) goto fail; } #if EFSYS_OPT_QSTATS sfxge_ev_stat_init(sc); #endif return (0); fail: while (--index >= 0) sfxge_ev_qfini(sc, index); +#if EFSYS_OPT_QSTATS +fail_evqs_stats_node: +#endif sc->evq_count = 0; return (rc); }