Index: head/sys/dev/sfxge/sfxge.h =================================================================== --- head/sys/dev/sfxge/sfxge.h (revision 278939) +++ head/sys/dev/sfxge/sfxge.h (revision 278940) @@ -1,404 +1,405 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SFXGE_H #define _SFXGE_H #include #include #include #include #include #include #include #include #include #include #include #include /* * Backward-compatibility */ #ifndef CACHE_LINE_SIZE /* This should be right on most machines the driver will be used on, and * we needn't care too much about wasting a few KB per interface. */ #define CACHE_LINE_SIZE 128 #endif #ifndef IFCAP_LINKSTATE #define IFCAP_LINKSTATE 0 #endif #ifndef IFCAP_VLAN_HWTSO #define IFCAP_VLAN_HWTSO 0 #endif #ifndef IFM_10G_T #define IFM_10G_T IFM_UNKNOWN #endif #ifndef IFM_10G_KX4 #define IFM_10G_KX4 IFM_10G_CX4 #endif #if __FreeBSD_version >= 800054 /* Networking core is multiqueue aware. We can manage our own TX * queues and use m_pkthdr.flowid. */ #define SFXGE_HAVE_MQ #endif #if (__FreeBSD_version >= 800501 && __FreeBSD_version < 900000) || \ __FreeBSD_version >= 900003 #define SFXGE_HAVE_DESCRIBE_INTR #endif #ifdef IFM_ETH_RXPAUSE #define SFXGE_HAVE_PAUSE_MEDIAOPTS #endif #ifndef CTLTYPE_U64 #define CTLTYPE_U64 CTLTYPE_QUAD #endif #include "sfxge_rx.h" #include "sfxge_tx.h" #define ROUNDUP_POW_OF_TWO(_n) (1ULL << flsl((_n) - 1)) #define SFXGE_IP_ALIGN 2 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */ enum sfxge_evq_state { SFXGE_EVQ_UNINITIALIZED = 0, SFXGE_EVQ_INITIALIZED, SFXGE_EVQ_STARTING, SFXGE_EVQ_STARTED }; #define SFXGE_EV_BATCH 16384 struct sfxge_evq { /* Structure members below are sorted by usage order */ struct sfxge_softc *sc; struct mtx lock; unsigned int index; enum sfxge_evq_state init_state; efsys_mem_t mem; efx_evq_t *common; unsigned int read_ptr; boolean_t exception; unsigned int rx_done; unsigned int tx_done; /* Linked list of TX queues with completions to process */ struct sfxge_txq *txq; struct sfxge_txq **txqs; /* Structure members not used on event processing path */ unsigned int buf_base_id; unsigned int entries; char lock_name[SFXGE_LOCK_NAME_MAX]; } __aligned(CACHE_LINE_SIZE); #define SFXGE_NDESCS 1024 #define SFXGE_MODERATION 30 enum sfxge_intr_state { SFXGE_INTR_UNINITIALIZED = 0, SFXGE_INTR_INITIALIZED, SFXGE_INTR_TESTING, SFXGE_INTR_STARTED }; struct sfxge_intr_hdl { int eih_rid; void *eih_tag; struct resource *eih_res; }; struct sfxge_intr { enum sfxge_intr_state state; struct resource *msix_res; struct sfxge_intr_hdl *table; int n_alloc; int type; efsys_mem_t status; uint32_t zero_count; }; enum sfxge_mcdi_state { SFXGE_MCDI_UNINITIALIZED = 0, SFXGE_MCDI_INITIALIZED, SFXGE_MCDI_BUSY, SFXGE_MCDI_COMPLETED }; struct sfxge_mcdi { struct mtx lock; struct cv cv; enum sfxge_mcdi_state state; efx_mcdi_transport_t transport; /* Only used in debugging output */ char lock_name[SFXGE_LOCK_NAME_MAX]; }; struct sfxge_hw_stats { clock_t update_time; efsys_mem_t dma_buf; void *decode_buf; }; enum sfxge_port_state { SFXGE_PORT_UNINITIALIZED = 0, SFXGE_PORT_INITIALIZED, SFXGE_PORT_STARTED }; struct sfxge_port { struct sfxge_softc *sc; struct mtx lock; enum sfxge_port_state init_state; #ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS unsigned int wanted_fc; #endif struct sfxge_hw_stats phy_stats; struct sfxge_hw_stats mac_stats; efx_link_mode_t link_mode; /* Only used in debugging output */ char lock_name[SFXGE_LOCK_NAME_MAX]; }; enum sfxge_softc_state { SFXGE_UNINITIALIZED = 0, SFXGE_INITIALIZED, SFXGE_REGISTERED, SFXGE_STARTED }; struct sfxge_softc { device_t dev; struct sx softc_lock; char softc_lock_name[SFXGE_LOCK_NAME_MAX]; enum sfxge_softc_state init_state; struct ifnet *ifnet; unsigned int if_flags; struct sysctl_oid *stats_node; struct sysctl_oid *txqs_node; struct task task_reset; efx_family_t family; caddr_t vpd_data; size_t vpd_size; efx_nic_t *enp; efsys_lock_t enp_lock; unsigned int rxq_entries; unsigned int txq_entries; bus_dma_tag_t parent_dma_tag; efsys_bar_t bar; struct sfxge_intr intr; struct sfxge_mcdi mcdi; struct sfxge_port port; uint32_t buffer_table_next; struct sfxge_evq *evq[SFXGE_RX_SCALE_MAX]; unsigned int ev_moderation; #if EFSYS_OPT_QSTATS clock_t ev_stats_update_time; uint64_t ev_stats[EV_NQSTATS]; #endif unsigned int max_rss_channels; uma_zone_t rxq_cache; struct sfxge_rxq *rxq[SFXGE_RX_SCALE_MAX]; unsigned int rx_indir_table[SFXGE_RX_SCALE_MAX]; #ifdef SFXGE_HAVE_MQ struct sfxge_txq *txq[SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX]; #else struct sfxge_txq *txq[SFXGE_TXQ_NTYPES]; #endif struct ifmedia media; size_t rx_prefix_size; size_t rx_buffer_size; uma_zone_t rx_buffer_zone; #ifndef SFXGE_HAVE_MQ struct mtx tx_lock __aligned(CACHE_LINE_SIZE); char tx_lock_name[SFXGE_LOCK_NAME_MAX]; #endif + unsigned int evq_count; unsigned int rxq_count; unsigned int txq_count; }; #define SFXGE_LINK_UP(sc) ((sc)->port.link_mode != EFX_LINK_DOWN) #define SFXGE_RUNNING(sc) ((sc)->ifnet->if_drv_flags & IFF_DRV_RUNNING) #define SFXGE_PARAM(_name) "hw.sfxge." #_name SYSCTL_DECL(_hw_sfxge); /* * From sfxge.c. */ extern void sfxge_schedule_reset(struct sfxge_softc *sc); extern void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp); /* * From sfxge_dma.c. */ extern int sfxge_dma_init(struct sfxge_softc *sc); extern void sfxge_dma_fini(struct sfxge_softc *sc); extern int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len, efsys_mem_t *esmp); extern void sfxge_dma_free(efsys_mem_t *esmp); extern int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs); /* * From sfxge_ev.c. */ extern int sfxge_ev_init(struct sfxge_softc *sc); extern void sfxge_ev_fini(struct sfxge_softc *sc); extern int sfxge_ev_start(struct sfxge_softc *sc); extern void sfxge_ev_stop(struct sfxge_softc *sc); extern int sfxge_ev_qpoll(struct sfxge_evq *evq); /* * From sfxge_intr.c. */ extern int sfxge_intr_init(struct sfxge_softc *sc); extern void sfxge_intr_fini(struct sfxge_softc *sc); extern int sfxge_intr_start(struct sfxge_softc *sc); extern void sfxge_intr_stop(struct sfxge_softc *sc); /* * From sfxge_mcdi.c. */ extern int sfxge_mcdi_init(struct sfxge_softc *sc); extern void sfxge_mcdi_fini(struct sfxge_softc *sc); /* * From sfxge_port.c. */ extern int sfxge_port_init(struct sfxge_softc *sc); extern void sfxge_port_fini(struct sfxge_softc *sc); extern int sfxge_port_start(struct sfxge_softc *sc); extern void sfxge_port_stop(struct sfxge_softc *sc); extern void sfxge_mac_link_update(struct sfxge_softc *sc, efx_link_mode_t mode); extern int sfxge_mac_filter_set(struct sfxge_softc *sc); extern int sfxge_port_ifmedia_init(struct sfxge_softc *sc); #define SFXGE_MAX_MTU (9 * 1024) #define SFXGE_ADAPTER_LOCK_INIT(_sc, _ifname) \ do { \ struct sfxge_softc *__sc = (_sc); \ \ snprintf((__sc)->softc_lock_name, \ sizeof((__sc)->softc_lock_name), \ "%s:softc", (_ifname)); \ sx_init(&(__sc)->softc_lock, (__sc)->softc_lock_name); \ } while (B_FALSE) #define SFXGE_ADAPTER_LOCK_DESTROY(_sc) \ sx_destroy(&(_sc)->softc_lock) #define SFXGE_ADAPTER_LOCK(_sc) \ sx_xlock(&(_sc)->softc_lock) #define SFXGE_ADAPTER_UNLOCK(_sc) \ sx_xunlock(&(_sc)->softc_lock) #define SFXGE_ADAPTER_LOCK_ASSERT_OWNED(_sc) \ sx_assert(&(_sc)->softc_lock, LA_XLOCKED) #define SFXGE_PORT_LOCK_INIT(_port, _ifname) \ do { \ struct sfxge_port *__port = (_port); \ \ snprintf((__port)->lock_name, \ sizeof((__port)->lock_name), \ "%s:port", (_ifname)); \ mtx_init(&(__port)->lock, (__port)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_PORT_LOCK_DESTROY(_port) \ mtx_destroy(&(_port)->lock) #define SFXGE_PORT_LOCK(_port) \ mtx_lock(&(_port)->lock) #define SFXGE_PORT_UNLOCK(_port) \ mtx_unlock(&(_port)->lock) #define SFXGE_PORT_LOCK_ASSERT_OWNED(_port) \ mtx_assert(&(_port)->lock, MA_OWNED) #define SFXGE_MCDI_LOCK_INIT(_mcdi, _ifname) \ do { \ struct sfxge_mcdi *__mcdi = (_mcdi); \ \ snprintf((__mcdi)->lock_name, \ sizeof((__mcdi)->lock_name), \ "%s:mcdi", (_ifname)); \ mtx_init(&(__mcdi)->lock, (__mcdi)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_MCDI_LOCK_DESTROY(_mcdi) \ mtx_destroy(&(_mcdi)->lock) #define SFXGE_MCDI_LOCK(_mcdi) \ mtx_lock(&(_mcdi)->lock) #define SFXGE_MCDI_UNLOCK(_mcdi) \ mtx_unlock(&(_mcdi)->lock) #define SFXGE_MCDI_LOCK_ASSERT_OWNED(_mcdi) \ mtx_assert(&(_mcdi)->lock, MA_OWNED) #define SFXGE_EVQ_LOCK_INIT(_evq, _ifname, _evq_index) \ do { \ struct sfxge_evq *__evq = (_evq); \ \ snprintf((__evq)->lock_name, \ sizeof((__evq)->lock_name), \ "%s:evq%u", (_ifname), (_evq_index)); \ mtx_init(&(__evq)->lock, (__evq)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_EVQ_LOCK_DESTROY(_evq) \ mtx_destroy(&(_evq)->lock) #define SFXGE_EVQ_LOCK(_evq) \ mtx_lock(&(_evq)->lock) #define SFXGE_EVQ_UNLOCK(_evq) \ mtx_unlock(&(_evq)->lock) #define SFXGE_EVQ_LOCK_ASSERT_OWNED(_evq) \ mtx_assert(&(_evq)->lock, MA_OWNED) #endif /* _SFXGE_H */ Index: head/sys/dev/sfxge/sfxge_ev.c =================================================================== --- head/sys/dev/sfxge/sfxge_ev.c (revision 278939) +++ head/sys/dev/sfxge/sfxge_ev.c (revision 278940) @@ -1,903 +1,908 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" static void sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) { struct sfxge_softc *sc; unsigned int index; struct sfxge_rxq *rxq; struct sfxge_txq *txq; sc = evq->sc; index = evq->index; rxq = sc->rxq[index]; if ((txq = evq->txq) != NULL) { evq->txq = NULL; evq->txqs = &(evq->txq); do { struct sfxge_txq *next; next = txq->next; txq->next = NULL; KASSERT(txq->evq_index == index, ("txq->evq_index != index")); if (txq->pending != txq->completed) sfxge_tx_qcomplete(txq, evq); txq = next; } while (txq != NULL); } if (rxq->pending != rxq->completed) sfxge_rx_qcomplete(rxq, eop); } static boolean_t sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int expected; struct sfxge_rx_sw_desc *rx_desc; evq = arg; sc = evq->sc; if (evq->exception) goto done; rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); if (rxq->init_state != SFXGE_RXQ_STARTED) goto done; expected = rxq->pending++ & rxq->ptr_mask; if (id != expected) { evq->exception = B_TRUE; device_printf(sc->dev, "RX completion out of order" " (id=%#x expected=%#x flags=%#x); resetting\n", id, expected, flags); sfxge_schedule_reset(sc); goto done; } rx_desc = &rxq->queue[id]; KASSERT(rx_desc->flags == EFX_DISCARD, ("rx_desc->flags != EFX_DISCARD")); rx_desc->flags = flags; KASSERT(size < (1 << 16), ("size > (1 << 16)")); rx_desc->size = (uint16_t)size; prefetch_read_many(rx_desc->mbuf); evq->rx_done++; if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) sfxge_ev_qcomplete(evq, B_FALSE); done: return (evq->rx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; sc = evq->sc; evq->exception = B_TRUE; if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { device_printf(sc->dev, "hardware exception (code=%u); resetting\n", code); sfxge_schedule_reset(sc); } return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; evq = sc->evq[index]; label = rxq_index; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; evq = sc->evq[index]; label = rxq_index; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static struct sfxge_txq * sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) { unsigned int index; KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); return (evq->sc->txq[index]); } static boolean_t sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) { struct sfxge_evq *evq; struct sfxge_txq *txq; unsigned int stop; unsigned int delta; evq = (struct sfxge_evq *)arg; txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); if (txq->init_state != SFXGE_TXQ_STARTED) goto done; stop = (id + 1) & txq->ptr_mask; id = txq->pending & txq->ptr_mask; delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); txq->pending += delta; evq->tx_done++; if (txq->next == NULL && evq->txqs != &(txq->next)) { *(evq->txqs) = txq; evq->txqs = &(txq->next); } if (txq->pending - txq->completed >= SFXGE_TX_BATCH) sfxge_tx_qcomplete(txq, evq); done: return (evq->tx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_txq *txq; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; txq = sc->txq[txq_index]; KASSERT(txq != NULL, ("txq == NULL")); KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, ("txq not initialized")); /* Resend a software event on the correct queue */ evq = sc->evq[txq->evq_index]; label = txq->type; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_software(void *arg, uint16_t magic) { struct sfxge_evq *evq; struct sfxge_softc *sc; unsigned int label; evq = (struct sfxge_evq *)arg; sc = evq->sc; label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; switch (magic) { case SFXGE_MAGIC_RX_QFLUSH_DONE: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qflush_done(rxq); break; } case SFXGE_MAGIC_RX_QFLUSH_FAILED: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qflush_failed(rxq); break; } case SFXGE_MAGIC_RX_QREFILL: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qrefill(rxq); break; } case SFXGE_MAGIC_TX_QFLUSH_DONE: { struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); sfxge_tx_qflush_done(txq); break; } default: break; } return (B_FALSE); } static boolean_t sfxge_ev_sram(void *arg, uint32_t code) { (void)arg; (void)code; switch (code) { case EFX_SRAM_UPDATE: EFSYS_PROBE(sram_update); break; case EFX_SRAM_CLEAR: EFSYS_PROBE(sram_clear); break; case EFX_SRAM_ILLEGAL_CLEAR: EFSYS_PROBE(sram_illegal_clear); break; default: KASSERT(B_FALSE, ("Impossible SRAM event")); break; } return (B_FALSE); } static boolean_t sfxge_ev_timer(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } static boolean_t sfxge_ev_wake_up(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } #if EFSYS_OPT_QSTATS static void sfxge_ev_stat_update(struct sfxge_softc *sc) { struct sfxge_evq *evq; unsigned int index; clock_t now; SFXGE_ADAPTER_LOCK(sc); if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) goto out; now = ticks; if (now - sc->ev_stats_update_time < hz) goto out; sc->ev_stats_update_time = now; /* Add event counts from each event queue in turn */ - for (index = 0; index < sc->intr.n_alloc; index++) { + for (index = 0; index < sc->evq_count; index++) { evq = sc->evq[index]; SFXGE_EVQ_LOCK(evq); efx_ev_qstats_update(evq->common, sc->ev_stats); SFXGE_EVQ_UNLOCK(evq); } out: SFXGE_ADAPTER_UNLOCK(sc); } static int sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; sfxge_ev_stat_update(sc); return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); } static void sfxge_ev_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; char name[40]; stat_list = SYSCTL_CHILDREN(sc->stats_node); for (id = 0; id < EV_NQSTATS; id++) { snprintf(name, sizeof(name), "ev_%s", efx_ev_qstat_name(sc->enp, id)); SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, sc, id, sfxge_ev_stat_handler, "Q", ""); } } #endif /* EFSYS_OPT_QSTATS */ static void sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) { struct sfxge_evq *evq; efx_evq_t *eep; evq = sc->evq[idx]; eep = evq->common; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); (void)efx_ev_qmoderate(eep, us); } static int sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; struct sfxge_intr *intr = &sc->intr; unsigned int moderation; int error; - int index; + unsigned int index; SFXGE_ADAPTER_LOCK(sc); if (req->newptr != NULL) { if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) != 0) goto out; /* We may not be calling efx_ev_qmoderate() now, * so we have to range-check the value ourselves. */ if (moderation > efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { error = EINVAL; goto out; } sc->ev_moderation = moderation; if (intr->state == SFXGE_INTR_STARTED) { - for (index = 0; index < intr->n_alloc; index++) + for (index = 0; index < sc->evq_count; index++) sfxge_ev_qmoderate(sc, index, moderation); } } else { error = SYSCTL_OUT(req, &sc->ev_moderation, sizeof(sc->ev_moderation)); } out: SFXGE_ADAPTER_UNLOCK(sc); return (error); } static boolean_t sfxge_ev_initialized(void *arg) { struct sfxge_evq *evq; evq = (struct sfxge_evq *)arg; KASSERT(evq->init_state == SFXGE_EVQ_STARTING, ("evq not starting")); evq->init_state = SFXGE_EVQ_STARTED; return (0); } static boolean_t sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; sc = evq->sc; sfxge_mac_link_update(sc, link_mode); return (0); } static const efx_ev_callbacks_t sfxge_ev_callbacks = { .eec_initialized = sfxge_ev_initialized, .eec_rx = sfxge_ev_rx, .eec_tx = sfxge_ev_tx, .eec_exception = sfxge_ev_exception, .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, .eec_txq_flush_done = sfxge_ev_txq_flush_done, .eec_software = sfxge_ev_software, .eec_sram = sfxge_ev_sram, .eec_wake_up = sfxge_ev_wake_up, .eec_timer = sfxge_ev_timer, .eec_link_change = sfxge_ev_link_change, }; int sfxge_ev_qpoll(struct sfxge_evq *evq) { int rc; SFXGE_EVQ_LOCK(evq); if (evq->init_state != SFXGE_EVQ_STARTING && evq->init_state != SFXGE_EVQ_STARTED) { rc = EINVAL; goto fail; } /* Synchronize the DMA memory for reading */ bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, BUS_DMASYNC_POSTREAD); KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); KASSERT(evq->txq == NULL, ("evq->txq != NULL")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); /* Poll the queue */ efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); evq->rx_done = 0; evq->tx_done = 0; /* Perform any pending completion processing */ sfxge_ev_qcomplete(evq, B_TRUE); /* Re-prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail; SFXGE_EVQ_UNLOCK(evq); return (0); fail: SFXGE_EVQ_UNLOCK(evq); return (rc); } static void sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; evq->read_ptr = 0; evq->exception = B_FALSE; #if EFSYS_OPT_QSTATS /* Add event counts before discarding the common evq state */ efx_ev_qstats_update(evq->common, sc->ev_stats); #endif efx_ev_qdestroy(evq->common); efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); SFXGE_EVQ_UNLOCK(evq); } static int sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int count; int rc; evq = sc->evq[index]; esmp = &evq->mem; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); /* Clear all events. */ (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); /* Program the buffer table. */ if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, EFX_EVQ_NBUFS(evq->entries))) != 0) return (rc); /* Create the common code event queue. */ if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, evq->buf_base_id, &evq->common)) != 0) goto fail; SFXGE_EVQ_LOCK(evq); /* Set the default moderation */ (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); /* Prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail2; evq->init_state = SFXGE_EVQ_STARTING; SFXGE_EVQ_UNLOCK(evq); /* Wait for the initialization event */ count = 0; do { /* Pause for 100 ms */ pause("sfxge evq init", hz / 10); /* Check to see if the test event has been processed */ if (evq->init_state == SFXGE_EVQ_STARTED) goto done; } while (++count < 20); rc = ETIMEDOUT; goto fail3; done: return (0); fail3: SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; fail2: SFXGE_EVQ_UNLOCK(evq); efx_ev_qdestroy(evq->common); fail: efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); return (rc); } void sfxge_ev_stop(struct sfxge_softc *sc) { struct sfxge_intr *intr; efx_nic_t *enp; int index; intr = &sc->intr; enp = sc->enp; KASSERT(intr->state == SFXGE_INTR_STARTED, ("Interrupts not started")); /* Stop the event queue(s) */ - index = intr->n_alloc; + index = sc->evq_count; while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(enp); } int sfxge_ev_start(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_STARTED, ("intr->state != SFXGE_INTR_STARTED")); /* Initialize the event module */ if ((rc = efx_ev_init(sc->enp)) != 0) return (rc); /* Start the event queues */ - for (index = 0; index < intr->n_alloc; index++) { + for (index = 0; index < sc->evq_count; index++) { if ((rc = sfxge_ev_qstart(sc, index)) != 0) goto fail; } return (0); fail: /* Stop the event queue(s) */ while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(sc->enp); return (rc); } static void sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); sfxge_dma_free(&evq->mem); sc->evq[index] = NULL; SFXGE_EVQ_LOCK_DESTROY(evq); free(evq, M_SFXGE); } static int sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int rc; KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); evq->sc = sc; evq->index = index; sc->evq[index] = evq; esmp = &evq->mem; /* Build an event queue with room for one event per tx and rx buffer, * plus some extra for link state events and MCDI completions. * There are three tx queues in the first event queue and one in * other. */ if (index == 0) evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + 3 * sc->txq_entries + 128); else evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + sc->txq_entries + 128); /* Initialise TX completion list */ evq->txqs = &evq->txq; /* Allocate DMA space. */ if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) return (rc); /* Allocate buffer table entries. */ sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), &evq->buf_base_id); SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); evq->init_state = SFXGE_EVQ_INITIALIZED; return (0); } void sfxge_ev_fini(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); sc->ev_moderation = 0; /* Tear down the event queue(s). */ - index = intr->n_alloc; + index = sc->evq_count; while (--index >= 0) sfxge_ev_qfini(sc, index); + + sc->evq_count = 0; } int sfxge_ev_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; + sc->evq_count = intr->n_alloc; + KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); /* Set default interrupt moderation; add a sysctl to * read and change it. */ sc->ev_moderation = SFXGE_MODERATION; SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, sc, 0, sfxge_int_mod_handler, "IU", "sfxge interrupt moderation (us)"); /* * Initialize the event queue(s) - one per interrupt. */ - for (index = 0; index < intr->n_alloc; index++) { + for (index = 0; index < sc->evq_count; index++) { if ((rc = sfxge_ev_qinit(sc, index)) != 0) goto fail; } #if EFSYS_OPT_QSTATS sfxge_ev_stat_init(sc); #endif return (0); fail: while (--index >= 0) sfxge_ev_qfini(sc, index); + sc->evq_count = 0; return (rc); }