diff --git a/sys/arm/ti/cpsw/if_cpsw.c b/sys/arm/ti/cpsw/if_cpsw.c index 674a7ed8575a..5b443e6c2712 100644 --- a/sys/arm/ti/cpsw/if_cpsw.c +++ b/sys/arm/ti/cpsw/if_cpsw.c @@ -1,3014 +1,3014 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 Damjan Marion * Copyright (c) 2016 Rubicon Communications, LLC (Netgate) * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * TI Common Platform Ethernet Switch (CPSW) Driver * Found in TI8148 "DaVinci" and AM335x "Sitara" SoCs. * * This controller is documented in the AM335x Technical Reference * Manual, in the TMS320DM814x DaVinci Digital Video Processors TRM * and in the TMS320C6452 3 Port Switch Ethernet Subsystem TRM. * * It is basically a single Ethernet port (port 0) wired internally to * a 3-port store-and-forward switch connected to two independent * "sliver" controllers (port 1 and port 2). You can operate the * controller in a variety of different ways by suitably configuring * the slivers and the Address Lookup Engine (ALE) that routes packets * between the ports. * * This code was developed and tested on a BeagleBone with * an AM335x SoC. */ #include #include "opt_cpsw.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "syscon_if.h" #include #include #include #include #include #include #ifdef CPSW_ETHERSWITCH #include #include "etherswitch_if.h" #endif #include "if_cpswreg.h" #include "if_cpswvar.h" #include "miibus_if.h" /* Device probe/attach/detach. */ static int cpsw_probe(device_t); static int cpsw_attach(device_t); static int cpsw_detach(device_t); static int cpswp_probe(device_t); static int cpswp_attach(device_t); static int cpswp_detach(device_t); static phandle_t cpsw_get_node(device_t, device_t); /* Device Init/shutdown. */ static int cpsw_shutdown(device_t); static void cpswp_init(void *); static void cpswp_init_locked(void *); static void cpswp_stop_locked(struct cpswp_softc *); /* Device Suspend/Resume. */ static int cpsw_suspend(device_t); static int cpsw_resume(device_t); /* Ioctl. */ static int cpswp_ioctl(if_t, u_long command, caddr_t data); static int cpswp_miibus_readreg(device_t, int phy, int reg); static int cpswp_miibus_writereg(device_t, int phy, int reg, int value); static void cpswp_miibus_statchg(device_t); /* Send/Receive packets. */ static void cpsw_intr_rx(void *arg); static struct mbuf *cpsw_rx_dequeue(struct cpsw_softc *); static void cpsw_rx_enqueue(struct cpsw_softc *); static void cpswp_start(if_t); static void cpsw_intr_tx(void *); static void cpswp_tx_enqueue(struct cpswp_softc *); static int cpsw_tx_dequeue(struct cpsw_softc *); /* Misc interrupts and watchdog. */ static void cpsw_intr_rx_thresh(void *); static void cpsw_intr_misc(void *); static void cpswp_tick(void *); static void cpswp_ifmedia_sts(if_t, struct ifmediareq *); static int cpswp_ifmedia_upd(if_t); static void cpsw_tx_watchdog(void *); /* ALE support */ static void cpsw_ale_read_entry(struct cpsw_softc *, uint16_t, uint32_t *); static void cpsw_ale_write_entry(struct cpsw_softc *, uint16_t, uint32_t *); static int cpsw_ale_mc_entry_set(struct cpsw_softc *, uint8_t, int, uint8_t *); static void cpsw_ale_dump_table(struct cpsw_softc *); static int cpsw_ale_update_vlan_table(struct cpsw_softc *, int, int, int, int, int); static int cpswp_ale_update_addresses(struct cpswp_softc *, int); /* Statistics and sysctls. */ static void cpsw_add_sysctls(struct cpsw_softc *); static void cpsw_stats_collect(struct cpsw_softc *); static int cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS); #ifdef CPSW_ETHERSWITCH static etherswitch_info_t *cpsw_getinfo(device_t); static int cpsw_getport(device_t, etherswitch_port_t *); static int cpsw_setport(device_t, etherswitch_port_t *); static int cpsw_getconf(device_t, etherswitch_conf_t *); static int cpsw_getvgroup(device_t, etherswitch_vlangroup_t *); static int cpsw_setvgroup(device_t, etherswitch_vlangroup_t *); static int cpsw_readreg(device_t, int); static int cpsw_writereg(device_t, int, int); static int cpsw_readphy(device_t, int, int); static int cpsw_writephy(device_t, int, int, int); #endif /* * Arbitrary limit on number of segments in an mbuf to be transmitted. * Packets with more segments than this will be defragmented before * they are queued. */ #define CPSW_TXFRAGS 16 /* Shared resources. */ static device_method_t cpsw_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cpsw_probe), DEVMETHOD(device_attach, cpsw_attach), DEVMETHOD(device_detach, cpsw_detach), DEVMETHOD(device_shutdown, cpsw_shutdown), DEVMETHOD(device_suspend, cpsw_suspend), DEVMETHOD(device_resume, cpsw_resume), /* Bus interface */ DEVMETHOD(bus_add_child, device_add_child_ordered), /* OFW methods */ DEVMETHOD(ofw_bus_get_node, cpsw_get_node), #ifdef CPSW_ETHERSWITCH /* etherswitch interface */ DEVMETHOD(etherswitch_getinfo, cpsw_getinfo), DEVMETHOD(etherswitch_readreg, cpsw_readreg), DEVMETHOD(etherswitch_writereg, cpsw_writereg), DEVMETHOD(etherswitch_readphyreg, cpsw_readphy), DEVMETHOD(etherswitch_writephyreg, cpsw_writephy), DEVMETHOD(etherswitch_getport, cpsw_getport), DEVMETHOD(etherswitch_setport, cpsw_setport), DEVMETHOD(etherswitch_getvgroup, cpsw_getvgroup), DEVMETHOD(etherswitch_setvgroup, cpsw_setvgroup), DEVMETHOD(etherswitch_getconf, cpsw_getconf), #endif DEVMETHOD_END }; static driver_t cpsw_driver = { "cpswss", cpsw_methods, sizeof(struct cpsw_softc), }; DRIVER_MODULE(cpswss, simplebus, cpsw_driver, 0, 0); /* Port/Slave resources. */ static device_method_t cpswp_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cpswp_probe), DEVMETHOD(device_attach, cpswp_attach), DEVMETHOD(device_detach, cpswp_detach), /* MII interface */ DEVMETHOD(miibus_readreg, cpswp_miibus_readreg), DEVMETHOD(miibus_writereg, cpswp_miibus_writereg), DEVMETHOD(miibus_statchg, cpswp_miibus_statchg), DEVMETHOD_END }; static driver_t cpswp_driver = { "cpsw", cpswp_methods, sizeof(struct cpswp_softc), }; #ifdef CPSW_ETHERSWITCH DRIVER_MODULE(etherswitch, cpswss, etherswitch_driver, 0, 0); MODULE_DEPEND(cpswss, etherswitch, 1, 1, 1); #endif DRIVER_MODULE(cpsw, cpswss, cpswp_driver, 0, 0); DRIVER_MODULE(miibus, cpsw, miibus_driver, 0, 0); MODULE_DEPEND(cpsw, ether, 1, 1, 1); MODULE_DEPEND(cpsw, miibus, 1, 1, 1); #ifdef CPSW_ETHERSWITCH static struct cpsw_vlangroups cpsw_vgroups[CPSW_VLANS]; #endif static uint32_t slave_mdio_addr[] = { 0x4a100200, 0x4a100300 }; static struct resource_spec irq_res_spec[] = { { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, { SYS_RES_IRQ, 1, RF_ACTIVE | RF_SHAREABLE }, { SYS_RES_IRQ, 2, RF_ACTIVE | RF_SHAREABLE }, { SYS_RES_IRQ, 3, RF_ACTIVE | RF_SHAREABLE }, { -1, 0 } }; static struct { void (*cb)(void *); } cpsw_intr_cb[] = { { cpsw_intr_rx_thresh }, { cpsw_intr_rx }, { cpsw_intr_tx }, { cpsw_intr_misc }, }; /* Number of entries here must match size of stats * array in struct cpswp_softc. */ static struct cpsw_stat { int reg; char *oid; } cpsw_stat_sysctls[CPSW_SYSCTL_COUNT] = { {0x00, "GoodRxFrames"}, {0x04, "BroadcastRxFrames"}, {0x08, "MulticastRxFrames"}, {0x0C, "PauseRxFrames"}, {0x10, "RxCrcErrors"}, {0x14, "RxAlignErrors"}, {0x18, "OversizeRxFrames"}, {0x1c, "RxJabbers"}, {0x20, "ShortRxFrames"}, {0x24, "RxFragments"}, {0x30, "RxOctets"}, {0x34, "GoodTxFrames"}, {0x38, "BroadcastTxFrames"}, {0x3c, "MulticastTxFrames"}, {0x40, "PauseTxFrames"}, {0x44, "DeferredTxFrames"}, {0x48, "CollisionsTxFrames"}, {0x4c, "SingleCollisionTxFrames"}, {0x50, "MultipleCollisionTxFrames"}, {0x54, "ExcessiveCollisions"}, {0x58, "LateCollisions"}, {0x5c, "TxUnderrun"}, {0x60, "CarrierSenseErrors"}, {0x64, "TxOctets"}, {0x68, "RxTx64OctetFrames"}, {0x6c, "RxTx65to127OctetFrames"}, {0x70, "RxTx128to255OctetFrames"}, {0x74, "RxTx256to511OctetFrames"}, {0x78, "RxTx512to1024OctetFrames"}, {0x7c, "RxTx1024upOctetFrames"}, {0x80, "NetOctets"}, {0x84, "RxStartOfFrameOverruns"}, {0x88, "RxMiddleOfFrameOverruns"}, {0x8c, "RxDmaOverruns"} }; /* * Basic debug support. */ static void cpsw_debugf_head(const char *funcname) { int t = (int)(time_second % (24 * 60 * 60)); printf("%02d:%02d:%02d %s ", t / (60 * 60), (t / 60) % 60, t % 60, funcname); } static void cpsw_debugf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); vprintf(fmt, ap); va_end(ap); printf("\n"); } #define CPSW_DEBUGF(_sc, a) do { \ if ((_sc)->debug) { \ cpsw_debugf_head(__func__); \ cpsw_debugf a; \ } \ } while (0) /* * Locking macros */ #define CPSW_TX_LOCK(sc) do { \ mtx_assert(&(sc)->rx.lock, MA_NOTOWNED); \ mtx_lock(&(sc)->tx.lock); \ } while (0) #define CPSW_TX_UNLOCK(sc) mtx_unlock(&(sc)->tx.lock) #define CPSW_TX_LOCK_ASSERT(sc) mtx_assert(&(sc)->tx.lock, MA_OWNED) #define CPSW_RX_LOCK(sc) do { \ mtx_assert(&(sc)->tx.lock, MA_NOTOWNED); \ mtx_lock(&(sc)->rx.lock); \ } while (0) #define CPSW_RX_UNLOCK(sc) mtx_unlock(&(sc)->rx.lock) #define CPSW_RX_LOCK_ASSERT(sc) mtx_assert(&(sc)->rx.lock, MA_OWNED) #define CPSW_PORT_LOCK(_sc) do { \ mtx_assert(&(_sc)->lock, MA_NOTOWNED); \ mtx_lock(&(_sc)->lock); \ } while (0) #define CPSW_PORT_UNLOCK(_sc) mtx_unlock(&(_sc)->lock) #define CPSW_PORT_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->lock, MA_OWNED) /* * Read/Write macros */ #define cpsw_read_4(_sc, _reg) bus_read_4((_sc)->mem_res, (_reg)) #define cpsw_write_4(_sc, _reg, _val) \ bus_write_4((_sc)->mem_res, (_reg), (_val)) #define cpsw_cpdma_bd_offset(i) (CPSW_CPPI_RAM_OFFSET + ((i)*16)) #define cpsw_cpdma_bd_paddr(sc, slot) \ BUS_SPACE_PHYSADDR(sc->mem_res, slot->bd_offset) #define cpsw_cpdma_read_bd(sc, slot, val) \ bus_read_region_4(sc->mem_res, slot->bd_offset, (uint32_t *) val, 4) #define cpsw_cpdma_write_bd(sc, slot, val) \ bus_write_region_4(sc->mem_res, slot->bd_offset, (uint32_t *) val, 4) #define cpsw_cpdma_write_bd_next(sc, slot, next_slot) \ cpsw_write_4(sc, slot->bd_offset, cpsw_cpdma_bd_paddr(sc, next_slot)) #define cpsw_cpdma_write_bd_flags(sc, slot, val) \ bus_write_2(sc->mem_res, slot->bd_offset + 14, val) #define cpsw_cpdma_read_bd_flags(sc, slot) \ bus_read_2(sc->mem_res, slot->bd_offset + 14) #define cpsw_write_hdp_slot(sc, queue, slot) \ cpsw_write_4(sc, (queue)->hdp_offset, cpsw_cpdma_bd_paddr(sc, slot)) #define CP_OFFSET (CPSW_CPDMA_TX_CP(0) - CPSW_CPDMA_TX_HDP(0)) #define cpsw_read_cp(sc, queue) \ cpsw_read_4(sc, (queue)->hdp_offset + CP_OFFSET) #define cpsw_write_cp(sc, queue, val) \ cpsw_write_4(sc, (queue)->hdp_offset + CP_OFFSET, (val)) #define cpsw_write_cp_slot(sc, queue, slot) \ cpsw_write_cp(sc, queue, cpsw_cpdma_bd_paddr(sc, slot)) #if 0 /* XXX temporary function versions for debugging. */ static void cpsw_write_hdp_slotX(struct cpsw_softc *sc, struct cpsw_queue *queue, struct cpsw_slot *slot) { uint32_t reg = queue->hdp_offset; uint32_t v = cpsw_cpdma_bd_paddr(sc, slot); CPSW_DEBUGF(("HDP <=== 0x%08x (was 0x%08x)", v, cpsw_read_4(sc, reg))); cpsw_write_4(sc, reg, v); } static void cpsw_write_cp_slotX(struct cpsw_softc *sc, struct cpsw_queue *queue, struct cpsw_slot *slot) { uint32_t v = cpsw_cpdma_bd_paddr(sc, slot); CPSW_DEBUGF(("CP <=== 0x%08x (expecting 0x%08x)", v, cpsw_read_cp(sc, queue))); cpsw_write_cp(sc, queue, v); } #endif /* * Expanded dump routines for verbose debugging. */ static void cpsw_dump_slot(struct cpsw_softc *sc, struct cpsw_slot *slot) { static const char *flags[] = {"SOP", "EOP", "Owner", "EOQ", "TDownCmplt", "PassCRC", "Long", "Short", "MacCtl", "Overrun", "PktErr1", "PortEn/PktErr0", "RxVlanEncap", "Port2", "Port1", "Port0"}; struct cpsw_cpdma_bd bd; const char *sep; int i; cpsw_cpdma_read_bd(sc, slot, &bd); printf("BD Addr : 0x%08x Next : 0x%08x\n", cpsw_cpdma_bd_paddr(sc, slot), bd.next); printf(" BufPtr: 0x%08x BufLen: 0x%08x\n", bd.bufptr, bd.buflen); printf(" BufOff: 0x%08x PktLen: 0x%08x\n", bd.bufoff, bd.pktlen); printf(" Flags: "); sep = ""; for (i = 0; i < 16; ++i) { if (bd.flags & (1 << (15 - i))) { printf("%s%s", sep, flags[i]); sep = ","; } } printf("\n"); if (slot->mbuf) { printf(" Ether: %14D\n", (char *)(slot->mbuf->m_data), " "); printf(" Packet: %16D\n", (char *)(slot->mbuf->m_data) + 14, " "); } } #define CPSW_DUMP_SLOT(cs, slot) do { \ IF_DEBUG(sc) { \ cpsw_dump_slot(sc, slot); \ } \ } while (0) static void cpsw_dump_queue(struct cpsw_softc *sc, struct cpsw_slots *q) { struct cpsw_slot *slot; int i = 0; int others = 0; STAILQ_FOREACH(slot, q, next) { if (i > CPSW_TXFRAGS) ++others; else cpsw_dump_slot(sc, slot); ++i; } if (others) printf(" ... and %d more.\n", others); printf("\n"); } #define CPSW_DUMP_QUEUE(sc, q) do { \ IF_DEBUG(sc) { \ cpsw_dump_queue(sc, q); \ } \ } while (0) static void cpsw_init_slots(struct cpsw_softc *sc) { struct cpsw_slot *slot; int i; STAILQ_INIT(&sc->avail); /* Put the slot descriptors onto the global avail list. */ for (i = 0; i < nitems(sc->_slots); i++) { slot = &sc->_slots[i]; slot->bd_offset = cpsw_cpdma_bd_offset(i); STAILQ_INSERT_TAIL(&sc->avail, slot, next); } } static int cpsw_add_slots(struct cpsw_softc *sc, struct cpsw_queue *queue, int requested) { const int max_slots = nitems(sc->_slots); struct cpsw_slot *slot; int i; if (requested < 0) requested = max_slots; for (i = 0; i < requested; ++i) { slot = STAILQ_FIRST(&sc->avail); if (slot == NULL) return (0); if (bus_dmamap_create(sc->mbuf_dtag, 0, &slot->dmamap)) { device_printf(sc->dev, "failed to create dmamap\n"); return (ENOMEM); } STAILQ_REMOVE_HEAD(&sc->avail, next); STAILQ_INSERT_TAIL(&queue->avail, slot, next); ++queue->avail_queue_len; ++queue->queue_slots; } return (0); } static void cpsw_free_slot(struct cpsw_softc *sc, struct cpsw_slot *slot) { int error __diagused; if (slot->dmamap) { if (slot->mbuf) bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap); error = bus_dmamap_destroy(sc->mbuf_dtag, slot->dmamap); KASSERT(error == 0, ("Mapping still active")); slot->dmamap = NULL; } if (slot->mbuf) { m_freem(slot->mbuf); slot->mbuf = NULL; } } static void cpsw_reset(struct cpsw_softc *sc) { int i; callout_stop(&sc->watchdog.callout); /* Reset RMII/RGMII wrapper. */ cpsw_write_4(sc, CPSW_WR_SOFT_RESET, 1); while (cpsw_read_4(sc, CPSW_WR_SOFT_RESET) & 1) ; /* Disable TX and RX interrupts for all cores. */ for (i = 0; i < 3; ++i) { cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(i), 0x00); cpsw_write_4(sc, CPSW_WR_C_TX_EN(i), 0x00); cpsw_write_4(sc, CPSW_WR_C_RX_EN(i), 0x00); cpsw_write_4(sc, CPSW_WR_C_MISC_EN(i), 0x00); } /* Reset CPSW subsystem. */ cpsw_write_4(sc, CPSW_SS_SOFT_RESET, 1); while (cpsw_read_4(sc, CPSW_SS_SOFT_RESET) & 1) ; /* Reset Sliver port 1 and 2 */ for (i = 0; i < 2; i++) { /* Reset */ cpsw_write_4(sc, CPSW_SL_SOFT_RESET(i), 1); while (cpsw_read_4(sc, CPSW_SL_SOFT_RESET(i)) & 1) ; } /* Reset DMA controller. */ cpsw_write_4(sc, CPSW_CPDMA_SOFT_RESET, 1); while (cpsw_read_4(sc, CPSW_CPDMA_SOFT_RESET) & 1) ; /* Disable TX & RX DMA */ cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 0); cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 0); /* Clear all queues. */ for (i = 0; i < 8; i++) { cpsw_write_4(sc, CPSW_CPDMA_TX_HDP(i), 0); cpsw_write_4(sc, CPSW_CPDMA_RX_HDP(i), 0); cpsw_write_4(sc, CPSW_CPDMA_TX_CP(i), 0); cpsw_write_4(sc, CPSW_CPDMA_RX_CP(i), 0); } /* Clear all interrupt Masks */ cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_CLEAR, 0xFFFFFFFF); cpsw_write_4(sc, CPSW_CPDMA_TX_INTMASK_CLEAR, 0xFFFFFFFF); } static void cpsw_init(struct cpsw_softc *sc) { struct cpsw_slot *slot; uint32_t reg; /* Disable the interrupt pacing. */ reg = cpsw_read_4(sc, CPSW_WR_INT_CONTROL); reg &= ~(CPSW_WR_INT_PACE_EN | CPSW_WR_INT_PRESCALE_MASK); cpsw_write_4(sc, CPSW_WR_INT_CONTROL, reg); /* Clear ALE */ cpsw_write_4(sc, CPSW_ALE_CONTROL, CPSW_ALE_CTL_CLEAR_TBL); /* Enable ALE */ reg = CPSW_ALE_CTL_ENABLE; if (sc->dualemac) reg |= CPSW_ALE_CTL_VLAN_AWARE; cpsw_write_4(sc, CPSW_ALE_CONTROL, reg); /* Set Host Port Mapping. */ cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_TX_PRI_MAP, 0x76543210); cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_RX_CH_MAP, 0); /* Initialize ALE: set host port to forwarding(3). */ cpsw_write_4(sc, CPSW_ALE_PORTCTL(0), ALE_PORTCTL_INGRESS | ALE_PORTCTL_FORWARD); cpsw_write_4(sc, CPSW_SS_PTYPE, 0); /* Enable statistics for ports 0, 1 and 2 */ cpsw_write_4(sc, CPSW_SS_STAT_PORT_EN, 7); /* Turn off flow control. */ cpsw_write_4(sc, CPSW_SS_FLOW_CONTROL, 0); /* Make IP hdr aligned with 4 */ cpsw_write_4(sc, CPSW_CPDMA_RX_BUFFER_OFFSET, 2); /* Initialize RX Buffer Descriptors */ cpsw_write_4(sc, CPSW_CPDMA_RX_PENDTHRESH(0), 0); cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), 0); /* Enable TX & RX DMA */ cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 1); cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 1); /* Enable Interrupts for core 0 */ cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(0), 0xFF); cpsw_write_4(sc, CPSW_WR_C_RX_EN(0), 0xFF); cpsw_write_4(sc, CPSW_WR_C_TX_EN(0), 0xFF); cpsw_write_4(sc, CPSW_WR_C_MISC_EN(0), 0x1F); /* Enable host Error Interrupt */ cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_SET, 3); /* Enable interrupts for RX and TX on Channel 0 */ cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_SET, CPSW_CPDMA_RX_INT(0) | CPSW_CPDMA_RX_INT_THRESH(0)); cpsw_write_4(sc, CPSW_CPDMA_TX_INTMASK_SET, 1); /* Initialze MDIO - ENABLE, PREAMBLE=0, FAULTENB, CLKDIV=0xFF */ /* TODO Calculate MDCLK=CLK/(CLKDIV+1) */ cpsw_write_4(sc, MDIOCONTROL, MDIOCTL_ENABLE | MDIOCTL_FAULTENB | 0xff); /* Select MII in GMII_SEL, Internal Delay mode */ //ti_scm_reg_write_4(0x650, 0); /* Initialize active queues. */ slot = STAILQ_FIRST(&sc->tx.active); if (slot != NULL) cpsw_write_hdp_slot(sc, &sc->tx, slot); slot = STAILQ_FIRST(&sc->rx.active); if (slot != NULL) cpsw_write_hdp_slot(sc, &sc->rx, slot); cpsw_rx_enqueue(sc); cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), sc->rx.active_queue_len); cpsw_write_4(sc, CPSW_CPDMA_RX_PENDTHRESH(0), CPSW_TXFRAGS); /* Activate network interface. */ sc->rx.running = 1; sc->tx.running = 1; sc->watchdog.timer = 0; callout_init(&sc->watchdog.callout, 0); callout_reset(&sc->watchdog.callout, hz, cpsw_tx_watchdog, sc); } /* * * Device Probe, Attach, Detach. * */ static int cpsw_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "ti,cpsw")) return (ENXIO); device_set_desc(dev, "3-port Switch Ethernet Subsystem"); return (BUS_PROBE_DEFAULT); } static int cpsw_intr_attach(struct cpsw_softc *sc) { int i; for (i = 0; i < CPSW_INTR_COUNT; i++) { if (bus_setup_intr(sc->dev, sc->irq_res[i], INTR_TYPE_NET | INTR_MPSAFE, NULL, cpsw_intr_cb[i].cb, sc, &sc->ih_cookie[i]) != 0) { return (-1); } } return (0); } static void cpsw_intr_detach(struct cpsw_softc *sc) { int i; for (i = 0; i < CPSW_INTR_COUNT; i++) { if (sc->ih_cookie[i]) { bus_teardown_intr(sc->dev, sc->irq_res[i], sc->ih_cookie[i]); } } } static int cpsw_get_fdt_data(struct cpsw_softc *sc, int port) { char *name; int len, phy, vlan; pcell_t phy_id[3], vlan_id; phandle_t child; unsigned long mdio_child_addr; /* Find any slave with phy-handle/phy_id */ phy = -1; vlan = -1; for (child = OF_child(sc->node); child != 0; child = OF_peer(child)) { if (OF_getprop_alloc(child, "name", (void **)&name) < 0) continue; if (sscanf(name, "slave@%lx", &mdio_child_addr) != 1) { OF_prop_free(name); continue; } OF_prop_free(name); if (mdio_child_addr != slave_mdio_addr[port] && mdio_child_addr != (slave_mdio_addr[port] & 0xFFF)) continue; if (fdt_get_phyaddr(child, NULL, &phy, NULL) != 0){ /* Users with old DTB will have phy_id instead */ phy = -1; len = OF_getproplen(child, "phy_id"); if (len / sizeof(pcell_t) == 2) { /* Get phy address from fdt */ if (OF_getencprop(child, "phy_id", phy_id, len) > 0) phy = phy_id[1]; } } len = OF_getproplen(child, "dual_emac_res_vlan"); if (len / sizeof(pcell_t) == 1) { /* Get phy address from fdt */ if (OF_getencprop(child, "dual_emac_res_vlan", &vlan_id, len) > 0) { vlan = vlan_id; } } break; } if (phy == -1) return (ENXIO); sc->port[port].phy = phy; sc->port[port].vlan = vlan; return (0); } static int cpsw_attach(device_t dev) { int error, i; struct cpsw_softc *sc; uint32_t reg; sc = device_get_softc(dev); sc->dev = dev; sc->node = ofw_bus_get_node(dev); getbinuptime(&sc->attach_uptime); if (OF_getencprop(sc->node, "active_slave", &sc->active_slave, sizeof(sc->active_slave)) <= 0) { sc->active_slave = 0; } if (sc->active_slave > 1) sc->active_slave = 1; if (OF_hasprop(sc->node, "dual_emac")) sc->dualemac = 1; for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; if (cpsw_get_fdt_data(sc, i) != 0) { device_printf(dev, "failed to get PHY address from FDT\n"); return (ENXIO); } } /* Initialize mutexes */ mtx_init(&sc->tx.lock, device_get_nameunit(dev), "cpsw TX lock", MTX_DEF); mtx_init(&sc->rx.lock, device_get_nameunit(dev), "cpsw RX lock", MTX_DEF); /* Allocate IRQ resources */ error = bus_alloc_resources(dev, irq_res_spec, sc->irq_res); if (error) { device_printf(dev, "could not allocate IRQ resources\n"); cpsw_detach(dev); return (ENXIO); } sc->mem_rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(sc->dev, "failed to allocate memory resource\n"); cpsw_detach(dev); return (ENXIO); } reg = cpsw_read_4(sc, CPSW_SS_IDVER); device_printf(dev, "CPSW SS Version %d.%d (%d)\n", (reg >> 8 & 0x7), reg & 0xFF, (reg >> 11) & 0x1F); cpsw_add_sysctls(sc); /* Allocate a busdma tag and DMA safe memory for mbufs. */ error = bus_dma_tag_create( bus_get_dma_tag(sc->dev), /* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES, CPSW_TXFRAGS, /* maxsize, nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->mbuf_dtag); /* dmatag */ if (error) { device_printf(dev, "bus_dma_tag_create failed\n"); cpsw_detach(dev); return (error); } /* Allocate a NULL buffer for padding. */ sc->nullpad = malloc(ETHER_MIN_LEN, M_DEVBUF, M_WAITOK | M_ZERO); cpsw_init_slots(sc); /* Allocate slots to TX and RX queues. */ STAILQ_INIT(&sc->rx.avail); STAILQ_INIT(&sc->rx.active); STAILQ_INIT(&sc->tx.avail); STAILQ_INIT(&sc->tx.active); // For now: 128 slots to TX, rest to RX. // XXX TODO: start with 32/64 and grow dynamically based on demand. if (cpsw_add_slots(sc, &sc->tx, 128) || cpsw_add_slots(sc, &sc->rx, -1)) { device_printf(dev, "failed to allocate dmamaps\n"); cpsw_detach(dev); return (ENOMEM); } device_printf(dev, "Initial queue size TX=%d RX=%d\n", sc->tx.queue_slots, sc->rx.queue_slots); sc->tx.hdp_offset = CPSW_CPDMA_TX_HDP(0); sc->rx.hdp_offset = CPSW_CPDMA_RX_HDP(0); if (cpsw_intr_attach(sc) == -1) { device_printf(dev, "failed to setup interrupts\n"); cpsw_detach(dev); return (ENXIO); } #ifdef CPSW_ETHERSWITCH for (i = 0; i < CPSW_VLANS; i++) cpsw_vgroups[i].vid = -1; #endif /* Reset the controller. */ cpsw_reset(sc); cpsw_init(sc); for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; sc->port[i].dev = device_add_child(dev, "cpsw", i); if (sc->port[i].dev == NULL) { cpsw_detach(dev); return (ENXIO); } } bus_identify_children(dev); bus_attach_children(dev); return (0); } static int cpsw_detach(device_t dev) { struct cpsw_softc *sc; int error, i; error = bus_generic_detach(dev); if (error != 0) return (error); sc = device_get_softc(dev); if (device_is_attached(dev)) { callout_stop(&sc->watchdog.callout); callout_drain(&sc->watchdog.callout); } /* Stop and release all interrupts */ cpsw_intr_detach(sc); /* Free dmamaps and mbufs */ for (i = 0; i < nitems(sc->_slots); ++i) cpsw_free_slot(sc, &sc->_slots[i]); /* Free null padding buffer. */ if (sc->nullpad) free(sc->nullpad, M_DEVBUF); /* Free DMA tag */ if (sc->mbuf_dtag) { error = bus_dma_tag_destroy(sc->mbuf_dtag); KASSERT(error == 0, ("Unable to destroy DMA tag")); } /* Free IO memory handler */ if (sc->mem_res != NULL) bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); bus_release_resources(dev, irq_res_spec, sc->irq_res); /* Destroy mutexes */ mtx_destroy(&sc->rx.lock); mtx_destroy(&sc->tx.lock); /* Detach the switch device, if present. */ error = bus_generic_detach(dev); if (error != 0) return (error); - return (device_delete_children(dev)); + return (0); } static phandle_t cpsw_get_node(device_t bus, device_t dev) { /* Share controller node with port device. */ return (ofw_bus_get_node(bus)); } static int cpswp_probe(device_t dev) { if (device_get_unit(dev) > 1) { device_printf(dev, "Only two ports are supported.\n"); return (ENXIO); } device_set_desc(dev, "Ethernet Switch Port"); return (BUS_PROBE_DEFAULT); } static int cpswp_attach(device_t dev) { int error; if_t ifp; struct cpswp_softc *sc; uint32_t reg; uint8_t mac_addr[ETHER_ADDR_LEN]; phandle_t opp_table; struct syscon *syscon; sc = device_get_softc(dev); sc->dev = dev; sc->pdev = device_get_parent(dev); sc->swsc = device_get_softc(sc->pdev); sc->unit = device_get_unit(dev); sc->phy = sc->swsc->port[sc->unit].phy; sc->vlan = sc->swsc->port[sc->unit].vlan; if (sc->swsc->dualemac && sc->vlan == -1) sc->vlan = sc->unit + 1; if (sc->unit == 0) { sc->physel = MDIOUSERPHYSEL0; sc->phyaccess = MDIOUSERACCESS0; } else { sc->physel = MDIOUSERPHYSEL1; sc->phyaccess = MDIOUSERACCESS1; } mtx_init(&sc->lock, device_get_nameunit(dev), "cpsw port lock", MTX_DEF); /* Allocate network interface */ ifp = sc->ifp = if_alloc(IFT_ETHER); if_initname(ifp, device_get_name(sc->dev), sc->unit); if_setsoftc(ifp, sc); if_setflags(ifp, IFF_SIMPLEX | IFF_MULTICAST | IFF_BROADCAST); if_setcapenable(ifp, if_getcapabilities(ifp)); if_setinitfn(ifp, cpswp_init); if_setstartfn(ifp, cpswp_start); if_setioctlfn(ifp, cpswp_ioctl); if_setsendqlen(ifp, sc->swsc->tx.queue_slots); if_setsendqready(ifp); /* FIXME: For now; Go and kidnap syscon from opp-table */ /* ti,cpsw actually have an optional syscon reference but only for am33xx?? */ opp_table = OF_finddevice("/opp-table"); if (opp_table == -1) { device_printf(dev, "Cant find /opp-table\n"); cpswp_detach(dev); return (ENXIO); } if (!OF_hasprop(opp_table, "syscon")) { device_printf(dev, "/opp-table doesnt have required syscon property\n"); cpswp_detach(dev); return (ENXIO); } if (syscon_get_by_ofw_property(dev, opp_table, "syscon", &syscon) != 0) { device_printf(dev, "Failed to get syscon\n"); cpswp_detach(dev); return (ENXIO); } /* Get high part of MAC address from control module (mac_id[0|1]_hi) */ reg = SYSCON_READ_4(syscon, SCM_MAC_ID0_HI + sc->unit * 8); mac_addr[0] = reg & 0xFF; mac_addr[1] = (reg >> 8) & 0xFF; mac_addr[2] = (reg >> 16) & 0xFF; mac_addr[3] = (reg >> 24) & 0xFF; /* Get low part of MAC address from control module (mac_id[0|1]_lo) */ reg = SYSCON_READ_4(syscon, SCM_MAC_ID0_LO + sc->unit * 8); mac_addr[4] = reg & 0xFF; mac_addr[5] = (reg >> 8) & 0xFF; error = mii_attach(dev, &sc->miibus, ifp, cpswp_ifmedia_upd, cpswp_ifmedia_sts, BMSR_DEFCAPMASK, sc->phy, MII_OFFSET_ANY, 0); if (error) { device_printf(dev, "attaching PHYs failed\n"); cpswp_detach(dev); return (error); } sc->mii = device_get_softc(sc->miibus); /* Select PHY and enable interrupts */ cpsw_write_4(sc->swsc, sc->physel, MDIO_PHYSEL_LINKINTENB | (sc->phy & 0x1F)); ether_ifattach(sc->ifp, mac_addr); callout_init(&sc->mii_callout, 0); return (0); } static int cpswp_detach(device_t dev) { struct cpswp_softc *sc; sc = device_get_softc(dev); CPSW_DEBUGF(sc->swsc, ("")); if (device_is_attached(dev)) { ether_ifdetach(sc->ifp); CPSW_PORT_LOCK(sc); cpswp_stop_locked(sc); CPSW_PORT_UNLOCK(sc); callout_drain(&sc->mii_callout); } bus_generic_detach(dev); if_free(sc->ifp); mtx_destroy(&sc->lock); return (0); } /* * * Init/Shutdown. * */ static int cpsw_ports_down(struct cpsw_softc *sc) { struct cpswp_softc *psc; if_t ifp1, ifp2; if (!sc->dualemac) return (1); psc = device_get_softc(sc->port[0].dev); ifp1 = psc->ifp; psc = device_get_softc(sc->port[1].dev); ifp2 = psc->ifp; if ((if_getflags(ifp1) & IFF_UP) == 0 && (if_getflags(ifp2) & IFF_UP) == 0) return (1); return (0); } static void cpswp_init(void *arg) { struct cpswp_softc *sc = arg; CPSW_DEBUGF(sc->swsc, ("")); CPSW_PORT_LOCK(sc); cpswp_init_locked(arg); CPSW_PORT_UNLOCK(sc); } static void cpswp_init_locked(void *arg) { #ifdef CPSW_ETHERSWITCH int i; #endif struct cpswp_softc *sc = arg; if_t ifp; uint32_t reg; CPSW_DEBUGF(sc->swsc, ("")); CPSW_PORT_LOCK_ASSERT(sc); ifp = sc->ifp; if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) return; getbinuptime(&sc->init_uptime); if (!sc->swsc->rx.running && !sc->swsc->tx.running) { /* Reset the controller. */ cpsw_reset(sc->swsc); cpsw_init(sc->swsc); } /* Set Slave Mapping. */ cpsw_write_4(sc->swsc, CPSW_SL_RX_PRI_MAP(sc->unit), 0x76543210); cpsw_write_4(sc->swsc, CPSW_PORT_P_TX_PRI_MAP(sc->unit + 1), 0x33221100); cpsw_write_4(sc->swsc, CPSW_SL_RX_MAXLEN(sc->unit), 0x5f2); /* Enable MAC RX/TX modules. */ /* TODO: Docs claim that IFCTL_B and IFCTL_A do the same thing? */ /* Huh? Docs call bit 0 "Loopback" some places, "FullDuplex" others. */ reg = cpsw_read_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit)); reg |= CPSW_SL_MACTL_GMII_ENABLE; cpsw_write_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit), reg); /* Initialize ALE: set port to forwarding, initialize addrs */ cpsw_write_4(sc->swsc, CPSW_ALE_PORTCTL(sc->unit + 1), ALE_PORTCTL_INGRESS | ALE_PORTCTL_FORWARD); cpswp_ale_update_addresses(sc, 1); if (sc->swsc->dualemac) { /* Set Port VID. */ cpsw_write_4(sc->swsc, CPSW_PORT_P_VLAN(sc->unit + 1), sc->vlan & 0xfff); cpsw_ale_update_vlan_table(sc->swsc, sc->vlan, (1 << (sc->unit + 1)) | (1 << 0), /* Member list */ (1 << (sc->unit + 1)) | (1 << 0), /* Untagged egress */ (1 << (sc->unit + 1)) | (1 << 0), 0); /* mcast reg flood */ #ifdef CPSW_ETHERSWITCH for (i = 0; i < CPSW_VLANS; i++) { if (cpsw_vgroups[i].vid != -1) continue; cpsw_vgroups[i].vid = sc->vlan; break; } #endif } mii_mediachg(sc->mii); callout_reset(&sc->mii_callout, hz, cpswp_tick, sc); if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); } static int cpsw_shutdown(device_t dev) { struct cpsw_softc *sc; struct cpswp_softc *psc; int i; sc = device_get_softc(dev); CPSW_DEBUGF(sc, ("")); for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; psc = device_get_softc(sc->port[i].dev); CPSW_PORT_LOCK(psc); cpswp_stop_locked(psc); CPSW_PORT_UNLOCK(psc); } return (0); } static void cpsw_rx_teardown(struct cpsw_softc *sc) { int i = 0; CPSW_RX_LOCK(sc); CPSW_DEBUGF(sc, ("starting RX teardown")); sc->rx.teardown = 1; cpsw_write_4(sc, CPSW_CPDMA_RX_TEARDOWN, 0); CPSW_RX_UNLOCK(sc); while (sc->rx.running) { if (++i > 10) { device_printf(sc->dev, "Unable to cleanly shutdown receiver\n"); return; } DELAY(200); } if (!sc->rx.running) CPSW_DEBUGF(sc, ("finished RX teardown (%d retries)", i)); } static void cpsw_tx_teardown(struct cpsw_softc *sc) { int i = 0; CPSW_TX_LOCK(sc); CPSW_DEBUGF(sc, ("starting TX teardown")); /* Start the TX queue teardown if queue is not empty. */ if (STAILQ_FIRST(&sc->tx.active) != NULL) cpsw_write_4(sc, CPSW_CPDMA_TX_TEARDOWN, 0); else sc->tx.teardown = 1; cpsw_tx_dequeue(sc); while (sc->tx.running && ++i < 10) { DELAY(200); cpsw_tx_dequeue(sc); } if (sc->tx.running) { device_printf(sc->dev, "Unable to cleanly shutdown transmitter\n"); } CPSW_DEBUGF(sc, ("finished TX teardown (%d retries, %d idle buffers)", i, sc->tx.active_queue_len)); CPSW_TX_UNLOCK(sc); } static void cpswp_stop_locked(struct cpswp_softc *sc) { if_t ifp; uint32_t reg; ifp = sc->ifp; CPSW_DEBUGF(sc->swsc, ("")); CPSW_PORT_LOCK_ASSERT(sc); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) return; /* Disable interface */ if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); /* Stop ticker */ callout_stop(&sc->mii_callout); /* Tear down the RX/TX queues. */ if (cpsw_ports_down(sc->swsc)) { cpsw_rx_teardown(sc->swsc); cpsw_tx_teardown(sc->swsc); } /* Stop MAC RX/TX modules. */ reg = cpsw_read_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit)); reg &= ~CPSW_SL_MACTL_GMII_ENABLE; cpsw_write_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit), reg); if (cpsw_ports_down(sc->swsc)) { /* Capture stats before we reset controller. */ cpsw_stats_collect(sc->swsc); cpsw_reset(sc->swsc); cpsw_init(sc->swsc); } } /* * Suspend/Resume. */ static int cpsw_suspend(device_t dev) { struct cpsw_softc *sc; struct cpswp_softc *psc; int i; sc = device_get_softc(dev); CPSW_DEBUGF(sc, ("")); for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; psc = device_get_softc(sc->port[i].dev); CPSW_PORT_LOCK(psc); cpswp_stop_locked(psc); CPSW_PORT_UNLOCK(psc); } return (0); } static int cpsw_resume(device_t dev) { struct cpsw_softc *sc; sc = device_get_softc(dev); CPSW_DEBUGF(sc, ("UNIMPLEMENTED")); return (0); } /* * * IOCTL * */ static void cpsw_set_promisc(struct cpswp_softc *sc, int set) { uint32_t reg; /* * Enabling promiscuous mode requires ALE_BYPASS to be enabled. * That disables the ALE forwarding logic and causes every * packet to be sent only to the host port. In bypass mode, * the ALE processes host port transmit packets the same as in * normal mode. */ reg = cpsw_read_4(sc->swsc, CPSW_ALE_CONTROL); reg &= ~CPSW_ALE_CTL_BYPASS; if (set) reg |= CPSW_ALE_CTL_BYPASS; cpsw_write_4(sc->swsc, CPSW_ALE_CONTROL, reg); } static void cpsw_set_allmulti(struct cpswp_softc *sc, int set) { if (set) { printf("All-multicast mode unimplemented\n"); } } static int cpswp_ioctl(if_t ifp, u_long command, caddr_t data) { struct cpswp_softc *sc; struct ifreq *ifr; int error; uint32_t changed; error = 0; sc = if_getsoftc(ifp); ifr = (struct ifreq *)data; switch (command) { case SIOCSIFCAP: changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; if (changed & IFCAP_HWCSUM) { if ((ifr->ifr_reqcap & changed) & IFCAP_HWCSUM) if_setcapenablebit(ifp, IFCAP_HWCSUM, 0); else if_setcapenablebit(ifp, 0, IFCAP_HWCSUM); } error = 0; break; case SIOCSIFFLAGS: CPSW_PORT_LOCK(sc); if (if_getflags(ifp) & IFF_UP) { if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { changed = if_getflags(ifp) ^ sc->if_flags; CPSW_DEBUGF(sc->swsc, ("SIOCSIFFLAGS: UP & RUNNING (changed=0x%x)", changed)); if (changed & IFF_PROMISC) cpsw_set_promisc(sc, if_getflags(ifp) & IFF_PROMISC); if (changed & IFF_ALLMULTI) cpsw_set_allmulti(sc, if_getflags(ifp) & IFF_ALLMULTI); } else { CPSW_DEBUGF(sc->swsc, ("SIOCSIFFLAGS: starting up")); cpswp_init_locked(sc); } } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { CPSW_DEBUGF(sc->swsc, ("SIOCSIFFLAGS: shutting down")); cpswp_stop_locked(sc); } sc->if_flags = if_getflags(ifp); CPSW_PORT_UNLOCK(sc); break; case SIOCADDMULTI: cpswp_ale_update_addresses(sc, 0); break; case SIOCDELMULTI: /* Ugh. DELMULTI doesn't provide the specific address being removed, so the best we can do is remove everything and rebuild it all. */ cpswp_ale_update_addresses(sc, 1); break; case SIOCGIFMEDIA: case SIOCSIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, command); break; default: error = ether_ioctl(ifp, command, data); } return (error); } /* * * MIIBUS * */ static int cpswp_miibus_ready(struct cpsw_softc *sc, uint32_t reg) { uint32_t r, retries = CPSW_MIIBUS_RETRIES; while (--retries) { r = cpsw_read_4(sc, reg); if ((r & MDIO_PHYACCESS_GO) == 0) return (1); DELAY(CPSW_MIIBUS_DELAY); } return (0); } static int cpswp_miibus_readreg(device_t dev, int phy, int reg) { struct cpswp_softc *sc; uint32_t cmd, r; sc = device_get_softc(dev); if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) { device_printf(dev, "MDIO not ready to read\n"); return (0); } /* Set GO, reg, phy */ cmd = MDIO_PHYACCESS_GO | (reg & 0x1F) << 21 | (phy & 0x1F) << 16; cpsw_write_4(sc->swsc, sc->phyaccess, cmd); if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) { device_printf(dev, "MDIO timed out during read\n"); return (0); } r = cpsw_read_4(sc->swsc, sc->phyaccess); if ((r & MDIO_PHYACCESS_ACK) == 0) { device_printf(dev, "Failed to read from PHY.\n"); r = 0; } return (r & 0xFFFF); } static int cpswp_miibus_writereg(device_t dev, int phy, int reg, int value) { struct cpswp_softc *sc; uint32_t cmd; sc = device_get_softc(dev); if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) { device_printf(dev, "MDIO not ready to write\n"); return (0); } /* Set GO, WRITE, reg, phy, and value */ cmd = MDIO_PHYACCESS_GO | MDIO_PHYACCESS_WRITE | (reg & 0x1F) << 21 | (phy & 0x1F) << 16 | (value & 0xFFFF); cpsw_write_4(sc->swsc, sc->phyaccess, cmd); if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) { device_printf(dev, "MDIO timed out during write\n"); return (0); } return (0); } static void cpswp_miibus_statchg(device_t dev) { struct cpswp_softc *sc; uint32_t mac_control, reg; sc = device_get_softc(dev); CPSW_DEBUGF(sc->swsc, ("")); reg = CPSW_SL_MACCONTROL(sc->unit); mac_control = cpsw_read_4(sc->swsc, reg); mac_control &= ~(CPSW_SL_MACTL_GIG | CPSW_SL_MACTL_IFCTL_A | CPSW_SL_MACTL_IFCTL_B | CPSW_SL_MACTL_FULLDUPLEX); switch(IFM_SUBTYPE(sc->mii->mii_media_active)) { case IFM_1000_SX: case IFM_1000_LX: case IFM_1000_CX: case IFM_1000_T: mac_control |= CPSW_SL_MACTL_GIG; break; case IFM_100_TX: mac_control |= CPSW_SL_MACTL_IFCTL_A; break; } if (sc->mii->mii_media_active & IFM_FDX) mac_control |= CPSW_SL_MACTL_FULLDUPLEX; cpsw_write_4(sc->swsc, reg, mac_control); } /* * * Transmit/Receive Packets. * */ static void cpsw_intr_rx(void *arg) { struct cpsw_softc *sc; if_t ifp; struct mbuf *received, *next; sc = (struct cpsw_softc *)arg; CPSW_RX_LOCK(sc); if (sc->rx.teardown) { sc->rx.running = 0; sc->rx.teardown = 0; cpsw_write_cp(sc, &sc->rx, 0xfffffffc); } received = cpsw_rx_dequeue(sc); cpsw_rx_enqueue(sc); cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 1); CPSW_RX_UNLOCK(sc); while (received != NULL) { next = received->m_nextpkt; received->m_nextpkt = NULL; ifp = received->m_pkthdr.rcvif; if_input(ifp, received); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); received = next; } } static struct mbuf * cpsw_rx_dequeue(struct cpsw_softc *sc) { int nsegs, port, removed; struct cpsw_cpdma_bd bd; struct cpsw_slot *last, *slot; struct cpswp_softc *psc; struct mbuf *m, *m0, *mb_head, *mb_tail; uint16_t m0_flags; nsegs = 0; m0 = NULL; last = NULL; mb_head = NULL; mb_tail = NULL; removed = 0; /* Pull completed packets off hardware RX queue. */ while ((slot = STAILQ_FIRST(&sc->rx.active)) != NULL) { cpsw_cpdma_read_bd(sc, slot, &bd); /* * Stop on packets still in use by hardware, but do not stop * on packets with the teardown complete flag, they will be * discarded later. */ if ((bd.flags & (CPDMA_BD_OWNER | CPDMA_BD_TDOWNCMPLT)) == CPDMA_BD_OWNER) break; last = slot; ++removed; STAILQ_REMOVE_HEAD(&sc->rx.active, next); STAILQ_INSERT_TAIL(&sc->rx.avail, slot, next); bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap); m = slot->mbuf; slot->mbuf = NULL; if (bd.flags & CPDMA_BD_TDOWNCMPLT) { CPSW_DEBUGF(sc, ("RX teardown is complete")); m_freem(m); sc->rx.running = 0; sc->rx.teardown = 0; break; } port = (bd.flags & CPDMA_BD_PORT_MASK) - 1; KASSERT(port >= 0 && port <= 1, ("patcket received with invalid port: %d", port)); psc = device_get_softc(sc->port[port].dev); /* Set up mbuf */ m->m_data += bd.bufoff; m->m_len = bd.buflen; if (bd.flags & CPDMA_BD_SOP) { m->m_pkthdr.len = bd.pktlen; m->m_pkthdr.rcvif = psc->ifp; m->m_flags |= M_PKTHDR; m0_flags = bd.flags; m0 = m; } nsegs++; m->m_next = NULL; m->m_nextpkt = NULL; if (bd.flags & CPDMA_BD_EOP && m0 != NULL) { if (m0_flags & CPDMA_BD_PASS_CRC) m_adj(m0, -ETHER_CRC_LEN); m0_flags = 0; m0 = NULL; if (nsegs > sc->rx.longest_chain) sc->rx.longest_chain = nsegs; nsegs = 0; } if ((if_getcapenable(psc->ifp) & IFCAP_RXCSUM) != 0) { /* check for valid CRC by looking into pkt_err[5:4] */ if ((bd.flags & (CPDMA_BD_SOP | CPDMA_BD_PKT_ERR_MASK)) == CPDMA_BD_SOP) { m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; m->m_pkthdr.csum_flags |= CSUM_IP_VALID; m->m_pkthdr.csum_data = 0xffff; } } if (STAILQ_FIRST(&sc->rx.active) != NULL && (bd.flags & (CPDMA_BD_EOP | CPDMA_BD_EOQ)) == (CPDMA_BD_EOP | CPDMA_BD_EOQ)) { cpsw_write_hdp_slot(sc, &sc->rx, STAILQ_FIRST(&sc->rx.active)); sc->rx.queue_restart++; } /* Add mbuf to packet list to be returned. */ if (mb_tail != NULL && (bd.flags & CPDMA_BD_SOP)) { mb_tail->m_nextpkt = m; } else if (mb_tail != NULL) { mb_tail->m_next = m; } else if (mb_tail == NULL && (bd.flags & CPDMA_BD_SOP) == 0) { if (bootverbose) printf( "%s: %s: discanding fragment packet w/o header\n", __func__, if_name(psc->ifp)); m_freem(m); continue; } else { mb_head = m; } mb_tail = m; } if (removed != 0) { cpsw_write_cp_slot(sc, &sc->rx, last); sc->rx.queue_removes += removed; sc->rx.avail_queue_len += removed; sc->rx.active_queue_len -= removed; if (sc->rx.avail_queue_len > sc->rx.max_avail_queue_len) sc->rx.max_avail_queue_len = sc->rx.avail_queue_len; CPSW_DEBUGF(sc, ("Removed %d received packet(s) from RX queue", removed)); } return (mb_head); } static void cpsw_rx_enqueue(struct cpsw_softc *sc) { bus_dma_segment_t seg[1]; struct cpsw_cpdma_bd bd; struct cpsw_slot *first_new_slot, *last_old_slot, *next, *slot; int error, nsegs, added = 0; /* Register new mbufs with hardware. */ first_new_slot = NULL; last_old_slot = STAILQ_LAST(&sc->rx.active, cpsw_slot, next); while ((slot = STAILQ_FIRST(&sc->rx.avail)) != NULL) { if (first_new_slot == NULL) first_new_slot = slot; if (slot->mbuf == NULL) { slot->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (slot->mbuf == NULL) { device_printf(sc->dev, "Unable to fill RX queue\n"); break; } slot->mbuf->m_len = slot->mbuf->m_pkthdr.len = slot->mbuf->m_ext.ext_size; } error = bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, slot->dmamap, slot->mbuf, seg, &nsegs, BUS_DMA_NOWAIT); KASSERT(nsegs == 1, ("More than one segment (nsegs=%d)", nsegs)); KASSERT(error == 0, ("DMA error (error=%d)", error)); if (error != 0 || nsegs != 1) { device_printf(sc->dev, "%s: Can't prep RX buf for DMA (nsegs=%d, error=%d)\n", __func__, nsegs, error); bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap); m_freem(slot->mbuf); slot->mbuf = NULL; break; } bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_PREREAD); /* Create and submit new rx descriptor. */ if ((next = STAILQ_NEXT(slot, next)) != NULL) bd.next = cpsw_cpdma_bd_paddr(sc, next); else bd.next = 0; bd.bufptr = seg->ds_addr; bd.bufoff = 0; bd.buflen = MCLBYTES - 1; bd.pktlen = bd.buflen; bd.flags = CPDMA_BD_OWNER; cpsw_cpdma_write_bd(sc, slot, &bd); ++added; STAILQ_REMOVE_HEAD(&sc->rx.avail, next); STAILQ_INSERT_TAIL(&sc->rx.active, slot, next); } if (added == 0 || first_new_slot == NULL) return; CPSW_DEBUGF(sc, ("Adding %d buffers to RX queue", added)); /* Link new entries to hardware RX queue. */ if (last_old_slot == NULL) { /* Start a fresh queue. */ cpsw_write_hdp_slot(sc, &sc->rx, first_new_slot); } else { /* Add buffers to end of current queue. */ cpsw_cpdma_write_bd_next(sc, last_old_slot, first_new_slot); } sc->rx.queue_adds += added; sc->rx.avail_queue_len -= added; sc->rx.active_queue_len += added; cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), added); if (sc->rx.active_queue_len > sc->rx.max_active_queue_len) sc->rx.max_active_queue_len = sc->rx.active_queue_len; } static void cpswp_start(if_t ifp) { struct cpswp_softc *sc; sc = if_getsoftc(ifp); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || sc->swsc->tx.running == 0) { return; } CPSW_TX_LOCK(sc->swsc); cpswp_tx_enqueue(sc); cpsw_tx_dequeue(sc->swsc); CPSW_TX_UNLOCK(sc->swsc); } static void cpsw_intr_tx(void *arg) { struct cpsw_softc *sc; sc = (struct cpsw_softc *)arg; CPSW_TX_LOCK(sc); if (cpsw_read_4(sc, CPSW_CPDMA_TX_CP(0)) == 0xfffffffc) cpsw_write_cp(sc, &sc->tx, 0xfffffffc); cpsw_tx_dequeue(sc); cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 2); CPSW_TX_UNLOCK(sc); } static void cpswp_tx_enqueue(struct cpswp_softc *sc) { bus_dma_segment_t segs[CPSW_TXFRAGS]; struct cpsw_cpdma_bd bd; struct cpsw_slot *first_new_slot, *last, *last_old_slot, *next, *slot; struct mbuf *m0; int error, nsegs, seg, added = 0, padlen; /* Pull pending packets from IF queue and prep them for DMA. */ last = NULL; first_new_slot = NULL; last_old_slot = STAILQ_LAST(&sc->swsc->tx.active, cpsw_slot, next); while ((slot = STAILQ_FIRST(&sc->swsc->tx.avail)) != NULL) { m0 = if_dequeue(sc->ifp); if (m0 == NULL) break; slot->mbuf = m0; padlen = ETHER_MIN_LEN - ETHER_CRC_LEN - m0->m_pkthdr.len; if (padlen < 0) padlen = 0; else if (padlen > 0) m_append(slot->mbuf, padlen, sc->swsc->nullpad); /* Create mapping in DMA memory */ error = bus_dmamap_load_mbuf_sg(sc->swsc->mbuf_dtag, slot->dmamap, slot->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); /* If the packet is too fragmented, try to simplify. */ if (error == EFBIG || (error == 0 && nsegs > sc->swsc->tx.avail_queue_len)) { bus_dmamap_unload(sc->swsc->mbuf_dtag, slot->dmamap); m0 = m_defrag(slot->mbuf, M_NOWAIT); if (m0 == NULL) { device_printf(sc->dev, "Can't defragment packet; dropping\n"); m_freem(slot->mbuf); } else { CPSW_DEBUGF(sc->swsc, ("Requeueing defragmented packet")); if_sendq_prepend(sc->ifp, m0); } slot->mbuf = NULL; continue; } if (error != 0) { device_printf(sc->dev, "%s: Can't setup DMA (error=%d), dropping packet\n", __func__, error); bus_dmamap_unload(sc->swsc->mbuf_dtag, slot->dmamap); m_freem(slot->mbuf); slot->mbuf = NULL; break; } bus_dmamap_sync(sc->swsc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_PREWRITE); CPSW_DEBUGF(sc->swsc, ("Queueing TX packet: %d segments + %d pad bytes", nsegs, padlen)); if (first_new_slot == NULL) first_new_slot = slot; /* Link from the previous descriptor. */ if (last != NULL) cpsw_cpdma_write_bd_next(sc->swsc, last, slot); slot->ifp = sc->ifp; /* If there is only one segment, the for() loop * gets skipped and the single buffer gets set up * as both SOP and EOP. */ if (nsegs > 1) { next = STAILQ_NEXT(slot, next); bd.next = cpsw_cpdma_bd_paddr(sc->swsc, next); } else bd.next = 0; /* Start by setting up the first buffer. */ bd.bufptr = segs[0].ds_addr; bd.bufoff = 0; bd.buflen = segs[0].ds_len; bd.pktlen = m_length(slot->mbuf, NULL); bd.flags = CPDMA_BD_SOP | CPDMA_BD_OWNER; if (sc->swsc->dualemac) { bd.flags |= CPDMA_BD_TO_PORT; bd.flags |= ((sc->unit + 1) & CPDMA_BD_PORT_MASK); } for (seg = 1; seg < nsegs; ++seg) { /* Save the previous buffer (which isn't EOP) */ cpsw_cpdma_write_bd(sc->swsc, slot, &bd); STAILQ_REMOVE_HEAD(&sc->swsc->tx.avail, next); STAILQ_INSERT_TAIL(&sc->swsc->tx.active, slot, next); slot = STAILQ_FIRST(&sc->swsc->tx.avail); /* Setup next buffer (which isn't SOP) */ if (nsegs > seg + 1) { next = STAILQ_NEXT(slot, next); bd.next = cpsw_cpdma_bd_paddr(sc->swsc, next); } else bd.next = 0; bd.bufptr = segs[seg].ds_addr; bd.bufoff = 0; bd.buflen = segs[seg].ds_len; bd.pktlen = 0; bd.flags = CPDMA_BD_OWNER; } /* Save the final buffer. */ bd.flags |= CPDMA_BD_EOP; cpsw_cpdma_write_bd(sc->swsc, slot, &bd); STAILQ_REMOVE_HEAD(&sc->swsc->tx.avail, next); STAILQ_INSERT_TAIL(&sc->swsc->tx.active, slot, next); last = slot; added += nsegs; if (nsegs > sc->swsc->tx.longest_chain) sc->swsc->tx.longest_chain = nsegs; BPF_MTAP(sc->ifp, m0); } if (first_new_slot == NULL) return; /* Attach the list of new buffers to the hardware TX queue. */ if (last_old_slot != NULL && (cpsw_cpdma_read_bd_flags(sc->swsc, last_old_slot) & CPDMA_BD_EOQ) == 0) { /* Add buffers to end of current queue. */ cpsw_cpdma_write_bd_next(sc->swsc, last_old_slot, first_new_slot); } else { /* Start a fresh queue. */ cpsw_write_hdp_slot(sc->swsc, &sc->swsc->tx, first_new_slot); } sc->swsc->tx.queue_adds += added; sc->swsc->tx.avail_queue_len -= added; sc->swsc->tx.active_queue_len += added; if (sc->swsc->tx.active_queue_len > sc->swsc->tx.max_active_queue_len) { sc->swsc->tx.max_active_queue_len = sc->swsc->tx.active_queue_len; } CPSW_DEBUGF(sc->swsc, ("Queued %d TX packet(s)", added)); } static int cpsw_tx_dequeue(struct cpsw_softc *sc) { struct cpsw_slot *slot, *last_removed_slot = NULL; struct cpsw_cpdma_bd bd; uint32_t flags, removed = 0; /* Pull completed buffers off the hardware TX queue. */ slot = STAILQ_FIRST(&sc->tx.active); while (slot != NULL) { flags = cpsw_cpdma_read_bd_flags(sc, slot); /* TearDown complete is only marked on the SOP for the packet. */ if ((flags & (CPDMA_BD_SOP | CPDMA_BD_TDOWNCMPLT)) == (CPDMA_BD_SOP | CPDMA_BD_TDOWNCMPLT)) { sc->tx.teardown = 1; } if ((flags & (CPDMA_BD_SOP | CPDMA_BD_OWNER)) == (CPDMA_BD_SOP | CPDMA_BD_OWNER) && sc->tx.teardown == 0) break; /* Hardware is still using this packet. */ bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap); m_freem(slot->mbuf); slot->mbuf = NULL; if (slot->ifp) { if (sc->tx.teardown == 0) if_inc_counter(slot->ifp, IFCOUNTER_OPACKETS, 1); else if_inc_counter(slot->ifp, IFCOUNTER_OQDROPS, 1); } /* Dequeue any additional buffers used by this packet. */ while (slot != NULL && slot->mbuf == NULL) { STAILQ_REMOVE_HEAD(&sc->tx.active, next); STAILQ_INSERT_TAIL(&sc->tx.avail, slot, next); ++removed; last_removed_slot = slot; slot = STAILQ_FIRST(&sc->tx.active); } cpsw_write_cp_slot(sc, &sc->tx, last_removed_slot); /* Restart the TX queue if necessary. */ cpsw_cpdma_read_bd(sc, last_removed_slot, &bd); if (slot != NULL && bd.next != 0 && (bd.flags & (CPDMA_BD_EOP | CPDMA_BD_OWNER | CPDMA_BD_EOQ)) == (CPDMA_BD_EOP | CPDMA_BD_EOQ)) { cpsw_write_hdp_slot(sc, &sc->tx, slot); sc->tx.queue_restart++; break; } } if (removed != 0) { sc->tx.queue_removes += removed; sc->tx.active_queue_len -= removed; sc->tx.avail_queue_len += removed; if (sc->tx.avail_queue_len > sc->tx.max_avail_queue_len) sc->tx.max_avail_queue_len = sc->tx.avail_queue_len; CPSW_DEBUGF(sc, ("TX removed %d completed packet(s)", removed)); } if (sc->tx.teardown && STAILQ_EMPTY(&sc->tx.active)) { CPSW_DEBUGF(sc, ("TX teardown is complete")); sc->tx.teardown = 0; sc->tx.running = 0; } return (removed); } /* * * Miscellaneous interrupts. * */ static void cpsw_intr_rx_thresh(void *arg) { struct cpsw_softc *sc; if_t ifp; struct mbuf *received, *next; sc = (struct cpsw_softc *)arg; CPSW_RX_LOCK(sc); received = cpsw_rx_dequeue(sc); cpsw_rx_enqueue(sc); cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 0); CPSW_RX_UNLOCK(sc); while (received != NULL) { next = received->m_nextpkt; received->m_nextpkt = NULL; ifp = received->m_pkthdr.rcvif; if_input(ifp, received); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); received = next; } } static void cpsw_intr_misc_host_error(struct cpsw_softc *sc) { uint32_t intstat; uint32_t dmastat; int txerr, rxerr, txchan, rxchan; printf("\n\n"); device_printf(sc->dev, "HOST ERROR: PROGRAMMING ERROR DETECTED BY HARDWARE\n"); printf("\n\n"); intstat = cpsw_read_4(sc, CPSW_CPDMA_DMA_INTSTAT_MASKED); device_printf(sc->dev, "CPSW_CPDMA_DMA_INTSTAT_MASKED=0x%x\n", intstat); dmastat = cpsw_read_4(sc, CPSW_CPDMA_DMASTATUS); device_printf(sc->dev, "CPSW_CPDMA_DMASTATUS=0x%x\n", dmastat); txerr = (dmastat >> 20) & 15; txchan = (dmastat >> 16) & 7; rxerr = (dmastat >> 12) & 15; rxchan = (dmastat >> 8) & 7; switch (txerr) { case 0: break; case 1: printf("SOP error on TX channel %d\n", txchan); break; case 2: printf("Ownership bit not set on SOP buffer on TX channel %d\n", txchan); break; case 3: printf("Zero Next Buffer but not EOP on TX channel %d\n", txchan); break; case 4: printf("Zero Buffer Pointer on TX channel %d\n", txchan); break; case 5: printf("Zero Buffer Length on TX channel %d\n", txchan); break; case 6: printf("Packet length error on TX channel %d\n", txchan); break; default: printf("Unknown error on TX channel %d\n", txchan); break; } if (txerr != 0) { printf("CPSW_CPDMA_TX%d_HDP=0x%x\n", txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_HDP(txchan))); printf("CPSW_CPDMA_TX%d_CP=0x%x\n", txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_CP(txchan))); cpsw_dump_queue(sc, &sc->tx.active); } switch (rxerr) { case 0: break; case 2: printf("Ownership bit not set on RX channel %d\n", rxchan); break; case 4: printf("Zero Buffer Pointer on RX channel %d\n", rxchan); break; case 5: printf("Zero Buffer Length on RX channel %d\n", rxchan); break; case 6: printf("Buffer offset too big on RX channel %d\n", rxchan); break; default: printf("Unknown RX error on RX channel %d\n", rxchan); break; } if (rxerr != 0) { printf("CPSW_CPDMA_RX%d_HDP=0x%x\n", rxchan, cpsw_read_4(sc,CPSW_CPDMA_RX_HDP(rxchan))); printf("CPSW_CPDMA_RX%d_CP=0x%x\n", rxchan, cpsw_read_4(sc, CPSW_CPDMA_RX_CP(rxchan))); cpsw_dump_queue(sc, &sc->rx.active); } printf("\nALE Table\n"); cpsw_ale_dump_table(sc); // XXX do something useful here?? panic("CPSW HOST ERROR INTERRUPT"); // Suppress this interrupt in the future. cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_CLEAR, intstat); printf("XXX HOST ERROR INTERRUPT SUPPRESSED\n"); // The watchdog will probably reset the controller // in a little while. It will probably fail again. } static void cpsw_intr_misc(void *arg) { struct cpsw_softc *sc = arg; uint32_t stat = cpsw_read_4(sc, CPSW_WR_C_MISC_STAT(0)); if (stat & CPSW_WR_C_MISC_EVNT_PEND) CPSW_DEBUGF(sc, ("Time sync event interrupt unimplemented")); if (stat & CPSW_WR_C_MISC_STAT_PEND) cpsw_stats_collect(sc); if (stat & CPSW_WR_C_MISC_HOST_PEND) cpsw_intr_misc_host_error(sc); if (stat & CPSW_WR_C_MISC_MDIOLINK) { cpsw_write_4(sc, MDIOLINKINTMASKED, cpsw_read_4(sc, MDIOLINKINTMASKED)); } if (stat & CPSW_WR_C_MISC_MDIOUSER) { CPSW_DEBUGF(sc, ("MDIO operation completed interrupt unimplemented")); } cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 3); } /* * * Periodic Checks and Watchdog. * */ static void cpswp_tick(void *msc) { struct cpswp_softc *sc = msc; /* Check for media type change */ mii_tick(sc->mii); if (sc->media_status != sc->mii->mii_media.ifm_media) { printf("%s: media type changed (ifm_media=%x)\n", __func__, sc->mii->mii_media.ifm_media); cpswp_ifmedia_upd(sc->ifp); } /* Schedule another timeout one second from now */ callout_reset(&sc->mii_callout, hz, cpswp_tick, sc); } static void cpswp_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) { struct cpswp_softc *sc; struct mii_data *mii; sc = if_getsoftc(ifp); CPSW_DEBUGF(sc->swsc, ("")); CPSW_PORT_LOCK(sc); mii = sc->mii; mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; CPSW_PORT_UNLOCK(sc); } static int cpswp_ifmedia_upd(if_t ifp) { struct cpswp_softc *sc; sc = if_getsoftc(ifp); CPSW_DEBUGF(sc->swsc, ("")); CPSW_PORT_LOCK(sc); mii_mediachg(sc->mii); sc->media_status = sc->mii->mii_media.ifm_media; CPSW_PORT_UNLOCK(sc); return (0); } static void cpsw_tx_watchdog_full_reset(struct cpsw_softc *sc) { struct cpswp_softc *psc; int i; cpsw_debugf_head("CPSW watchdog"); device_printf(sc->dev, "watchdog timeout\n"); printf("CPSW_CPDMA_TX%d_HDP=0x%x\n", 0, cpsw_read_4(sc, CPSW_CPDMA_TX_HDP(0))); printf("CPSW_CPDMA_TX%d_CP=0x%x\n", 0, cpsw_read_4(sc, CPSW_CPDMA_TX_CP(0))); cpsw_dump_queue(sc, &sc->tx.active); for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; psc = device_get_softc(sc->port[i].dev); CPSW_PORT_LOCK(psc); cpswp_stop_locked(psc); CPSW_PORT_UNLOCK(psc); } } static void cpsw_tx_watchdog(void *msc) { struct cpsw_softc *sc; sc = msc; CPSW_TX_LOCK(sc); if (sc->tx.active_queue_len == 0 || !sc->tx.running) { sc->watchdog.timer = 0; /* Nothing to do. */ } else if (sc->tx.queue_removes > sc->tx.queue_removes_at_last_tick) { sc->watchdog.timer = 0; /* Stuff done while we weren't looking. */ } else if (cpsw_tx_dequeue(sc) > 0) { sc->watchdog.timer = 0; /* We just did something. */ } else { /* There was something to do but it didn't get done. */ ++sc->watchdog.timer; if (sc->watchdog.timer > 5) { sc->watchdog.timer = 0; ++sc->watchdog.resets; cpsw_tx_watchdog_full_reset(sc); } } sc->tx.queue_removes_at_last_tick = sc->tx.queue_removes; CPSW_TX_UNLOCK(sc); /* Schedule another timeout one second from now */ callout_reset(&sc->watchdog.callout, hz, cpsw_tx_watchdog, sc); } /* * * ALE support routines. * */ static void cpsw_ale_read_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry) { cpsw_write_4(sc, CPSW_ALE_TBLCTL, idx & 1023); ale_entry[0] = cpsw_read_4(sc, CPSW_ALE_TBLW0); ale_entry[1] = cpsw_read_4(sc, CPSW_ALE_TBLW1); ale_entry[2] = cpsw_read_4(sc, CPSW_ALE_TBLW2); } static void cpsw_ale_write_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry) { cpsw_write_4(sc, CPSW_ALE_TBLW0, ale_entry[0]); cpsw_write_4(sc, CPSW_ALE_TBLW1, ale_entry[1]); cpsw_write_4(sc, CPSW_ALE_TBLW2, ale_entry[2]); cpsw_write_4(sc, CPSW_ALE_TBLCTL, 1 << 31 | (idx & 1023)); } static void cpsw_ale_remove_all_mc_entries(struct cpsw_softc *sc) { int i; uint32_t ale_entry[3]; /* First four entries are link address and broadcast. */ for (i = 10; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); if ((ALE_TYPE(ale_entry) == ALE_TYPE_ADDR || ALE_TYPE(ale_entry) == ALE_TYPE_VLAN_ADDR) && ALE_MCAST(ale_entry) == 1) { /* MCast link addr */ ale_entry[0] = ale_entry[1] = ale_entry[2] = 0; cpsw_ale_write_entry(sc, i, ale_entry); } } } static int cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, int vlan, uint8_t *mac) { int free_index = -1, matching_index = -1, i; uint32_t ale_entry[3], ale_type; /* Find a matching entry or a free entry. */ for (i = 10; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); /* Entry Type[61:60] is 0 for free entry */ if (free_index < 0 && ALE_TYPE(ale_entry) == 0) free_index = i; if ((((ale_entry[1] >> 8) & 0xFF) == mac[0]) && (((ale_entry[1] >> 0) & 0xFF) == mac[1]) && (((ale_entry[0] >>24) & 0xFF) == mac[2]) && (((ale_entry[0] >>16) & 0xFF) == mac[3]) && (((ale_entry[0] >> 8) & 0xFF) == mac[4]) && (((ale_entry[0] >> 0) & 0xFF) == mac[5])) { matching_index = i; break; } } if (matching_index < 0) { if (free_index < 0) return (ENOMEM); i = free_index; } if (vlan != -1) ale_type = ALE_TYPE_VLAN_ADDR << 28 | vlan << 16; else ale_type = ALE_TYPE_ADDR << 28; /* Set MAC address */ ale_entry[0] = mac[2] << 24 | mac[3] << 16 | mac[4] << 8 | mac[5]; ale_entry[1] = mac[0] << 8 | mac[1]; /* Entry type[61:60] and Mcast fwd state[63:62] is fw(3). */ ale_entry[1] |= ALE_MCAST_FWD | ale_type; /* Set portmask [68:66] */ ale_entry[2] = (portmap & 7) << 2; cpsw_ale_write_entry(sc, i, ale_entry); return 0; } static void cpsw_ale_dump_table(struct cpsw_softc *sc) { int i; uint32_t ale_entry[3]; for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); switch (ALE_TYPE(ale_entry)) { case ALE_TYPE_VLAN: printf("ALE[%4u] %08x %08x %08x ", i, ale_entry[2], ale_entry[1], ale_entry[0]); printf("type: %u ", ALE_TYPE(ale_entry)); printf("vlan: %u ", ALE_VLAN(ale_entry)); printf("untag: %u ", ALE_VLAN_UNTAG(ale_entry)); printf("reg flood: %u ", ALE_VLAN_REGFLOOD(ale_entry)); printf("unreg flood: %u ", ALE_VLAN_UNREGFLOOD(ale_entry)); printf("members: %u ", ALE_VLAN_MEMBERS(ale_entry)); printf("\n"); break; case ALE_TYPE_ADDR: case ALE_TYPE_VLAN_ADDR: printf("ALE[%4u] %08x %08x %08x ", i, ale_entry[2], ale_entry[1], ale_entry[0]); printf("type: %u ", ALE_TYPE(ale_entry)); printf("mac: %02x:%02x:%02x:%02x:%02x:%02x ", (ale_entry[1] >> 8) & 0xFF, (ale_entry[1] >> 0) & 0xFF, (ale_entry[0] >>24) & 0xFF, (ale_entry[0] >>16) & 0xFF, (ale_entry[0] >> 8) & 0xFF, (ale_entry[0] >> 0) & 0xFF); printf(ALE_MCAST(ale_entry) ? "mcast " : "ucast "); if (ALE_TYPE(ale_entry) == ALE_TYPE_VLAN_ADDR) printf("vlan: %u ", ALE_VLAN(ale_entry)); printf("port: %u ", ALE_PORTS(ale_entry)); printf("\n"); break; } } printf("\n"); } static u_int cpswp_set_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) { struct cpswp_softc *sc = arg; uint32_t portmask; if (sc->swsc->dualemac) portmask = 1 << (sc->unit + 1) | 1 << 0; else portmask = 7; cpsw_ale_mc_entry_set(sc->swsc, portmask, sc->vlan, LLADDR(sdl)); return (1); } static int cpswp_ale_update_addresses(struct cpswp_softc *sc, int purge) { uint8_t *mac; uint32_t ale_entry[3], ale_type, portmask; if (sc->swsc->dualemac) { ale_type = ALE_TYPE_VLAN_ADDR << 28 | sc->vlan << 16; portmask = 1 << (sc->unit + 1) | 1 << 0; } else { ale_type = ALE_TYPE_ADDR << 28; portmask = 7; } /* * Route incoming packets for our MAC address to Port 0 (host). * For simplicity, keep this entry at table index 0 for port 1 and * at index 2 for port 2 in the ALE. */ mac = LLADDR((struct sockaddr_dl *)if_getifaddr(sc->ifp)->ifa_addr); ale_entry[0] = mac[2] << 24 | mac[3] << 16 | mac[4] << 8 | mac[5]; ale_entry[1] = ale_type | mac[0] << 8 | mac[1]; /* addr entry + mac */ ale_entry[2] = 0; /* port = 0 */ cpsw_ale_write_entry(sc->swsc, 0 + 2 * sc->unit, ale_entry); /* Set outgoing MAC Address for slave port. */ cpsw_write_4(sc->swsc, CPSW_PORT_P_SA_HI(sc->unit + 1), mac[3] << 24 | mac[2] << 16 | mac[1] << 8 | mac[0]); cpsw_write_4(sc->swsc, CPSW_PORT_P_SA_LO(sc->unit + 1), mac[5] << 8 | mac[4]); /* Keep the broadcast address at table entry 1 (or 3). */ ale_entry[0] = 0xffffffff; /* Lower 32 bits of MAC */ /* ALE_MCAST_FWD, Addr type, upper 16 bits of Mac */ ale_entry[1] = ALE_MCAST_FWD | ale_type | 0xffff; ale_entry[2] = portmask << 2; cpsw_ale_write_entry(sc->swsc, 1 + 2 * sc->unit, ale_entry); /* SIOCDELMULTI doesn't specify the particular address being removed, so we have to remove all and rebuild. */ if (purge) cpsw_ale_remove_all_mc_entries(sc->swsc); /* Set other multicast addrs desired. */ if_foreach_llmaddr(sc->ifp, cpswp_set_maddr, sc); return (0); } static int cpsw_ale_update_vlan_table(struct cpsw_softc *sc, int vlan, int ports, int untag, int mcregflood, int mcunregflood) { int free_index, i, matching_index; uint32_t ale_entry[3]; free_index = matching_index = -1; /* Find a matching entry or a free entry. */ for (i = 5; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); /* Entry Type[61:60] is 0 for free entry */ if (free_index < 0 && ALE_TYPE(ale_entry) == 0) free_index = i; if (ALE_VLAN(ale_entry) == vlan) { matching_index = i; break; } } if (matching_index < 0) { if (free_index < 0) return (-1); i = free_index; } ale_entry[0] = (untag & 7) << 24 | (mcregflood & 7) << 16 | (mcunregflood & 7) << 8 | (ports & 7); ale_entry[1] = ALE_TYPE_VLAN << 28 | vlan << 16; ale_entry[2] = 0; cpsw_ale_write_entry(sc, i, ale_entry); return (0); } /* * * Statistics and Sysctls. * */ #if 0 static void cpsw_stats_dump(struct cpsw_softc *sc) { int i; uint32_t r; for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) { r = cpsw_read_4(sc, CPSW_STATS_OFFSET + cpsw_stat_sysctls[i].reg); CPSW_DEBUGF(sc, ("%s: %ju + %u = %ju", cpsw_stat_sysctls[i].oid, (intmax_t)sc->shadow_stats[i], r, (intmax_t)sc->shadow_stats[i] + r)); } } #endif static void cpsw_stats_collect(struct cpsw_softc *sc) { int i; uint32_t r; CPSW_DEBUGF(sc, ("Controller shadow statistics updated.")); for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) { r = cpsw_read_4(sc, CPSW_STATS_OFFSET + cpsw_stat_sysctls[i].reg); sc->shadow_stats[i] += r; cpsw_write_4(sc, CPSW_STATS_OFFSET + cpsw_stat_sysctls[i].reg, r); } } static int cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS) { struct cpsw_softc *sc; struct cpsw_stat *stat; uint64_t result; sc = (struct cpsw_softc *)arg1; stat = &cpsw_stat_sysctls[oidp->oid_number]; result = sc->shadow_stats[oidp->oid_number]; result += cpsw_read_4(sc, CPSW_STATS_OFFSET + stat->reg); return (sysctl_handle_64(oidp, &result, 0, req)); } static int cpsw_stat_attached(SYSCTL_HANDLER_ARGS) { struct cpsw_softc *sc; struct bintime t; unsigned result; sc = (struct cpsw_softc *)arg1; getbinuptime(&t); bintime_sub(&t, &sc->attach_uptime); result = t.sec; return (sysctl_handle_int(oidp, &result, 0, req)); } static int cpsw_intr_coalesce(SYSCTL_HANDLER_ARGS) { int error; struct cpsw_softc *sc; uint32_t ctrl, intr_per_ms; sc = (struct cpsw_softc *)arg1; error = sysctl_handle_int(oidp, &sc->coal_us, 0, req); if (error != 0 || req->newptr == NULL) return (error); ctrl = cpsw_read_4(sc, CPSW_WR_INT_CONTROL); ctrl &= ~(CPSW_WR_INT_PACE_EN | CPSW_WR_INT_PRESCALE_MASK); if (sc->coal_us == 0) { /* Disable the interrupt pace hardware. */ cpsw_write_4(sc, CPSW_WR_INT_CONTROL, ctrl); cpsw_write_4(sc, CPSW_WR_C_RX_IMAX(0), 0); cpsw_write_4(sc, CPSW_WR_C_TX_IMAX(0), 0); return (0); } if (sc->coal_us > CPSW_WR_C_IMAX_US_MAX) sc->coal_us = CPSW_WR_C_IMAX_US_MAX; if (sc->coal_us < CPSW_WR_C_IMAX_US_MIN) sc->coal_us = CPSW_WR_C_IMAX_US_MIN; intr_per_ms = 1000 / sc->coal_us; /* Just to make sure... */ if (intr_per_ms > CPSW_WR_C_IMAX_MAX) intr_per_ms = CPSW_WR_C_IMAX_MAX; if (intr_per_ms < CPSW_WR_C_IMAX_MIN) intr_per_ms = CPSW_WR_C_IMAX_MIN; /* Set the prescale to produce 4us pulses from the 125 Mhz clock. */ ctrl |= (125 * 4) & CPSW_WR_INT_PRESCALE_MASK; /* Enable the interrupt pace hardware. */ cpsw_write_4(sc, CPSW_WR_C_RX_IMAX(0), intr_per_ms); cpsw_write_4(sc, CPSW_WR_C_TX_IMAX(0), intr_per_ms); ctrl |= CPSW_WR_INT_C0_RX_PULSE | CPSW_WR_INT_C0_TX_PULSE; cpsw_write_4(sc, CPSW_WR_INT_CONTROL, ctrl); return (0); } static int cpsw_stat_uptime(SYSCTL_HANDLER_ARGS) { struct cpsw_softc *swsc; struct cpswp_softc *sc; struct bintime t; unsigned result; swsc = arg1; sc = device_get_softc(swsc->port[arg2].dev); if (if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) { getbinuptime(&t); bintime_sub(&t, &sc->init_uptime); result = t.sec; } else result = 0; return (sysctl_handle_int(oidp, &result, 0, req)); } static void cpsw_add_queue_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *node, struct cpsw_queue *queue) { struct sysctl_oid_list *parent; parent = SYSCTL_CHILDREN(node); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "totalBuffers", CTLFLAG_RD, &queue->queue_slots, 0, "Total buffers currently assigned to this queue"); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "activeBuffers", CTLFLAG_RD, &queue->active_queue_len, 0, "Buffers currently registered with hardware controller"); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxActiveBuffers", CTLFLAG_RD, &queue->max_active_queue_len, 0, "Max value of activeBuffers since last driver reset"); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "availBuffers", CTLFLAG_RD, &queue->avail_queue_len, 0, "Buffers allocated to this queue but not currently " "registered with hardware controller"); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxAvailBuffers", CTLFLAG_RD, &queue->max_avail_queue_len, 0, "Max value of availBuffers since last driver reset"); SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalEnqueued", CTLFLAG_RD, &queue->queue_adds, 0, "Total buffers added to queue"); SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalDequeued", CTLFLAG_RD, &queue->queue_removes, 0, "Total buffers removed from queue"); SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "queueRestart", CTLFLAG_RD, &queue->queue_restart, 0, "Total times the queue has been restarted"); SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "longestChain", CTLFLAG_RD, &queue->longest_chain, 0, "Max buffers used for a single packet"); } static void cpsw_add_watchdog_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *node, struct cpsw_softc *sc) { struct sysctl_oid_list *parent; parent = SYSCTL_CHILDREN(node); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "resets", CTLFLAG_RD, &sc->watchdog.resets, 0, "Total number of watchdog resets"); } static void cpsw_add_sysctls(struct cpsw_softc *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *stats_node, *queue_node, *node; struct sysctl_oid_list *parent, *stats_parent, *queue_parent; struct sysctl_oid_list *ports_parent, *port_parent; char port[16]; int i; ctx = device_get_sysctl_ctx(sc->dev); parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "debug", CTLFLAG_RW, &sc->debug, 0, "Enable switch debug messages"); SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "attachedSecs", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, cpsw_stat_attached, "IU", "Time since driver attach"); SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "intr_coalesce_us", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, cpsw_intr_coalesce, "IU", "minimum time between interrupts"); node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "ports", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CPSW Ports Statistics"); ports_parent = SYSCTL_CHILDREN(node); for (i = 0; i < CPSW_PORTS; i++) { if (!sc->dualemac && i != sc->active_slave) continue; port[0] = '0' + i; port[1] = '\0'; node = SYSCTL_ADD_NODE(ctx, ports_parent, OID_AUTO, port, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CPSW Port Statistics"); port_parent = SYSCTL_CHILDREN(node); SYSCTL_ADD_PROC(ctx, port_parent, OID_AUTO, "uptime", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, i, cpsw_stat_uptime, "IU", "Seconds since driver init"); } stats_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CPSW Statistics"); stats_parent = SYSCTL_CHILDREN(stats_node); for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) { SYSCTL_ADD_PROC(ctx, stats_parent, i, cpsw_stat_sysctls[i].oid, CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, cpsw_stats_sysctl, "IU", cpsw_stat_sysctls[i].oid); } queue_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "queue", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "CPSW Queue Statistics"); queue_parent = SYSCTL_CHILDREN(queue_node); node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "tx", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Statistics"); cpsw_add_queue_sysctls(ctx, node, &sc->tx); node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "rx", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Statistics"); cpsw_add_queue_sysctls(ctx, node, &sc->rx); node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "watchdog", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Watchdog Statistics"); cpsw_add_watchdog_sysctls(ctx, node, sc); } #ifdef CPSW_ETHERSWITCH static etherswitch_info_t etherswitch_info = { .es_nports = CPSW_PORTS + 1, .es_nvlangroups = CPSW_VLANS, .es_name = "TI Common Platform Ethernet Switch (CPSW)", .es_vlan_caps = ETHERSWITCH_VLAN_DOT1Q, }; static etherswitch_info_t * cpsw_getinfo(device_t dev) { return (ðerswitch_info); } static int cpsw_getport(device_t dev, etherswitch_port_t *p) { int err; struct cpsw_softc *sc; struct cpswp_softc *psc; struct ifmediareq *ifmr; uint32_t reg; if (p->es_port < 0 || p->es_port > CPSW_PORTS) return (ENXIO); err = 0; sc = device_get_softc(dev); if (p->es_port == CPSW_CPU_PORT) { p->es_flags |= ETHERSWITCH_PORT_CPU; ifmr = &p->es_ifmr; ifmr->ifm_current = ifmr->ifm_active = IFM_ETHER | IFM_1000_T | IFM_FDX; ifmr->ifm_mask = 0; ifmr->ifm_status = IFM_ACTIVE | IFM_AVALID; ifmr->ifm_count = 0; } else { psc = device_get_softc(sc->port[p->es_port - 1].dev); err = ifmedia_ioctl(psc->ifp, &p->es_ifr, &psc->mii->mii_media, SIOCGIFMEDIA); } reg = cpsw_read_4(sc, CPSW_PORT_P_VLAN(p->es_port)); p->es_pvid = reg & ETHERSWITCH_VID_MASK; reg = cpsw_read_4(sc, CPSW_ALE_PORTCTL(p->es_port)); if (reg & ALE_PORTCTL_DROP_UNTAGGED) p->es_flags |= ETHERSWITCH_PORT_DROPUNTAGGED; if (reg & ALE_PORTCTL_INGRESS) p->es_flags |= ETHERSWITCH_PORT_INGRESS; return (err); } static int cpsw_setport(device_t dev, etherswitch_port_t *p) { struct cpsw_softc *sc; struct cpswp_softc *psc; struct ifmedia *ifm; uint32_t reg; if (p->es_port < 0 || p->es_port > CPSW_PORTS) return (ENXIO); sc = device_get_softc(dev); if (p->es_pvid != 0) { cpsw_write_4(sc, CPSW_PORT_P_VLAN(p->es_port), p->es_pvid & ETHERSWITCH_VID_MASK); } reg = cpsw_read_4(sc, CPSW_ALE_PORTCTL(p->es_port)); if (p->es_flags & ETHERSWITCH_PORT_DROPUNTAGGED) reg |= ALE_PORTCTL_DROP_UNTAGGED; else reg &= ~ALE_PORTCTL_DROP_UNTAGGED; if (p->es_flags & ETHERSWITCH_PORT_INGRESS) reg |= ALE_PORTCTL_INGRESS; else reg &= ~ALE_PORTCTL_INGRESS; cpsw_write_4(sc, CPSW_ALE_PORTCTL(p->es_port), reg); /* CPU port does not allow media settings. */ if (p->es_port == CPSW_CPU_PORT) return (0); psc = device_get_softc(sc->port[p->es_port - 1].dev); ifm = &psc->mii->mii_media; return (ifmedia_ioctl(psc->ifp, &p->es_ifr, ifm, SIOCSIFMEDIA)); } static int cpsw_getconf(device_t dev, etherswitch_conf_t *conf) { /* Return the VLAN mode. */ conf->cmd = ETHERSWITCH_CONF_VLAN_MODE; conf->vlan_mode = ETHERSWITCH_VLAN_DOT1Q; return (0); } static int cpsw_getvgroup(device_t dev, etherswitch_vlangroup_t *vg) { int i, vid; uint32_t ale_entry[3]; struct cpsw_softc *sc; sc = device_get_softc(dev); if (vg->es_vlangroup >= CPSW_VLANS) return (EINVAL); vg->es_vid = 0; vid = cpsw_vgroups[vg->es_vlangroup].vid; if (vid == -1) return (0); for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); if (ALE_TYPE(ale_entry) != ALE_TYPE_VLAN) continue; if (vid != ALE_VLAN(ale_entry)) continue; vg->es_fid = 0; vg->es_vid = ALE_VLAN(ale_entry) | ETHERSWITCH_VID_VALID; vg->es_member_ports = ALE_VLAN_MEMBERS(ale_entry); vg->es_untagged_ports = ALE_VLAN_UNTAG(ale_entry); } return (0); } static void cpsw_remove_vlan(struct cpsw_softc *sc, int vlan) { int i; uint32_t ale_entry[3]; for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) { cpsw_ale_read_entry(sc, i, ale_entry); if (ALE_TYPE(ale_entry) != ALE_TYPE_VLAN) continue; if (vlan != ALE_VLAN(ale_entry)) continue; ale_entry[0] = ale_entry[1] = ale_entry[2] = 0; cpsw_ale_write_entry(sc, i, ale_entry); break; } } static int cpsw_setvgroup(device_t dev, etherswitch_vlangroup_t *vg) { int i; struct cpsw_softc *sc; sc = device_get_softc(dev); for (i = 0; i < CPSW_VLANS; i++) { /* Is this Vlan ID in use by another vlangroup ? */ if (vg->es_vlangroup != i && cpsw_vgroups[i].vid == vg->es_vid) return (EINVAL); } if (vg->es_vid == 0) { if (cpsw_vgroups[vg->es_vlangroup].vid == -1) return (0); cpsw_remove_vlan(sc, cpsw_vgroups[vg->es_vlangroup].vid); cpsw_vgroups[vg->es_vlangroup].vid = -1; vg->es_untagged_ports = 0; vg->es_member_ports = 0; vg->es_vid = 0; return (0); } vg->es_vid &= ETHERSWITCH_VID_MASK; vg->es_member_ports &= CPSW_PORTS_MASK; vg->es_untagged_ports &= CPSW_PORTS_MASK; if (cpsw_vgroups[vg->es_vlangroup].vid != -1 && cpsw_vgroups[vg->es_vlangroup].vid != vg->es_vid) return (EINVAL); cpsw_vgroups[vg->es_vlangroup].vid = vg->es_vid; cpsw_ale_update_vlan_table(sc, vg->es_vid, vg->es_member_ports, vg->es_untagged_ports, vg->es_member_ports, 0); return (0); } static int cpsw_readreg(device_t dev, int addr) { /* Not supported. */ return (0); } static int cpsw_writereg(device_t dev, int addr, int value) { /* Not supported. */ return (0); } static int cpsw_readphy(device_t dev, int phy, int reg) { /* Not supported. */ return (0); } static int cpsw_writephy(device_t dev, int phy, int reg, int data) { /* Not supported. */ return (0); } #endif diff --git a/sys/dev/cardbus/cardbus.c b/sys/dev/cardbus/cardbus.c index f375eb112cce..21467a11cb68 100644 --- a/sys/dev/cardbus/cardbus.c +++ b/sys/dev/cardbus/cardbus.c @@ -1,356 +1,354 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2000,2001 Jonathan Chen. All rights reserved. * Copyright (c) 2003-2008 M. Warner Losh * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "power_if.h" #include "pcib_if.h" /* sysctl vars */ static SYSCTL_NODE(_hw, OID_AUTO, cardbus, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CardBus parameters"); int cardbus_debug = 0; SYSCTL_INT(_hw_cardbus, OID_AUTO, debug, CTLFLAG_RWTUN, &cardbus_debug, 0, "CardBus debug"); int cardbus_cis_debug = 0; SYSCTL_INT(_hw_cardbus, OID_AUTO, cis_debug, CTLFLAG_RWTUN, &cardbus_cis_debug, 0, "CardBus CIS debug"); #define DPRINTF(a) if (cardbus_debug) printf a #define DEVPRINTF(x) if (cardbus_debug) device_printf x static int cardbus_attach(device_t cbdev); static int cardbus_attach_card(device_t cbdev); static int cardbus_detach(device_t cbdev); static int cardbus_detach_card(device_t cbdev); static void cardbus_device_setup_regs(pcicfgregs *cfg); static void cardbus_driver_added(device_t cbdev, driver_t *driver); static int cardbus_probe(device_t cbdev); static int cardbus_read_ivar(device_t cbdev, device_t child, int which, uintptr_t *result); /************************************************************************/ /* Probe/Attach */ /************************************************************************/ static int cardbus_probe(device_t cbdev) { device_set_desc(cbdev, "CardBus bus"); return (0); } static int cardbus_attach(device_t cbdev) { struct cardbus_softc *sc; int rid; sc = device_get_softc(cbdev); sc->sc_dev = cbdev; rid = 0; sc->sc_bus = bus_alloc_resource(cbdev, PCI_RES_BUS, &rid, pcib_get_bus(cbdev), pcib_get_bus(cbdev), 1, 0); if (sc->sc_bus == NULL) { device_printf(cbdev, "failed to allocate bus number\n"); return (ENXIO); } return (0); } static int cardbus_detach(device_t cbdev) { struct cardbus_softc *sc; cardbus_detach_card(cbdev); sc = device_get_softc(cbdev); device_printf(cbdev, "Freeing up the allocatd bus\n"); (void)bus_release_resource(cbdev, PCI_RES_BUS, 0, sc->sc_bus); return (0); } static int cardbus_suspend(device_t self) { cardbus_detach_card(self); return (0); } static int cardbus_resume(device_t self) { return (0); } /************************************************************************/ /* Attach/Detach card */ /************************************************************************/ static void cardbus_device_setup_regs(pcicfgregs *cfg) { device_t dev = cfg->dev; int i; /* * Some cards power up with garbage in their BARs. This * code clears all that junk out. */ for (i = 0; i < PCIR_MAX_BAR_0; i++) pci_write_config(dev, PCIR_BAR(i), 0, 4); cfg->intline = pci_get_irq(device_get_parent(device_get_parent(dev))); pci_write_config(dev, PCIR_INTLINE, cfg->intline, 1); pci_write_config(dev, PCIR_CACHELNSZ, 0x08, 1); pci_write_config(dev, PCIR_LATTIMER, 0xa8, 1); pci_write_config(dev, PCIR_MINGNT, 0x14, 1); pci_write_config(dev, PCIR_MAXLAT, 0x14, 1); } static struct pci_devinfo * cardbus_alloc_devinfo(device_t dev) { struct cardbus_devinfo *dinfo; dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); return (&dinfo->pci); } static int cardbus_attach_card(device_t cbdev) { device_t brdev = device_get_parent(cbdev); device_t child; int bus, domain, slot, func; int cardattached = 0; int cardbusfunchigh = 0; struct cardbus_softc *sc; sc = device_get_softc(cbdev); cardbus_detach_card(cbdev); /* detach existing cards */ POWER_DISABLE_SOCKET(brdev, cbdev); /* Turn the socket off first */ POWER_ENABLE_SOCKET(brdev, cbdev); domain = pcib_get_domain(cbdev); bus = pcib_get_bus(cbdev); slot = 0; bus_topo_lock(); /* For each function, set it up and try to attach a driver to it */ for (func = 0; func <= cardbusfunchigh; func++) { struct cardbus_devinfo *dinfo; dinfo = (struct cardbus_devinfo *) pci_read_device(brdev, cbdev, domain, bus, slot, func); if (dinfo == NULL) continue; if (dinfo->pci.cfg.mfdev) cardbusfunchigh = PCI_FUNCMAX; child = device_add_child(cbdev, NULL, DEVICE_UNIT_ANY); if (child == NULL) { DEVPRINTF((cbdev, "Cannot add child!\n")); pci_freecfg((struct pci_devinfo *)dinfo); continue; } dinfo->pci.cfg.dev = child; resource_list_init(&dinfo->pci.resources); device_set_ivars(child, dinfo); cardbus_device_create(sc, dinfo, cbdev, child); if (cardbus_do_cis(cbdev, child) != 0) DEVPRINTF((cbdev, "Warning: Bogus CIS ignored\n")); pci_cfg_save(dinfo->pci.cfg.dev, &dinfo->pci, 0); pci_cfg_restore(dinfo->pci.cfg.dev, &dinfo->pci); cardbus_device_setup_regs(&dinfo->pci.cfg); pci_add_resources(cbdev, child, 1, dinfo->mprefetchable); pci_print_verbose(&dinfo->pci); if (device_probe_and_attach(child) == 0) cardattached++; else pci_cfg_save(dinfo->pci.cfg.dev, &dinfo->pci, 1); } bus_topo_unlock(); if (cardattached > 0) return (0); /* POWER_DISABLE_SOCKET(brdev, cbdev); */ return (ENOENT); } static void cardbus_child_deleted(device_t cbdev, device_t child) { struct cardbus_devinfo *dinfo = device_get_ivars(child); if (dinfo->pci.cfg.dev != child) device_printf(cbdev, "devinfo dev mismatch\n"); cardbus_device_destroy(dinfo); pci_child_deleted(cbdev, child); } static int cardbus_detach_card(device_t cbdev) { int err = 0; bus_topo_lock(); err = bus_generic_detach(cbdev); - if (err == 0) - err = device_delete_children(cbdev); bus_topo_unlock(); if (err) return (err); POWER_DISABLE_SOCKET(device_get_parent(cbdev), cbdev); return (err); } static void cardbus_driver_added(device_t cbdev, driver_t *driver) { int numdevs; device_t *devlist; device_t dev; int i; struct cardbus_devinfo *dinfo; DEVICE_IDENTIFY(driver, cbdev); if (device_get_children(cbdev, &devlist, &numdevs) != 0) return; /* * If there are no drivers attached, but there are children, * then power the card up. */ for (i = 0; i < numdevs; i++) { dev = devlist[i]; if (device_get_state(dev) != DS_NOTPRESENT) break; } if (i > 0 && i == numdevs) POWER_ENABLE_SOCKET(device_get_parent(cbdev), cbdev); for (i = 0; i < numdevs; i++) { dev = devlist[i]; if (device_get_state(dev) != DS_NOTPRESENT) continue; dinfo = device_get_ivars(dev); pci_print_verbose(&dinfo->pci); if (bootverbose) printf("pci%d:%d:%d:%d: reprobing on driver added\n", dinfo->pci.cfg.domain, dinfo->pci.cfg.bus, dinfo->pci.cfg.slot, dinfo->pci.cfg.func); pci_cfg_restore(dinfo->pci.cfg.dev, &dinfo->pci); if (device_probe_and_attach(dev) != 0) pci_cfg_save(dev, &dinfo->pci, 1); } free(devlist, M_TEMP); } /************************************************************************/ /* Other Bus Methods */ /************************************************************************/ static int cardbus_read_ivar(device_t cbdev, device_t child, int which, uintptr_t *result) { struct cardbus_devinfo *dinfo; dinfo = device_get_ivars(child); switch (which) { case PCI_IVAR_ETHADDR: /* * The generic accessor doesn't deal with failure, so * we set the return value, then return an error. */ if (dinfo->fepresent & (1 << PCCARD_TPLFE_TYPE_LAN_NID)) { *((uint8_t **) result) = dinfo->funce.lan.nid; break; } *((uint8_t **) result) = NULL; return (EINVAL); default: return (pci_read_ivar(cbdev, child, which, result)); } return 0; } static device_method_t cardbus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cardbus_probe), DEVMETHOD(device_attach, cardbus_attach), DEVMETHOD(device_detach, cardbus_detach), DEVMETHOD(device_suspend, cardbus_suspend), DEVMETHOD(device_resume, cardbus_resume), /* Bus interface */ DEVMETHOD(bus_child_deleted, cardbus_child_deleted), DEVMETHOD(bus_get_dma_tag, bus_generic_get_dma_tag), DEVMETHOD(bus_read_ivar, cardbus_read_ivar), DEVMETHOD(bus_driver_added, cardbus_driver_added), DEVMETHOD(bus_rescan, bus_null_rescan), /* Card Interface */ DEVMETHOD(card_attach_card, cardbus_attach_card), DEVMETHOD(card_detach_card, cardbus_detach_card), /* PCI interface */ DEVMETHOD(pci_alloc_devinfo, cardbus_alloc_devinfo), {0,0} }; DEFINE_CLASS_1(cardbus, cardbus_driver, cardbus_methods, sizeof(struct cardbus_softc), pci_driver); DRIVER_MODULE(cardbus, cbb, cardbus_driver, 0, 0); MODULE_VERSION(cardbus, 1); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 024c97dcb78c..e4120e76000c 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -1,13324 +1,13323 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_ddb.h" #include "opt_inet.h" #include "opt_inet6.h" #include "opt_kern_tls.h" #include "opt_ratelimit.h" #include "opt_rss.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RSS #include #endif #include #include #ifdef KERN_TLS #include #endif #if defined(__i386__) || defined(__amd64__) #include #include #include #include #endif #ifdef DDB #include #include #endif #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_regs_values.h" #include "cudbg/cudbg.h" #include "t4_clip.h" #include "t4_ioctl.h" #include "t4_l2t.h" #include "t4_mp_ring.h" #include "t4_if.h" #include "t4_smt.h" /* T4 bus driver interface */ static int t4_probe(device_t); static int t4_attach(device_t); static int t4_detach(device_t); static int t4_child_location(device_t, device_t, struct sbuf *); static int t4_ready(device_t); static int t4_read_port_device(device_t, int, device_t *); static int t4_suspend(device_t); static int t4_resume(device_t); static int t4_reset_prepare(device_t, device_t); static int t4_reset_post(device_t, device_t); static device_method_t t4_methods[] = { DEVMETHOD(device_probe, t4_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(device_suspend, t4_suspend), DEVMETHOD(device_resume, t4_resume), DEVMETHOD(bus_child_location, t4_child_location), DEVMETHOD(bus_reset_prepare, t4_reset_prepare), DEVMETHOD(bus_reset_post, t4_reset_post), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t4_driver = { "t4nex", t4_methods, sizeof(struct adapter) }; /* T4 port (cxgbe) interface */ static int cxgbe_probe(device_t); static int cxgbe_attach(device_t); static int cxgbe_detach(device_t); device_method_t cxgbe_methods[] = { DEVMETHOD(device_probe, cxgbe_probe), DEVMETHOD(device_attach, cxgbe_attach), DEVMETHOD(device_detach, cxgbe_detach), { 0, 0 } }; static driver_t cxgbe_driver = { "cxgbe", cxgbe_methods, sizeof(struct port_info) }; /* T4 VI (vcxgbe) interface */ static int vcxgbe_probe(device_t); static int vcxgbe_attach(device_t); static int vcxgbe_detach(device_t); static device_method_t vcxgbe_methods[] = { DEVMETHOD(device_probe, vcxgbe_probe), DEVMETHOD(device_attach, vcxgbe_attach), DEVMETHOD(device_detach, vcxgbe_detach), { 0, 0 } }; static driver_t vcxgbe_driver = { "vcxgbe", vcxgbe_methods, sizeof(struct vi_info) }; static d_ioctl_t t4_ioctl; static struct cdevsw t4_cdevsw = { .d_version = D_VERSION, .d_ioctl = t4_ioctl, .d_name = "t4nex", }; /* T5 bus driver interface */ static int t5_probe(device_t); static device_method_t t5_methods[] = { DEVMETHOD(device_probe, t5_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(device_suspend, t4_suspend), DEVMETHOD(device_resume, t4_resume), DEVMETHOD(bus_child_location, t4_child_location), DEVMETHOD(bus_reset_prepare, t4_reset_prepare), DEVMETHOD(bus_reset_post, t4_reset_post), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t5_driver = { "t5nex", t5_methods, sizeof(struct adapter) }; /* T5 port (cxl) interface */ static driver_t cxl_driver = { "cxl", cxgbe_methods, sizeof(struct port_info) }; /* T5 VI (vcxl) interface */ static driver_t vcxl_driver = { "vcxl", vcxgbe_methods, sizeof(struct vi_info) }; /* T6 bus driver interface */ static int t6_probe(device_t); static device_method_t t6_methods[] = { DEVMETHOD(device_probe, t6_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(device_suspend, t4_suspend), DEVMETHOD(device_resume, t4_resume), DEVMETHOD(bus_child_location, t4_child_location), DEVMETHOD(bus_reset_prepare, t4_reset_prepare), DEVMETHOD(bus_reset_post, t4_reset_post), DEVMETHOD(t4_is_main_ready, t4_ready), DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; static driver_t t6_driver = { "t6nex", t6_methods, sizeof(struct adapter) }; /* T6 port (cc) interface */ static driver_t cc_driver = { "cc", cxgbe_methods, sizeof(struct port_info) }; /* T6 VI (vcc) interface */ static driver_t vcc_driver = { "vcc", vcxgbe_methods, sizeof(struct vi_info) }; /* ifnet interface */ static void cxgbe_init(void *); static int cxgbe_ioctl(if_t, unsigned long, caddr_t); static int cxgbe_transmit(if_t, struct mbuf *); static void cxgbe_qflush(if_t); #if defined(KERN_TLS) || defined(RATELIMIT) static int cxgbe_snd_tag_alloc(if_t, union if_snd_tag_alloc_params *, struct m_snd_tag **); #endif MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); /* * Correct lock order when you need to acquire multiple locks is t4_list_lock, * then ADAPTER_LOCK, then t4_uld_list_lock. */ static struct sx t4_list_lock; SLIST_HEAD(, adapter) t4_list; #ifdef TCP_OFFLOAD static struct sx t4_uld_list_lock; struct uld_info *t4_uld_list[ULD_MAX + 1]; #endif /* * Tunables. See tweak_tunables() too. * * Each tunable is set to a default value here if it's known at compile-time. * Otherwise it is set to -n as an indication to tweak_tunables() that it should * provide a reasonable default (upto n) when the driver is loaded. * * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to * T5 are under hw.cxl. */ SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "cxgbe(4) parameters"); SYSCTL_NODE(_hw, OID_AUTO, cxl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "cxgbe(4) T5+ parameters"); SYSCTL_NODE(_hw_cxgbe, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "cxgbe(4) TOE parameters"); /* * Number of queues for tx and rx, NIC and offload. */ #define NTXQ 16 int t4_ntxq = -NTXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq, CTLFLAG_RDTUN, &t4_ntxq, 0, "Number of TX queues per port"); TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */ #define NRXQ 8 int t4_nrxq = -NRXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq, CTLFLAG_RDTUN, &t4_nrxq, 0, "Number of RX queues per port"); TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */ #define NTXQ_VI 1 static int t4_ntxq_vi = -NTXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq_vi, CTLFLAG_RDTUN, &t4_ntxq_vi, 0, "Number of TX queues per VI"); #define NRXQ_VI 1 static int t4_nrxq_vi = -NRXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq_vi, CTLFLAG_RDTUN, &t4_nrxq_vi, 0, "Number of RX queues per VI"); static int t4_rsrv_noflowq = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq, 0, "Reserve TX queue 0 of each VI for non-flowid packets"); #if defined(TCP_OFFLOAD) || defined(RATELIMIT) #define NOFLDTXQ 8 static int t4_nofldtxq = -NOFLDTXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq, CTLFLAG_RDTUN, &t4_nofldtxq, 0, "Number of offload TX queues per port"); #define NOFLDTXQ_VI 1 static int t4_nofldtxq_vi = -NOFLDTXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq_vi, CTLFLAG_RDTUN, &t4_nofldtxq_vi, 0, "Number of offload TX queues per VI"); #endif #if defined(TCP_OFFLOAD) #define NOFLDRXQ 2 static int t4_nofldrxq = -NOFLDRXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq, CTLFLAG_RDTUN, &t4_nofldrxq, 0, "Number of offload RX queues per port"); #define NOFLDRXQ_VI 1 static int t4_nofldrxq_vi = -NOFLDRXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq_vi, CTLFLAG_RDTUN, &t4_nofldrxq_vi, 0, "Number of offload RX queues per VI"); #define TMR_IDX_OFLD 1 static int t4_tmr_idx_ofld = TMR_IDX_OFLD; SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_ofld, CTLFLAG_RDTUN, &t4_tmr_idx_ofld, 0, "Holdoff timer index for offload queues"); #define PKTC_IDX_OFLD (-1) static int t4_pktc_idx_ofld = PKTC_IDX_OFLD; SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_ofld, CTLFLAG_RDTUN, &t4_pktc_idx_ofld, 0, "holdoff packet counter index for offload queues"); /* 0 means chip/fw default, non-zero number is value in microseconds */ static u_long t4_toe_keepalive_idle = 0; SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_idle, CTLFLAG_RDTUN, &t4_toe_keepalive_idle, 0, "TOE keepalive idle timer (us)"); /* 0 means chip/fw default, non-zero number is value in microseconds */ static u_long t4_toe_keepalive_interval = 0; SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_interval, CTLFLAG_RDTUN, &t4_toe_keepalive_interval, 0, "TOE keepalive interval timer (us)"); /* 0 means chip/fw default, non-zero number is # of keepalives before abort */ static int t4_toe_keepalive_count = 0; SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, keepalive_count, CTLFLAG_RDTUN, &t4_toe_keepalive_count, 0, "Number of TOE keepalive probes before abort"); /* 0 means chip/fw default, non-zero number is value in microseconds */ static u_long t4_toe_rexmt_min = 0; SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_min, CTLFLAG_RDTUN, &t4_toe_rexmt_min, 0, "Minimum TOE retransmit interval (us)"); /* 0 means chip/fw default, non-zero number is value in microseconds */ static u_long t4_toe_rexmt_max = 0; SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_max, CTLFLAG_RDTUN, &t4_toe_rexmt_max, 0, "Maximum TOE retransmit interval (us)"); /* 0 means chip/fw default, non-zero number is # of rexmt before abort */ static int t4_toe_rexmt_count = 0; SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, rexmt_count, CTLFLAG_RDTUN, &t4_toe_rexmt_count, 0, "Number of TOE retransmissions before abort"); /* -1 means chip/fw default, other values are raw backoff values to use */ static int t4_toe_rexmt_backoff[16] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; SYSCTL_NODE(_hw_cxgbe_toe, OID_AUTO, rexmt_backoff, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "cxgbe(4) TOE retransmit backoff values"); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 0, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[0], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 1, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[1], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 2, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[2], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 3, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[3], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 4, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[4], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 5, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[5], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 6, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[6], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 7, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[7], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 8, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[8], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 9, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[9], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 10, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[10], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 11, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[11], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 12, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[12], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 13, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[13], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[14], 0, ""); SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN, &t4_toe_rexmt_backoff[15], 0, ""); int t4_ddp_rcvbuf_len = 256 * 1024; SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_len, CTLFLAG_RWTUN, &t4_ddp_rcvbuf_len, 0, "length of each DDP RX buffer"); unsigned int t4_ddp_rcvbuf_cache = 4; SYSCTL_UINT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_cache, CTLFLAG_RWTUN, &t4_ddp_rcvbuf_cache, 0, "maximum number of free DDP RX buffers to cache per connection"); #endif #ifdef DEV_NETMAP #define NN_MAIN_VI (1 << 0) /* Native netmap on the main VI */ #define NN_EXTRA_VI (1 << 1) /* Native netmap on the extra VI(s) */ static int t4_native_netmap = NN_EXTRA_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, native_netmap, CTLFLAG_RDTUN, &t4_native_netmap, 0, "Native netmap support. bit 0 = main VI, bit 1 = extra VIs"); #define NNMTXQ 8 static int t4_nnmtxq = -NNMTXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq, CTLFLAG_RDTUN, &t4_nnmtxq, 0, "Number of netmap TX queues"); #define NNMRXQ 8 static int t4_nnmrxq = -NNMRXQ; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq, CTLFLAG_RDTUN, &t4_nnmrxq, 0, "Number of netmap RX queues"); #define NNMTXQ_VI 2 static int t4_nnmtxq_vi = -NNMTXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq_vi, CTLFLAG_RDTUN, &t4_nnmtxq_vi, 0, "Number of netmap TX queues per VI"); #define NNMRXQ_VI 2 static int t4_nnmrxq_vi = -NNMRXQ_VI; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq_vi, CTLFLAG_RDTUN, &t4_nnmrxq_vi, 0, "Number of netmap RX queues per VI"); #endif /* * Holdoff parameters for ports. */ #define TMR_IDX 1 int t4_tmr_idx = TMR_IDX; SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx, CTLFLAG_RDTUN, &t4_tmr_idx, 0, "Holdoff timer index"); TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */ #define PKTC_IDX (-1) int t4_pktc_idx = PKTC_IDX; SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx, CTLFLAG_RDTUN, &t4_pktc_idx, 0, "Holdoff packet counter index"); TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */ /* * Size (# of entries) of each tx and rx queue. */ unsigned int t4_qsize_txq = TX_EQ_QSIZE; SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, &t4_qsize_txq, 0, "Number of descriptors in each TX queue"); unsigned int t4_qsize_rxq = RX_IQ_QSIZE; SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, &t4_qsize_rxq, 0, "Number of descriptors in each RX queue"); /* * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). */ int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; SYSCTL_INT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &t4_intr_types, 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)"); /* * Configuration file. All the _CF names here are special. */ #define DEFAULT_CF "default" #define BUILTIN_CF "built-in" #define FLASH_CF "flash" #define UWIRE_CF "uwire" #define FPGA_CF "fpga" static char t4_cfg_file[32] = DEFAULT_CF; SYSCTL_STRING(_hw_cxgbe, OID_AUTO, config_file, CTLFLAG_RDTUN, t4_cfg_file, sizeof(t4_cfg_file), "Firmware configuration file"); /* * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively). * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water * mark or when signalled to do so, 0 to never emit PAUSE. * pause_autoneg = 1 means PAUSE will be negotiated if possible and the * negotiated settings will override rx_pause/tx_pause. * Otherwise rx_pause/tx_pause are applied forcibly. */ static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG; SYSCTL_INT(_hw_cxgbe, OID_AUTO, pause_settings, CTLFLAG_RDTUN, &t4_pause_settings, 0, "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)"); /* * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively). * -1 to run with the firmware default. Same as FEC_AUTO (bit 5) * 0 to disable FEC. */ static int t4_fec = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0, "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)"); /* * Controls when the driver sets the FORCE_FEC bit in the L1_CFG32 that it * issues to the firmware. If the firmware doesn't support FORCE_FEC then the * driver runs as if this is set to 0. * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay. * 0 to never set FORCE_FEC. requested_fec = AUTO means use the hint from the * transceiver. Multiple FEC bits may not be okay but will be passed on to * the firmware anyway (may result in l1cfg errors with old firmwares). * 1 to always set FORCE_FEC. Multiple FEC bits are okay. requested_fec = AUTO * means set all FEC bits that are valid for the speed. */ static int t4_force_fec = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, force_fec, CTLFLAG_RDTUN, &t4_force_fec, 0, "Controls the use of FORCE_FEC bit in L1 configuration."); /* * Link autonegotiation. * -1 to run with the firmware default. * 0 to disable. * 1 to enable. */ static int t4_autoneg = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, autoneg, CTLFLAG_RDTUN, &t4_autoneg, 0, "Link autonegotiation"); /* * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, * encouraged respectively). '-n' is the same as 'n' except the firmware * version used in the checks is read from the firmware bundled with the driver. */ static int t4_fw_install = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fw_install, CTLFLAG_RDTUN, &t4_fw_install, 0, "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)"); /* * ASIC features that will be used. Disable the ones you don't want so that the * chip resources aren't wasted on features that will not be used. */ static int t4_nbmcaps_allowed = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, nbmcaps_allowed, CTLFLAG_RDTUN, &t4_nbmcaps_allowed, 0, "Default NBM capabilities"); static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ SYSCTL_INT(_hw_cxgbe, OID_AUTO, linkcaps_allowed, CTLFLAG_RDTUN, &t4_linkcaps_allowed, 0, "Default link capabilities"); static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS | FW_CAPS_CONFIG_SWITCH_EGRESS; SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN, &t4_switchcaps_allowed, 0, "Default switch capabilities"); #ifdef RATELIMIT static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD; #else static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC | FW_CAPS_CONFIG_NIC_HASHFILTER; #endif SYSCTL_INT(_hw_cxgbe, OID_AUTO, niccaps_allowed, CTLFLAG_RDTUN, &t4_niccaps_allowed, 0, "Default NIC capabilities"); static int t4_toecaps_allowed = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, toecaps_allowed, CTLFLAG_RDTUN, &t4_toecaps_allowed, 0, "Default TCP offload capabilities"); static int t4_rdmacaps_allowed = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, rdmacaps_allowed, CTLFLAG_RDTUN, &t4_rdmacaps_allowed, 0, "Default RDMA capabilities"); static int t4_cryptocaps_allowed = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, cryptocaps_allowed, CTLFLAG_RDTUN, &t4_cryptocaps_allowed, 0, "Default crypto capabilities"); static int t4_iscsicaps_allowed = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, iscsicaps_allowed, CTLFLAG_RDTUN, &t4_iscsicaps_allowed, 0, "Default iSCSI capabilities"); static int t4_fcoecaps_allowed = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, fcoecaps_allowed, CTLFLAG_RDTUN, &t4_fcoecaps_allowed, 0, "Default FCoE capabilities"); static int t5_write_combine = 0; SYSCTL_INT(_hw_cxl, OID_AUTO, write_combine, CTLFLAG_RDTUN, &t5_write_combine, 0, "Use WC instead of UC for BAR2"); /* From t4_sysctls: doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"} */ static int t4_doorbells_allowed = 0xf; SYSCTL_INT(_hw_cxgbe, OID_AUTO, doorbells_allowed, CTLFLAG_RDTUN, &t4_doorbells_allowed, 0, "Limit tx queues to these doorbells"); static int t4_num_vis = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, num_vis, CTLFLAG_RDTUN, &t4_num_vis, 0, "Number of VIs per port"); /* * PCIe Relaxed Ordering. * -1: driver should figure out a good value. * 0: disable RO. * 1: enable RO. * 2: leave RO alone. */ static int pcie_relaxed_ordering = -1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, pcie_relaxed_ordering, CTLFLAG_RDTUN, &pcie_relaxed_ordering, 0, "PCIe Relaxed Ordering: 0 = disable, 1 = enable, 2 = leave alone"); static int t4_panic_on_fatal_err = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, panic_on_fatal_err, CTLFLAG_RWTUN, &t4_panic_on_fatal_err, 0, "panic on fatal errors"); static int t4_reset_on_fatal_err = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, reset_on_fatal_err, CTLFLAG_RWTUN, &t4_reset_on_fatal_err, 0, "reset adapter on fatal errors"); static int t4_clock_gate_on_suspend = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, clock_gate_on_suspend, CTLFLAG_RWTUN, &t4_clock_gate_on_suspend, 0, "gate the clock on suspend"); static int t4_tx_vm_wr = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_vm_wr, CTLFLAG_RWTUN, &t4_tx_vm_wr, 0, "Use VM work requests to transmit packets."); /* * Set to non-zero to enable the attack filter. A packet that matches any of * these conditions will get dropped on ingress: * 1) IP && source address == destination address. * 2) TCP/IP && source address is not a unicast address. * 3) TCP/IP && destination address is not a unicast address. * 4) IP && source address is loopback (127.x.y.z). * 5) IP && destination address is loopback (127.x.y.z). * 6) IPv6 && source address == destination address. * 7) IPv6 && source address is not a unicast address. * 8) IPv6 && source address is loopback (::1/128). * 9) IPv6 && destination address is loopback (::1/128). * 10) IPv6 && source address is unspecified (::/128). * 11) IPv6 && destination address is unspecified (::/128). * 12) TCP/IPv6 && source address is multicast (ff00::/8). * 13) TCP/IPv6 && destination address is multicast (ff00::/8). */ static int t4_attack_filter = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, attack_filter, CTLFLAG_RDTUN, &t4_attack_filter, 0, "Drop suspicious traffic"); static int t4_drop_ip_fragments = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_ip_fragments, CTLFLAG_RDTUN, &t4_drop_ip_fragments, 0, "Drop IP fragments"); static int t4_drop_pkts_with_l2_errors = 1; SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l2_errors, CTLFLAG_RDTUN, &t4_drop_pkts_with_l2_errors, 0, "Drop all frames with Layer 2 length or checksum errors"); static int t4_drop_pkts_with_l3_errors = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l3_errors, CTLFLAG_RDTUN, &t4_drop_pkts_with_l3_errors, 0, "Drop all frames with IP version, length, or checksum errors"); static int t4_drop_pkts_with_l4_errors = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l4_errors, CTLFLAG_RDTUN, &t4_drop_pkts_with_l4_errors, 0, "Drop all frames with Layer 4 length, checksum, or other errors"); #ifdef TCP_OFFLOAD /* * TOE tunables. */ static int t4_cop_managed_offloading = 0; SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN, &t4_cop_managed_offloading, 0, "COP (Connection Offload Policy) controls all TOE offload"); TUNABLE_INT("hw.cxgbe.cop_managed_offloading", &t4_cop_managed_offloading); #endif #ifdef KERN_TLS /* * This enables KERN_TLS for all adapters if set. */ static int t4_kern_tls = 0; SYSCTL_INT(_hw_cxgbe, OID_AUTO, kern_tls, CTLFLAG_RDTUN, &t4_kern_tls, 0, "Enable KERN_TLS mode for T6 adapters"); SYSCTL_NODE(_hw_cxgbe, OID_AUTO, tls, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "cxgbe(4) KERN_TLS parameters"); static int t4_tls_inline_keys = 0; SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, inline_keys, CTLFLAG_RDTUN, &t4_tls_inline_keys, 0, "Always pass TLS keys in work requests (1) or attempt to store TLS keys " "in card memory."); static int t4_tls_combo_wrs = 0; SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs, 0, "Attempt to combine TCB field updates with TLS record work requests."); #endif /* Functions used by VIs to obtain unique MAC addresses for each VI. */ static int vi_mac_funcs[] = { FW_VI_FUNC_ETH, FW_VI_FUNC_OFLD, FW_VI_FUNC_IWARP, FW_VI_FUNC_OPENISCSI, FW_VI_FUNC_OPENFCOE, FW_VI_FUNC_FOISCSI, FW_VI_FUNC_FOFCOE, }; struct intrs_and_queues { uint16_t intr_type; /* INTx, MSI, or MSI-X */ uint16_t num_vis; /* number of VIs for each port */ uint16_t nirq; /* Total # of vectors */ uint16_t ntxq; /* # of NIC txq's for each port */ uint16_t nrxq; /* # of NIC rxq's for each port */ uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */ uint16_t nofldrxq; /* # of TOE rxq's for each port */ uint16_t nnmtxq; /* # of netmap txq's */ uint16_t nnmrxq; /* # of netmap rxq's */ /* The vcxgbe/vcxl interfaces use these and not the ones above. */ uint16_t ntxq_vi; /* # of NIC txq's */ uint16_t nrxq_vi; /* # of NIC rxq's */ uint16_t nofldtxq_vi; /* # of TOE txq's */ uint16_t nofldrxq_vi; /* # of TOE rxq's */ uint16_t nnmtxq_vi; /* # of netmap txq's */ uint16_t nnmrxq_vi; /* # of netmap rxq's */ }; static void setup_memwin(struct adapter *); static void position_memwin(struct adapter *, int, uint32_t); static int validate_mem_range(struct adapter *, uint32_t, uint32_t); static int fwmtype_to_hwmtype(int); static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t, uint32_t *); static int fixup_devlog_params(struct adapter *); static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *); static int contact_firmware(struct adapter *); static int partition_resources(struct adapter *); static int get_params__pre_init(struct adapter *); static int set_params__pre_init(struct adapter *); static int get_params__post_init(struct adapter *); static int set_params__post_init(struct adapter *); static void t4_set_desc(struct adapter *); static bool fixed_ifmedia(struct port_info *); static void build_medialist(struct port_info *); static void init_link_config(struct port_info *); static int fixup_link_config(struct port_info *); static int apply_link_config(struct port_info *); static int cxgbe_init_synchronized(struct vi_info *); static int cxgbe_uninit_synchronized(struct vi_info *); static int adapter_full_init(struct adapter *); static void adapter_full_uninit(struct adapter *); static int vi_full_init(struct vi_info *); static void vi_full_uninit(struct vi_info *); static int alloc_extra_vi(struct adapter *, struct port_info *, struct vi_info *); static void quiesce_txq(struct sge_txq *); static void quiesce_wrq(struct sge_wrq *); static void quiesce_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *); static void quiesce_vi(struct vi_info *); static int t4_alloc_irq(struct adapter *, struct irq *, int rid, driver_intr_t *, void *, char *); static int t4_free_irq(struct adapter *, struct irq *); static void t4_init_atid_table(struct adapter *); static void t4_free_atid_table(struct adapter *); static void stop_atid_allocator(struct adapter *); static void restart_atid_allocator(struct adapter *); static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *); static void vi_refresh_stats(struct vi_info *); static void cxgbe_refresh_stats(struct vi_info *); static void cxgbe_tick(void *); static void vi_tick(void *); static void cxgbe_sysctls(struct port_info *); static int sysctl_int_array(SYSCTL_HANDLER_ARGS); static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS); static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS); static int sysctl_btphy(SYSCTL_HANDLER_ARGS); static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); static int sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); static int sysctl_link_fec(SYSCTL_HANDLER_ARGS); static int sysctl_requested_fec(SYSCTL_HANDLER_ARGS); static int sysctl_module_fec(SYSCTL_HANDLER_ARGS); static int sysctl_autoneg(SYSCTL_HANDLER_ARGS); static int sysctl_force_fec(SYSCTL_HANDLER_ARGS); static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); static int sysctl_temperature(SYSCTL_HANDLER_ARGS); static int sysctl_vdd(SYSCTL_HANDLER_ARGS); static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS); static int sysctl_loadavg(SYSCTL_HANDLER_ARGS); static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tid_stats(SYSCTL_HANDLER_ARGS); static int sysctl_devlog(SYSCTL_HANDLER_ARGS); static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tids(SYSCTL_HANDLER_ARGS); static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tnl_stats(SYSCTL_HANDLER_ARGS); static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS); static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); static int sysctl_cpus(SYSCTL_HANDLER_ARGS); static int sysctl_reset(SYSCTL_HANDLER_ARGS); #ifdef TCP_OFFLOAD static int sysctl_tls(SYSCTL_HANDLER_ARGS); static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS); static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS); static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS); #endif static int get_sge_context(struct adapter *, struct t4_sge_context *); static int load_fw(struct adapter *, struct t4_data *); static int load_cfg(struct adapter *, struct t4_data *); static int load_boot(struct adapter *, struct t4_bootrom *); static int load_bootcfg(struct adapter *, struct t4_data *); static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *); static void free_offload_policy(struct t4_offload_policy *); static int set_offload_policy(struct adapter *, struct t4_offload_policy *); static int read_card_mem(struct adapter *, int, struct t4_mem_range *); static int read_i2c(struct adapter *, struct t4_i2c_data *); static int clear_stats(struct adapter *, u_int); static int hold_clip_addr(struct adapter *, struct t4_clip_addr *); static int release_clip_addr(struct adapter *, struct t4_clip_addr *); static inline int stop_adapter(struct adapter *); static inline void set_adapter_hwstatus(struct adapter *, const bool); static int stop_lld(struct adapter *); static inline int restart_adapter(struct adapter *); static int restart_lld(struct adapter *); #ifdef TCP_OFFLOAD static int deactivate_all_uld(struct adapter *); static void stop_all_uld(struct adapter *); static void restart_all_uld(struct adapter *); #endif #ifdef KERN_TLS static int ktls_capability(struct adapter *, bool); #endif static int mod_event(module_t, int, void *); static int notify_siblings(device_t, int); static uint64_t vi_get_counter(if_t, ift_counter); static uint64_t cxgbe_get_counter(if_t, ift_counter); static void enable_vxlan_rx(struct adapter *); static void reset_adapter_task(void *, int); static void fatal_error_task(void *, int); static void dump_devlog(struct adapter *); static void dump_cim_regs(struct adapter *); static void dump_cimla(struct adapter *); struct { uint16_t device; char *desc; } t4_pciids[] = { {0xa000, "Chelsio Terminator 4 FPGA"}, {0x4400, "Chelsio T440-dbg"}, {0x4401, "Chelsio T420-CR"}, {0x4402, "Chelsio T422-CR"}, {0x4403, "Chelsio T440-CR"}, {0x4404, "Chelsio T420-BCH"}, {0x4405, "Chelsio T440-BCH"}, {0x4406, "Chelsio T440-CH"}, {0x4407, "Chelsio T420-SO"}, {0x4408, "Chelsio T420-CX"}, {0x4409, "Chelsio T420-BT"}, {0x440a, "Chelsio T404-BT"}, {0x440e, "Chelsio T440-LP-CR"}, }, t5_pciids[] = { {0xb000, "Chelsio Terminator 5 FPGA"}, {0x5400, "Chelsio T580-dbg"}, {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */ {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */ {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */ {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */ /* Custom */ {0x5483, "Custom T540-CR"}, {0x5484, "Custom T540-BT"}, }, t6_pciids[] = { {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */ {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ {0x6405, "Chelsio T6225-SO-OCP3"}, /* 2 x 10/25G, nomem */ {0x6406, "Chelsio T6225-OCP3"}, /* 2 x 10/25G */ {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ {0x6414, "Chelsio T62100-SO-OCP3"}, /* 2 x 40/50/100G, nomem */ {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ /* Custom */ {0x6480, "Custom T6225-CR"}, {0x6481, "Custom T62100-CR"}, {0x6482, "Custom T6225-CR"}, {0x6483, "Custom T62100-CR"}, {0x6484, "Custom T64100-CR"}, {0x6485, "Custom T6240-SO"}, {0x6486, "Custom T6225-SO-CR"}, {0x6487, "Custom T6225-CR"}, }; #ifdef TCP_OFFLOAD /* * service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should * be exactly the same for both rxq and ofld_rxq. */ CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); #endif CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); static int t4_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); uint8_t f = pci_get_function(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); /* Attach only to PF0 of the FPGA */ if (d == 0xa000 && f != 0) return (ENXIO); for (i = 0; i < nitems(t4_pciids); i++) { if (d == t4_pciids[i].device) { device_set_desc(dev, t4_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t5_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); uint8_t f = pci_get_function(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); /* Attach only to PF0 of the FPGA */ if (d == 0xb000 && f != 0) return (ENXIO); for (i = 0; i < nitems(t5_pciids); i++) { if (d == t5_pciids[i].device) { device_set_desc(dev, t5_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t6_probe(device_t dev) { int i; uint16_t v = pci_get_vendor(dev); uint16_t d = pci_get_device(dev); if (v != PCI_VENDOR_ID_CHELSIO) return (ENXIO); for (i = 0; i < nitems(t6_pciids); i++) { if (d == t6_pciids[i].device) { device_set_desc(dev, t6_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static void t5_attribute_workaround(device_t dev) { device_t root_port; uint32_t v; /* * The T5 chips do not properly echo the No Snoop and Relaxed * Ordering attributes when replying to a TLP from a Root * Port. As a workaround, find the parent Root Port and * disable No Snoop and Relaxed Ordering. Note that this * affects all devices under this root port. */ root_port = pci_find_pcie_root_port(dev); if (root_port == NULL) { device_printf(dev, "Unable to find parent root port\n"); return; } v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 0) device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", device_get_nameunit(root_port)); } static const struct devnames devnames[] = { { .nexus_name = "t4nex", .ifnet_name = "cxgbe", .vi_ifnet_name = "vcxgbe", .pf03_drv_name = "t4iov", .vf_nexus_name = "t4vf", .vf_ifnet_name = "cxgbev" }, { .nexus_name = "t5nex", .ifnet_name = "cxl", .vi_ifnet_name = "vcxl", .pf03_drv_name = "t5iov", .vf_nexus_name = "t5vf", .vf_ifnet_name = "cxlv" }, { .nexus_name = "t6nex", .ifnet_name = "cc", .vi_ifnet_name = "vcc", .pf03_drv_name = "t6iov", .vf_nexus_name = "t6vf", .vf_ifnet_name = "ccv" } }; void t4_init_devnames(struct adapter *sc) { int id; id = chip_id(sc); if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames)) sc->names = &devnames[id - CHELSIO_T4]; else { device_printf(sc->dev, "chip id %d is not supported.\n", id); sc->names = NULL; } } static int t4_ifnet_unit(struct adapter *sc, struct port_info *pi) { const char *parent, *name; long value; int line, unit; line = 0; parent = device_get_nameunit(sc->dev); name = sc->names->ifnet_name; while (resource_find_dev(&line, name, &unit, "at", parent) == 0) { if (resource_long_value(name, unit, "port", &value) == 0 && value == pi->port_id) return (unit); } return (-1); } static void t4_calibration(void *arg) { struct adapter *sc; struct clock_sync *cur, *nex; uint64_t hw; sbintime_t sbt; int next_up; sc = (struct adapter *)arg; KASSERT((hw_off_limits(sc) == 0), ("hw_off_limits at t4_calibration")); hw = t4_read_reg64(sc, A_SGE_TIMESTAMP_LO); sbt = sbinuptime(); cur = &sc->cal_info[sc->cal_current]; next_up = (sc->cal_current + 1) % CNT_CAL_INFO; nex = &sc->cal_info[next_up]; if (__predict_false(sc->cal_count == 0)) { /* First time in, just get the values in */ cur->hw_cur = hw; cur->sbt_cur = sbt; sc->cal_count++; goto done; } if (cur->hw_cur == hw) { /* The clock is not advancing? */ sc->cal_count = 0; atomic_store_rel_int(&cur->gen, 0); goto done; } seqc_write_begin(&nex->gen); nex->hw_prev = cur->hw_cur; nex->sbt_prev = cur->sbt_cur; nex->hw_cur = hw; nex->sbt_cur = sbt; seqc_write_end(&nex->gen); sc->cal_current = next_up; done: callout_reset_sbt_curcpu(&sc->cal_callout, SBT_1S, 0, t4_calibration, sc, C_DIRECT_EXEC); } static void t4_calibration_start(struct adapter *sc) { /* * Here if we have not done a calibration * then do so otherwise start the appropriate * timer. */ int i; for (i = 0; i < CNT_CAL_INFO; i++) { sc->cal_info[i].gen = 0; } sc->cal_current = 0; sc->cal_count = 0; sc->cal_gen = 0; t4_calibration(sc); } static int t4_attach(device_t dev) { struct adapter *sc; int rc = 0, i, j, rqidx, tqidx, nports; struct make_dev_args mda; struct intrs_and_queues iaq; struct sge *s; uint32_t *buf; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) int ofld_tqidx; #endif #ifdef TCP_OFFLOAD int ofld_rqidx; #endif #ifdef DEV_NETMAP int nm_rqidx, nm_tqidx; #endif int num_vis; sc = device_get_softc(dev); sc->dev = dev; sysctl_ctx_init(&sc->ctx); TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); if ((pci_get_device(dev) & 0xff00) == 0x5400) t5_attribute_workaround(dev); pci_enable_busmaster(dev); if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { uint32_t v; pci_set_max_read_req(dev, 4096); v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); if (pcie_relaxed_ordering == 0 && (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) { v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE; pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); } else if (pcie_relaxed_ordering == 1 && (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) { v |= PCIEM_CTL_RELAXED_ORD_ENABLE; pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); } } sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); sc->traceq = -1; mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", device_get_nameunit(dev)); snprintf(sc->lockname, sizeof(sc->lockname), "%s", device_get_nameunit(dev)); mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); t4_add_adapter(sc); mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); TAILQ_INIT(&sc->sfl); callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); sc->policy = NULL; rw_init(&sc->policy_lock, "connection offload policy"); callout_init(&sc->ktls_tick, 1); callout_init(&sc->cal_callout, 1); refcount_init(&sc->vxlan_refcount, 0); TASK_INIT(&sc->reset_task, 0, reset_adapter_task, sc); TASK_INIT(&sc->fatal_error_task, 0, fatal_error_task, sc); sc->ctrlq_oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "ctrlq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "control queues"); sc->fwq_oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "fwq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "firmware event queue"); rc = t4_map_bars_0_and_4(sc); if (rc != 0) goto done; /* error message displayed already */ memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); /* Prepare the adapter for operation. */ buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); rc = -t4_prep_adapter(sc, buf); free(buf, M_CXGBE); if (rc != 0) { device_printf(dev, "failed to prepare adapter: %d.\n", rc); goto done; } /* * This is the real PF# to which we're attaching. Works from within PCI * passthrough environments too, where pci_get_function() could return a * different PF# depending on the passthrough configuration. We need to * use the real PF# in all our communication with the firmware. */ j = t4_read_reg(sc, A_PL_WHOAMI); sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j); sc->mbox = sc->pf; t4_init_devnames(sc); if (sc->names == NULL) { rc = ENOTSUP; goto done; /* error message displayed already */ } /* * Do this really early, with the memory windows set up even before the * character device. The userland tool's register i/o and mem read * will work even in "recovery mode". */ setup_memwin(sc); if (t4_init_devlog_params(sc, 0) == 0) fixup_devlog_params(sc); make_dev_args_init(&mda); mda.mda_devsw = &t4_cdevsw; mda.mda_uid = UID_ROOT; mda.mda_gid = GID_WHEEL; mda.mda_mode = 0600; mda.mda_si_drv1 = sc; rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); if (rc != 0) device_printf(dev, "failed to create nexus char device: %d.\n", rc); /* Go no further if recovery mode has been requested. */ if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { device_printf(dev, "recovery mode.\n"); goto done; } #if defined(__i386__) if ((cpu_feature & CPUID_CX8) == 0) { device_printf(dev, "64 bit atomics not available.\n"); rc = ENOTSUP; goto done; } #endif /* Contact the firmware and try to become the master driver. */ rc = contact_firmware(sc); if (rc != 0) goto done; /* error message displayed already */ MPASS(sc->flags & FW_OK); rc = get_params__pre_init(sc); if (rc != 0) goto done; /* error message displayed already */ if (sc->flags & MASTER_PF) { rc = partition_resources(sc); if (rc != 0) goto done; /* error message displayed already */ } rc = get_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = set_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_map_bar_2(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_adj_doorbells(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_create_dma_tag(sc); if (rc != 0) goto done; /* error message displayed already */ /* * First pass over all the ports - allocate VIs and initialize some * basic parameters like mac address, port type, etc. */ for_each_port(sc, i) { struct port_info *pi; pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); sc->port[i] = pi; /* These must be set before t4_port_init */ pi->adapter = sc; pi->port_id = i; /* * XXX: vi[0] is special so we can't delay this allocation until * pi->nvi's final value is known. */ pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE, M_ZERO | M_WAITOK); /* * Allocate the "main" VI and initialize parameters * like mac addr. */ rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); if (rc != 0) { device_printf(dev, "unable to initialize port %d: %d\n", i, rc); free(pi->vi, M_CXGBE); free(pi, M_CXGBE); sc->port[i] = NULL; goto done; } if (is_bt(pi->port_type)) setbit(&sc->bt_map, pi->tx_chan); else MPASS(!isset(&sc->bt_map, pi->tx_chan)); snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", device_get_nameunit(dev), i); mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); sc->chan_map[pi->tx_chan] = i; /* * The MPS counter for FCS errors doesn't work correctly on the * T6 so we use the MAC counter here. Which MAC is in use * depends on the link settings which will be known when the * link comes up. */ if (is_t6(sc)) pi->fcs_reg = -1; else { pi->fcs_reg = t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L); } pi->fcs_base = 0; /* All VIs on this port share this media. */ ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, cxgbe_media_status); PORT_LOCK(pi); init_link_config(pi); fixup_link_config(pi); build_medialist(pi); if (fixed_ifmedia(pi)) pi->flags |= FIXED_IFMEDIA; PORT_UNLOCK(pi); pi->dev = device_add_child(dev, sc->names->ifnet_name, t4_ifnet_unit(sc, pi)); if (pi->dev == NULL) { device_printf(dev, "failed to add device for port %d.\n", i); rc = ENXIO; goto done; } pi->vi[0].dev = pi->dev; device_set_softc(pi->dev, pi); } /* * Interrupt type, # of interrupts, # of rx/tx queues, etc. */ nports = sc->params.nports; rc = cfg_itype_and_nqueues(sc, &iaq); if (rc != 0) goto done; /* error message displayed already */ num_vis = iaq.num_vis; sc->intr_type = iaq.intr_type; sc->intr_count = iaq.nirq; s = &sc->sge; s->nrxq = nports * iaq.nrxq; s->ntxq = nports * iaq.ntxq; if (num_vis > 1) { s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi; s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi; } s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ s->neq += nports; /* ctrl queues: 1 per port */ s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ #if defined(TCP_OFFLOAD) || defined(RATELIMIT) if (is_offload(sc) || is_ethoffload(sc)) { s->nofldtxq = nports * iaq.nofldtxq; if (num_vis > 1) s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi; s->neq += s->nofldtxq; s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_ofld_txq), M_CXGBE, M_ZERO | M_WAITOK); } #endif #ifdef TCP_OFFLOAD if (is_offload(sc)) { s->nofldrxq = nports * iaq.nofldrxq; if (num_vis > 1) s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi; s->neq += s->nofldrxq; /* free list */ s->niq += s->nofldrxq; s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), M_CXGBE, M_ZERO | M_WAITOK); } #endif #ifdef DEV_NETMAP s->nnmrxq = 0; s->nnmtxq = 0; if (t4_native_netmap & NN_MAIN_VI) { s->nnmrxq += nports * iaq.nnmrxq; s->nnmtxq += nports * iaq.nnmtxq; } if (num_vis > 1 && t4_native_netmap & NN_EXTRA_VI) { s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi; s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi; } s->neq += s->nnmtxq + s->nnmrxq; s->niq += s->nnmrxq; s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), M_CXGBE, M_ZERO | M_WAITOK); #endif MPASS(s->niq <= s->iqmap_sz); MPASS(s->neq <= s->eqmap_sz); s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE, M_ZERO | M_WAITOK); s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, M_ZERO | M_WAITOK); s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE, M_ZERO | M_WAITOK); s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE, M_ZERO | M_WAITOK); sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, M_ZERO | M_WAITOK); t4_init_l2t(sc, M_WAITOK); t4_init_smt(sc, M_WAITOK); t4_init_tx_sched(sc); t4_init_atid_table(sc); #ifdef RATELIMIT t4_init_etid_table(sc); #endif #ifdef INET6 t4_init_clip_table(sc); #endif if (sc->vres.key.size != 0) sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start, sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK); /* * Second pass over the ports. This time we know the number of rx and * tx queues that each port should get. */ rqidx = tqidx = 0; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) ofld_tqidx = 0; #endif #ifdef TCP_OFFLOAD ofld_rqidx = 0; #endif #ifdef DEV_NETMAP nm_rqidx = nm_tqidx = 0; #endif for_each_port(sc, i) { struct port_info *pi = sc->port[i]; struct vi_info *vi; if (pi == NULL) continue; pi->nvi = num_vis; for_each_vi(pi, j, vi) { vi->pi = pi; vi->adapter = sc; vi->first_intr = -1; vi->qsize_rxq = t4_qsize_rxq; vi->qsize_txq = t4_qsize_txq; vi->first_rxq = rqidx; vi->first_txq = tqidx; vi->tmr_idx = t4_tmr_idx; vi->pktc_idx = t4_pktc_idx; vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi; vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi; rqidx += vi->nrxq; tqidx += vi->ntxq; if (j == 0 && vi->ntxq > 1) vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0; else vi->rsrv_noflowq = 0; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) vi->first_ofld_txq = ofld_tqidx; vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi; ofld_tqidx += vi->nofldtxq; #endif #ifdef TCP_OFFLOAD vi->ofld_tmr_idx = t4_tmr_idx_ofld; vi->ofld_pktc_idx = t4_pktc_idx_ofld; vi->first_ofld_rxq = ofld_rqidx; vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi; ofld_rqidx += vi->nofldrxq; #endif #ifdef DEV_NETMAP vi->first_nm_rxq = nm_rqidx; vi->first_nm_txq = nm_tqidx; if (j == 0) { vi->nnmrxq = iaq.nnmrxq; vi->nnmtxq = iaq.nnmtxq; } else { vi->nnmrxq = iaq.nnmrxq_vi; vi->nnmtxq = iaq.nnmtxq_vi; } nm_rqidx += vi->nnmrxq; nm_tqidx += vi->nnmtxq; #endif } } rc = t4_setup_intr_handlers(sc); if (rc != 0) { device_printf(dev, "failed to setup interrupt handlers: %d\n", rc); goto done; } bus_identify_children(dev); /* * Ensure thread-safe mailbox access (in debug builds). * * So far this was the only thread accessing the mailbox but various * ifnets and sysctls are about to be created and their handlers/ioctls * will access the mailbox from different threads. */ sc->flags |= CHK_MBOX_ACCESS; bus_attach_children(dev); t4_calibration_start(sc); device_printf(dev, "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", sc->params.pci.speed, sc->params.pci.width, sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); t4_set_desc(sc); notify_siblings(dev, 0); done: if (rc != 0 && sc->cdev) { /* cdev was created and so cxgbetool works; recover that way. */ device_printf(dev, "error during attach, adapter is now in recovery mode.\n"); rc = 0; } if (rc != 0) t4_detach_common(dev); else t4_sysctls(sc); return (rc); } static int t4_child_location(device_t bus, device_t dev, struct sbuf *sb) { struct adapter *sc; struct port_info *pi; int i; sc = device_get_softc(bus); for_each_port(sc, i) { pi = sc->port[i]; if (pi != NULL && pi->dev == dev) { sbuf_printf(sb, "port=%d", pi->port_id); break; } } return (0); } static int t4_ready(device_t dev) { struct adapter *sc; sc = device_get_softc(dev); if (sc->flags & FW_OK) return (0); return (ENXIO); } static int t4_read_port_device(device_t dev, int port, device_t *child) { struct adapter *sc; struct port_info *pi; sc = device_get_softc(dev); if (port < 0 || port >= MAX_NPORTS) return (EINVAL); pi = sc->port[port]; if (pi == NULL || pi->dev == NULL) return (ENXIO); *child = pi->dev; return (0); } static int notify_siblings(device_t dev, int detaching) { device_t sibling; int error, i; error = 0; for (i = 0; i < PCI_FUNCMAX; i++) { if (i == pci_get_function(dev)) continue; sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), i); if (sibling == NULL || !device_is_attached(sibling)) continue; if (detaching) error = T4_DETACH_CHILD(sibling); else (void)T4_ATTACH_CHILD(sibling); if (error) break; } return (error); } /* * Idempotent */ static int t4_detach(device_t dev) { int rc; rc = notify_siblings(dev, 1); if (rc) { device_printf(dev, "failed to detach sibling devices: %d\n", rc); return (rc); } return (t4_detach_common(dev)); } int t4_detach_common(device_t dev) { struct adapter *sc; struct port_info *pi; int i, rc; sc = device_get_softc(dev); #ifdef TCP_OFFLOAD rc = deactivate_all_uld(sc); if (rc) { device_printf(dev, "failed to detach upper layer drivers: %d\n", rc); return (rc); } #endif if (sc->cdev) { destroy_dev(sc->cdev); sc->cdev = NULL; } sx_xlock(&t4_list_lock); SLIST_REMOVE(&t4_list, sc, adapter, link); sx_xunlock(&t4_list_lock); sc->flags &= ~CHK_MBOX_ACCESS; if (sc->flags & FULL_INIT_DONE) { if (!(sc->flags & IS_VF)) t4_intr_disable(sc); } if (device_is_attached(dev)) { rc = bus_detach_children(dev); if (rc) { device_printf(dev, "failed to detach child devices: %d\n", rc); return (rc); } } for (i = 0; i < sc->intr_count; i++) t4_free_irq(sc, &sc->irq[i]); if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) t4_free_tx_sched(sc); for (i = 0; i < MAX_NPORTS; i++) { pi = sc->port[i]; if (pi) { t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); mtx_destroy(&pi->pi_lock); free(pi->vi, M_CXGBE); free(pi, M_CXGBE); } } callout_stop(&sc->cal_callout); callout_drain(&sc->cal_callout); device_delete_children(dev); sysctl_ctx_free(&sc->ctx); adapter_full_uninit(sc); if ((sc->flags & (IS_VF | FW_OK)) == FW_OK) t4_fw_bye(sc, sc->mbox); if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) pci_release_msi(dev); if (sc->regs_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, sc->regs_res); if (sc->udbs_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, sc->udbs_res); if (sc->msix_res) bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, sc->msix_res); if (sc->l2t) t4_free_l2t(sc); if (sc->smt) t4_free_smt(sc->smt); t4_free_atid_table(sc); #ifdef RATELIMIT t4_free_etid_table(sc); #endif if (sc->key_map) vmem_destroy(sc->key_map); #ifdef INET6 t4_destroy_clip_table(sc); #endif #if defined(TCP_OFFLOAD) || defined(RATELIMIT) free(sc->sge.ofld_txq, M_CXGBE); #endif #ifdef TCP_OFFLOAD free(sc->sge.ofld_rxq, M_CXGBE); #endif #ifdef DEV_NETMAP free(sc->sge.nm_rxq, M_CXGBE); free(sc->sge.nm_txq, M_CXGBE); #endif free(sc->irq, M_CXGBE); free(sc->sge.rxq, M_CXGBE); free(sc->sge.txq, M_CXGBE); free(sc->sge.ctrlq, M_CXGBE); free(sc->sge.iqmap, M_CXGBE); free(sc->sge.eqmap, M_CXGBE); free(sc->tids.ftid_tab, M_CXGBE); free(sc->tids.hpftid_tab, M_CXGBE); free_hftid_hash(&sc->tids); free(sc->tids.tid_tab, M_CXGBE); t4_destroy_dma_tag(sc); callout_drain(&sc->ktls_tick); callout_drain(&sc->sfl_callout); if (mtx_initialized(&sc->tids.ftid_lock)) { mtx_destroy(&sc->tids.ftid_lock); cv_destroy(&sc->tids.ftid_cv); } if (mtx_initialized(&sc->tids.atid_lock)) mtx_destroy(&sc->tids.atid_lock); if (mtx_initialized(&sc->ifp_lock)) mtx_destroy(&sc->ifp_lock); if (rw_initialized(&sc->policy_lock)) { rw_destroy(&sc->policy_lock); #ifdef TCP_OFFLOAD if (sc->policy != NULL) free_offload_policy(sc->policy); #endif } for (i = 0; i < NUM_MEMWIN; i++) { struct memwin *mw = &sc->memwin[i]; if (rw_initialized(&mw->mw_lock)) rw_destroy(&mw->mw_lock); } mtx_destroy(&sc->sfl_lock); mtx_destroy(&sc->reg_lock); mtx_destroy(&sc->sc_lock); bzero(sc, sizeof(*sc)); return (0); } static inline int stop_adapter(struct adapter *sc) { struct port_info *pi; int i; if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_STOPPED))) { CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x, EALREADY\n", __func__, curthread, sc->flags, sc->error_flags); return (EALREADY); } CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x\n", __func__, curthread, sc->flags, sc->error_flags); t4_shutdown_adapter(sc); for_each_port(sc, i) { pi = sc->port[i]; PORT_LOCK(pi); if (pi->up_vis > 0 && pi->link_cfg.link_ok) { /* * t4_shutdown_adapter has already shut down all the * PHYs but it also disables interrupts and DMA so there * won't be a link interrupt. Update the state manually * if the link was up previously and inform the kernel. */ pi->link_cfg.link_ok = false; t4_os_link_changed(pi); } PORT_UNLOCK(pi); } return (0); } static inline int restart_adapter(struct adapter *sc) { uint32_t val; if (!atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_STOPPED))) { CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x, EALREADY\n", __func__, curthread, sc->flags, sc->error_flags); return (EALREADY); } CH_ALERT(sc, "%s from %p, flags 0x%08x,0x%08x\n", __func__, curthread, sc->flags, sc->error_flags); MPASS(hw_off_limits(sc)); MPASS((sc->flags & FW_OK) == 0); MPASS((sc->flags & MASTER_PF) == 0); MPASS(sc->reset_thread == NULL); /* * The adapter is supposed to be back on PCIE with its config space and * BARs restored to their state before reset. Register access via * t4_read_reg BAR0 should just work. */ sc->reset_thread = curthread; val = t4_read_reg(sc, A_PL_WHOAMI); if (val == 0xffffffff || val == 0xeeeeeeee) { CH_ERR(sc, "%s: device registers not readable.\n", __func__); sc->reset_thread = NULL; atomic_set_int(&sc->error_flags, ADAP_STOPPED); return (ENXIO); } atomic_clear_int(&sc->error_flags, ADAP_FATAL_ERR); atomic_add_int(&sc->incarnation, 1); atomic_add_int(&sc->num_resets, 1); return (0); } static inline void set_adapter_hwstatus(struct adapter *sc, const bool usable) { if (usable) { /* Must be marked reusable by the designated thread. */ ASSERT_SYNCHRONIZED_OP(sc); MPASS(sc->reset_thread == curthread); mtx_lock(&sc->reg_lock); atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS); mtx_unlock(&sc->reg_lock); } else { /* Mark the adapter totally off limits. */ begin_synchronized_op(sc, NULL, SLEEP_OK, "t4hwsts"); mtx_lock(&sc->reg_lock); atomic_set_int(&sc->error_flags, HW_OFF_LIMITS); mtx_unlock(&sc->reg_lock); sc->flags &= ~(FW_OK | MASTER_PF); sc->reset_thread = NULL; end_synchronized_op(sc, 0); } } static int stop_lld(struct adapter *sc) { struct port_info *pi; struct vi_info *vi; if_t ifp; struct sge_rxq *rxq; struct sge_txq *txq; struct sge_wrq *wrq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif #if defined(TCP_OFFLOAD) || defined(RATELIMIT) struct sge_ofld_txq *ofld_txq; #endif int rc, i, j, k; /* * XXX: Can there be a synch_op in progress that will hang because * hardware has been stopped? We'll hang too and the solution will be * to use a version of begin_synch_op that wakes up existing synch_op * with errors. Maybe stop_adapter should do this wakeup? * * I don't think any synch_op could get stranded waiting for DMA or * interrupt so I think we're okay here. Remove this comment block * after testing. */ rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4slld"); if (rc != 0) return (ENXIO); /* Quiesce all activity. */ for_each_port(sc, i) { pi = sc->port[i]; pi->vxlan_tcam_entry = false; for_each_vi(pi, j, vi) { vi->xact_addr_filt = -1; mtx_lock(&vi->tick_mtx); vi->flags |= VI_SKIP_STATS; mtx_unlock(&vi->tick_mtx); if (!(vi->flags & VI_INIT_DONE)) continue; ifp = vi->ifp; if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { mtx_lock(&vi->tick_mtx); callout_stop(&vi->tick); mtx_unlock(&vi->tick_mtx); callout_drain(&vi->tick); } /* * Note that the HW is not available. */ for_each_txq(vi, k, txq) { TXQ_LOCK(txq); txq->eq.flags &= ~(EQ_ENABLED | EQ_HW_ALLOCATED); TXQ_UNLOCK(txq); } #if defined(TCP_OFFLOAD) || defined(RATELIMIT) for_each_ofld_txq(vi, k, ofld_txq) { TXQ_LOCK(&ofld_txq->wrq); ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED; TXQ_UNLOCK(&ofld_txq->wrq); } #endif for_each_rxq(vi, k, rxq) { rxq->iq.flags &= ~IQ_HW_ALLOCATED; } #if defined(TCP_OFFLOAD) for_each_ofld_rxq(vi, k, ofld_rxq) { ofld_rxq->iq.flags &= ~IQ_HW_ALLOCATED; } #endif quiesce_vi(vi); } if (sc->flags & FULL_INIT_DONE) { /* Control queue */ wrq = &sc->sge.ctrlq[i]; TXQ_LOCK(wrq); wrq->eq.flags &= ~EQ_HW_ALLOCATED; TXQ_UNLOCK(wrq); quiesce_wrq(wrq); } if (pi->flags & HAS_TRACEQ) { pi->flags &= ~HAS_TRACEQ; sc->traceq = -1; sc->tracer_valid = 0; sc->tracer_enabled = 0; } } if (sc->flags & FULL_INIT_DONE) { /* Firmware event queue */ sc->sge.fwq.flags &= ~IQ_HW_ALLOCATED; quiesce_iq_fl(sc, &sc->sge.fwq, NULL); } /* Stop calibration */ callout_stop(&sc->cal_callout); callout_drain(&sc->cal_callout); if (t4_clock_gate_on_suspend) { t4_set_reg_field(sc, A_PMU_PART_CG_PWRMODE, F_MA_PART_CGEN | F_LE_PART_CGEN | F_EDC1_PART_CGEN | F_EDC0_PART_CGEN | F_TP_PART_CGEN | F_PDP_PART_CGEN | F_SGE_PART_CGEN, 0); } end_synchronized_op(sc, 0); stop_atid_allocator(sc); t4_stop_l2t(sc); return (rc); } int suspend_adapter(struct adapter *sc) { stop_adapter(sc); stop_lld(sc); #ifdef TCP_OFFLOAD stop_all_uld(sc); #endif set_adapter_hwstatus(sc, false); return (0); } static int t4_suspend(device_t dev) { struct adapter *sc = device_get_softc(dev); int rc; CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); rc = suspend_adapter(sc); CH_ALERT(sc, "%s end (thread %p).\n", __func__, curthread); return (rc); } struct adapter_pre_reset_state { u_int flags; uint16_t nbmcaps; uint16_t linkcaps; uint16_t switchcaps; uint16_t niccaps; uint16_t toecaps; uint16_t rdmacaps; uint16_t cryptocaps; uint16_t iscsicaps; uint16_t fcoecaps; u_int cfcsum; char cfg_file[32]; struct adapter_params params; struct t4_virt_res vres; struct tid_info tids; struct sge sge; int rawf_base; int nrawf; }; static void save_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o) { ASSERT_SYNCHRONIZED_OP(sc); o->flags = sc->flags; o->nbmcaps = sc->nbmcaps; o->linkcaps = sc->linkcaps; o->switchcaps = sc->switchcaps; o->niccaps = sc->niccaps; o->toecaps = sc->toecaps; o->rdmacaps = sc->rdmacaps; o->cryptocaps = sc->cryptocaps; o->iscsicaps = sc->iscsicaps; o->fcoecaps = sc->fcoecaps; o->cfcsum = sc->cfcsum; MPASS(sizeof(o->cfg_file) == sizeof(sc->cfg_file)); memcpy(o->cfg_file, sc->cfg_file, sizeof(o->cfg_file)); o->params = sc->params; o->vres = sc->vres; o->tids = sc->tids; o->sge = sc->sge; o->rawf_base = sc->rawf_base; o->nrawf = sc->nrawf; } static int compare_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o) { int rc = 0; ASSERT_SYNCHRONIZED_OP(sc); /* Capabilities */ #define COMPARE_CAPS(c) do { \ if (o->c##caps != sc->c##caps) { \ CH_ERR(sc, "%scaps 0x%04x -> 0x%04x.\n", #c, o->c##caps, \ sc->c##caps); \ rc = EINVAL; \ } \ } while (0) COMPARE_CAPS(nbm); COMPARE_CAPS(link); COMPARE_CAPS(switch); COMPARE_CAPS(nic); COMPARE_CAPS(toe); COMPARE_CAPS(rdma); COMPARE_CAPS(crypto); COMPARE_CAPS(iscsi); COMPARE_CAPS(fcoe); #undef COMPARE_CAPS /* Firmware config file */ if (o->cfcsum != sc->cfcsum) { CH_ERR(sc, "config file %s (0x%x) -> %s (0x%x)\n", o->cfg_file, o->cfcsum, sc->cfg_file, sc->cfcsum); rc = EINVAL; } #define COMPARE_PARAM(p, name) do { \ if (o->p != sc->p) { \ CH_ERR(sc, #name " %d -> %d\n", o->p, sc->p); \ rc = EINVAL; \ } \ } while (0) COMPARE_PARAM(sge.iq_start, iq_start); COMPARE_PARAM(sge.eq_start, eq_start); COMPARE_PARAM(tids.ftid_base, ftid_base); COMPARE_PARAM(tids.ftid_end, ftid_end); COMPARE_PARAM(tids.nftids, nftids); COMPARE_PARAM(vres.l2t.start, l2t_start); COMPARE_PARAM(vres.l2t.size, l2t_size); COMPARE_PARAM(sge.iqmap_sz, iqmap_sz); COMPARE_PARAM(sge.eqmap_sz, eqmap_sz); COMPARE_PARAM(tids.tid_base, tid_base); COMPARE_PARAM(tids.hpftid_base, hpftid_base); COMPARE_PARAM(tids.hpftid_end, hpftid_end); COMPARE_PARAM(tids.nhpftids, nhpftids); COMPARE_PARAM(rawf_base, rawf_base); COMPARE_PARAM(nrawf, nrawf); COMPARE_PARAM(params.mps_bg_map, mps_bg_map); COMPARE_PARAM(params.filter2_wr_support, filter2_wr_support); COMPARE_PARAM(params.ulptx_memwrite_dsgl, ulptx_memwrite_dsgl); COMPARE_PARAM(params.fr_nsmr_tpte_wr_support, fr_nsmr_tpte_wr_support); COMPARE_PARAM(params.max_pkts_per_eth_tx_pkts_wr, max_pkts_per_eth_tx_pkts_wr); COMPARE_PARAM(tids.ntids, ntids); COMPARE_PARAM(tids.etid_base, etid_base); COMPARE_PARAM(tids.etid_end, etid_end); COMPARE_PARAM(tids.netids, netids); COMPARE_PARAM(params.eo_wr_cred, eo_wr_cred); COMPARE_PARAM(params.ethoffload, ethoffload); COMPARE_PARAM(tids.natids, natids); COMPARE_PARAM(tids.stid_base, stid_base); COMPARE_PARAM(vres.ddp.start, ddp_start); COMPARE_PARAM(vres.ddp.size, ddp_size); COMPARE_PARAM(params.ofldq_wr_cred, ofldq_wr_cred); COMPARE_PARAM(vres.stag.start, stag_start); COMPARE_PARAM(vres.stag.size, stag_size); COMPARE_PARAM(vres.rq.start, rq_start); COMPARE_PARAM(vres.rq.size, rq_size); COMPARE_PARAM(vres.pbl.start, pbl_start); COMPARE_PARAM(vres.pbl.size, pbl_size); COMPARE_PARAM(vres.qp.start, qp_start); COMPARE_PARAM(vres.qp.size, qp_size); COMPARE_PARAM(vres.cq.start, cq_start); COMPARE_PARAM(vres.cq.size, cq_size); COMPARE_PARAM(vres.ocq.start, ocq_start); COMPARE_PARAM(vres.ocq.size, ocq_size); COMPARE_PARAM(vres.srq.start, srq_start); COMPARE_PARAM(vres.srq.size, srq_size); COMPARE_PARAM(params.max_ordird_qp, max_ordird_qp); COMPARE_PARAM(params.max_ird_adapter, max_ird_adapter); COMPARE_PARAM(vres.iscsi.start, iscsi_start); COMPARE_PARAM(vres.iscsi.size, iscsi_size); COMPARE_PARAM(vres.key.start, key_start); COMPARE_PARAM(vres.key.size, key_size); #undef COMPARE_PARAM return (rc); } static int restart_lld(struct adapter *sc) { struct adapter_pre_reset_state *old_state = NULL; struct port_info *pi; struct vi_info *vi; if_t ifp; struct sge_txq *txq; int rc, i, j, k; rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4rlld"); if (rc != 0) return (ENXIO); /* Restore memory window. */ setup_memwin(sc); /* Go no further if recovery mode has been requested. */ if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { CH_ALERT(sc, "%s: recovery mode during restart.\n", __func__); rc = 0; set_adapter_hwstatus(sc, true); goto done; } old_state = malloc(sizeof(*old_state), M_CXGBE, M_ZERO | M_WAITOK); save_caps_and_params(sc, old_state); /* Reestablish contact with firmware and become the primary PF. */ rc = contact_firmware(sc); if (rc != 0) goto done; /* error message displayed already */ MPASS(sc->flags & FW_OK); if (sc->flags & MASTER_PF) { rc = partition_resources(sc); if (rc != 0) goto done; /* error message displayed already */ } rc = get_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = set_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = compare_caps_and_params(sc, old_state); if (rc != 0) goto done; /* error message displayed already */ for_each_port(sc, i) { pi = sc->port[i]; MPASS(pi != NULL); MPASS(pi->vi != NULL); MPASS(pi->vi[0].dev == pi->dev); rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); if (rc != 0) { CH_ERR(sc, "failed to re-initialize port %d: %d\n", i, rc); goto done; } MPASS(sc->chan_map[pi->tx_chan] == i); PORT_LOCK(pi); fixup_link_config(pi); build_medialist(pi); PORT_UNLOCK(pi); for_each_vi(pi, j, vi) { if (IS_MAIN_VI(vi)) continue; rc = alloc_extra_vi(sc, pi, vi); if (rc != 0) { CH_ERR(vi, "failed to re-allocate extra VI: %d\n", rc); goto done; } } } /* * Interrupts and queues are about to be enabled and other threads will * want to access the hardware too. It is safe to do so. Note that * this thread is still in the middle of a synchronized_op. */ set_adapter_hwstatus(sc, true); if (sc->flags & FULL_INIT_DONE) { rc = adapter_full_init(sc); if (rc != 0) { CH_ERR(sc, "failed to re-initialize adapter: %d\n", rc); goto done; } if (sc->vxlan_refcount > 0) enable_vxlan_rx(sc); for_each_port(sc, i) { pi = sc->port[i]; for_each_vi(pi, j, vi) { mtx_lock(&vi->tick_mtx); vi->flags &= ~VI_SKIP_STATS; mtx_unlock(&vi->tick_mtx); if (!(vi->flags & VI_INIT_DONE)) continue; rc = vi_full_init(vi); if (rc != 0) { CH_ERR(vi, "failed to re-initialize " "interface: %d\n", rc); goto done; } if (sc->traceq < 0 && IS_MAIN_VI(vi)) { sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | V_QUEUENUMBER(sc->traceq)); pi->flags |= HAS_TRACEQ; } ifp = vi->ifp; if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) continue; /* * Note that we do not setup multicast addresses * in the first pass. This ensures that the * unicast DMACs for all VIs on all ports get an * MPS TCAM entry. */ rc = update_mac_settings(ifp, XGMAC_ALL & ~XGMAC_MCADDRS); if (rc != 0) { CH_ERR(vi, "failed to re-configure MAC: %d\n", rc); goto done; } rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); if (rc != 0) { CH_ERR(vi, "failed to re-enable VI: %d\n", rc); goto done; } for_each_txq(vi, k, txq) { TXQ_LOCK(txq); txq->eq.flags |= EQ_ENABLED; TXQ_UNLOCK(txq); } mtx_lock(&vi->tick_mtx); callout_schedule(&vi->tick, hz); mtx_unlock(&vi->tick_mtx); } PORT_LOCK(pi); if (pi->up_vis > 0) { t4_update_port_info(pi); fixup_link_config(pi); build_medialist(pi); apply_link_config(pi); if (pi->link_cfg.link_ok) t4_os_link_changed(pi); } PORT_UNLOCK(pi); } /* Now reprogram the L2 multicast addresses. */ for_each_port(sc, i) { pi = sc->port[i]; for_each_vi(pi, j, vi) { if (!(vi->flags & VI_INIT_DONE)) continue; ifp = vi->ifp; if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) continue; rc = update_mac_settings(ifp, XGMAC_MCADDRS); if (rc != 0) { CH_ERR(vi, "failed to re-configure MCAST MACs: %d\n", rc); rc = 0; /* carry on */ } } } } /* Reset all calibration */ t4_calibration_start(sc); done: end_synchronized_op(sc, 0); free(old_state, M_CXGBE); restart_atid_allocator(sc); t4_restart_l2t(sc); return (rc); } int resume_adapter(struct adapter *sc) { restart_adapter(sc); restart_lld(sc); #ifdef TCP_OFFLOAD restart_all_uld(sc); #endif return (0); } static int t4_resume(device_t dev) { struct adapter *sc = device_get_softc(dev); int rc; CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); rc = resume_adapter(sc); CH_ALERT(sc, "%s end (thread %p).\n", __func__, curthread); return (rc); } static int t4_reset_prepare(device_t dev, device_t child) { struct adapter *sc = device_get_softc(dev); CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); return (0); } static int t4_reset_post(device_t dev, device_t child) { struct adapter *sc = device_get_softc(dev); CH_ALERT(sc, "%s from thread %p.\n", __func__, curthread); return (0); } static int reset_adapter_with_pci_bus_reset(struct adapter *sc) { int rc; mtx_lock(&Giant); rc = BUS_RESET_CHILD(device_get_parent(sc->dev), sc->dev, 0); mtx_unlock(&Giant); return (rc); } static int reset_adapter_with_pl_rst(struct adapter *sc) { suspend_adapter(sc); /* This is a t4_write_reg without the hw_off_limits check. */ MPASS(sc->error_flags & HW_OFF_LIMITS); bus_space_write_4(sc->bt, sc->bh, A_PL_RST, F_PIORSTMODE | F_PIORST | F_AUTOPCIEPAUSE); pause("pl_rst", 1 * hz); /* Wait 1s for reset */ resume_adapter(sc); return (0); } static inline int reset_adapter(struct adapter *sc) { if (vm_guest == 0) return (reset_adapter_with_pci_bus_reset(sc)); else return (reset_adapter_with_pl_rst(sc)); } static void reset_adapter_task(void *arg, int pending) { struct adapter *sc = arg; const int flags = sc->flags; const int eflags = sc->error_flags; int rc; if (pending > 1) CH_ALERT(sc, "%s: pending %d\n", __func__, pending); rc = reset_adapter(sc); if (rc != 0) { CH_ERR(sc, "adapter did not reset properly, rc = %d, " "flags 0x%08x -> 0x%08x, err_flags 0x%08x -> 0x%08x.\n", rc, flags, sc->flags, eflags, sc->error_flags); } } static int cxgbe_probe(device_t dev) { struct port_info *pi = device_get_softc(dev); device_set_descf(dev, "port %d", pi->port_id); return (BUS_PROBE_DEFAULT); } #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \ IFCAP_HWRXTSTMP | IFCAP_MEXTPG) #define T4_CAP_ENABLE (T4_CAP) static void cxgbe_vi_attach(device_t dev, struct vi_info *vi) { if_t ifp; struct sbuf *sb; struct sysctl_ctx_list *ctx = &vi->ctx; struct sysctl_oid_list *children; struct pfil_head_args pa; struct adapter *sc = vi->adapter; sysctl_ctx_init(ctx); children = SYSCTL_CHILDREN(device_get_sysctl_tree(vi->dev)); vi->rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rxq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC rx queues"); vi->txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "txq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC tx queues"); #ifdef DEV_NETMAP vi->nm_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_rxq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap rx queues"); vi->nm_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_txq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap tx queues"); #endif #ifdef TCP_OFFLOAD vi->ofld_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_rxq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE rx queues"); #endif #if defined(TCP_OFFLOAD) || defined(RATELIMIT) vi->ofld_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_txq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE/ETHOFLD tx queues"); #endif vi->xact_addr_filt = -1; mtx_init(&vi->tick_mtx, "vi tick", NULL, MTX_DEF); callout_init_mtx(&vi->tick, &vi->tick_mtx, 0); if (sc->flags & IS_VF || t4_tx_vm_wr != 0) vi->flags |= TX_USES_VM_WR; /* Allocate an ifnet and set it up */ ifp = if_alloc_dev(IFT_ETHER, dev); vi->ifp = ifp; if_setsoftc(ifp, vi); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); if_setinitfn(ifp, cxgbe_init); if_setioctlfn(ifp, cxgbe_ioctl); if_settransmitfn(ifp, cxgbe_transmit); if_setqflushfn(ifp, cxgbe_qflush); if (vi->pi->nvi > 1 || sc->flags & IS_VF) if_setgetcounterfn(ifp, vi_get_counter); else if_setgetcounterfn(ifp, cxgbe_get_counter); #if defined(KERN_TLS) || defined(RATELIMIT) if_setsndtagallocfn(ifp, cxgbe_snd_tag_alloc); #endif #ifdef RATELIMIT if_setratelimitqueryfn(ifp, cxgbe_ratelimit_query); #endif if_setcapabilities(ifp, T4_CAP); if_setcapenable(ifp, T4_CAP_ENABLE); if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | CSUM_UDP_IPV6 | CSUM_TCP_IPV6); if (chip_id(sc) >= CHELSIO_T6) { if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0); if_setcapenablebit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0); if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_ENCAP_VXLAN, 0); } #ifdef TCP_OFFLOAD if (vi->nofldrxq != 0) if_setcapabilitiesbit(ifp, IFCAP_TOE, 0); #endif #ifdef RATELIMIT if (is_ethoffload(sc) && vi->nofldtxq != 0) { if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT, 0); if_setcapenablebit(ifp, IFCAP_TXRTLMT, 0); } #endif if_sethwtsomax(ifp, IP_MAXPACKET); if (vi->flags & TX_USES_VM_WR) if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_VM_TSO); else if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_TSO); #ifdef RATELIMIT if (is_ethoffload(sc) && vi->nofldtxq != 0) if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_EO_TSO); #endif if_sethwtsomaxsegsize(ifp, 65536); #ifdef KERN_TLS if (is_ktls(sc)) { if_setcapabilitiesbit(ifp, IFCAP_TXTLS, 0); if (sc->flags & KERN_TLS_ON || !is_t6(sc)) if_setcapenablebit(ifp, IFCAP_TXTLS, 0); } #endif ether_ifattach(ifp, vi->hw_addr); #ifdef DEV_NETMAP if (vi->nnmrxq != 0) cxgbe_nm_attach(vi); #endif sb = sbuf_new_auto(); sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); #if defined(TCP_OFFLOAD) || defined(RATELIMIT) switch (if_getcapabilities(ifp) & (IFCAP_TOE | IFCAP_TXRTLMT)) { case IFCAP_TOE: sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq); break; case IFCAP_TOE | IFCAP_TXRTLMT: sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq); break; case IFCAP_TXRTLMT: sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq); break; } #endif #ifdef TCP_OFFLOAD if (if_getcapabilities(ifp) & IFCAP_TOE) sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq); #endif #ifdef DEV_NETMAP if (if_getcapabilities(ifp) & IFCAP_NETMAP) sbuf_printf(sb, "; %d txq, %d rxq (netmap)", vi->nnmtxq, vi->nnmrxq); #endif sbuf_finish(sb); device_printf(dev, "%s\n", sbuf_data(sb)); sbuf_delete(sb); vi_sysctls(vi); pa.pa_version = PFIL_VERSION; pa.pa_flags = PFIL_IN; pa.pa_type = PFIL_TYPE_ETHERNET; pa.pa_headname = if_name(ifp); vi->pfil = pfil_head_register(&pa); } static int cxgbe_attach(device_t dev) { struct port_info *pi = device_get_softc(dev); struct adapter *sc = pi->adapter; struct vi_info *vi; int i; sysctl_ctx_init(&pi->ctx); cxgbe_vi_attach(dev, &pi->vi[0]); for_each_vi(pi, i, vi) { if (i == 0) continue; vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, DEVICE_UNIT_ANY); if (vi->dev == NULL) { device_printf(dev, "failed to add VI %d\n", i); continue; } device_set_softc(vi->dev, vi); } cxgbe_sysctls(pi); bus_attach_children(dev); return (0); } static void cxgbe_vi_detach(struct vi_info *vi) { if_t ifp = vi->ifp; if (vi->pfil != NULL) { pfil_head_unregister(vi->pfil); vi->pfil = NULL; } ether_ifdetach(ifp); /* Let detach proceed even if these fail. */ #ifdef DEV_NETMAP if (if_getcapabilities(ifp) & IFCAP_NETMAP) cxgbe_nm_detach(vi); #endif cxgbe_uninit_synchronized(vi); callout_drain(&vi->tick); mtx_destroy(&vi->tick_mtx); sysctl_ctx_free(&vi->ctx); vi_full_uninit(vi); if_free(vi->ifp); vi->ifp = NULL; } static int cxgbe_detach(device_t dev) { struct port_info *pi = device_get_softc(dev); struct adapter *sc = pi->adapter; int rc; /* Detach the extra VIs first. */ rc = bus_generic_detach(dev); if (rc) return (rc); - device_delete_children(dev); sysctl_ctx_free(&pi->ctx); begin_vi_detach(sc, &pi->vi[0]); if (pi->flags & HAS_TRACEQ) { sc->traceq = -1; /* cloner should not create ifnet */ t4_tracer_port_detach(sc); } cxgbe_vi_detach(&pi->vi[0]); ifmedia_removeall(&pi->media); end_vi_detach(sc, &pi->vi[0]); return (0); } static void cxgbe_init(void *arg) { struct vi_info *vi = arg; struct adapter *sc = vi->adapter; if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) return; cxgbe_init_synchronized(vi); end_synchronized_op(sc, 0); } static int cxgbe_ioctl(if_t ifp, unsigned long cmd, caddr_t data) { int rc = 0, mtu, flags; struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct ifreq *ifr = (struct ifreq *)data; uint32_t mask; switch (cmd) { case SIOCSIFMTU: mtu = ifr->ifr_mtu; if (mtu < ETHERMIN || mtu > MAX_MTU) return (EINVAL); rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); if (rc) return (rc); if_setmtu(ifp, mtu); if (vi->flags & VI_INIT_DONE) { t4_update_fl_bufsize(ifp); if (!hw_off_limits(sc) && if_getdrvflags(ifp) & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_MTU); } end_synchronized_op(sc, 0); break; case SIOCSIFFLAGS: rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto fail; } if (if_getflags(ifp) & IFF_UP) { if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { flags = vi->if_flags; if ((if_getflags(ifp) ^ flags) & (IFF_PROMISC | IFF_ALLMULTI)) { rc = update_mac_settings(ifp, XGMAC_PROMISC | XGMAC_ALLMULTI); } } else { rc = cxgbe_init_synchronized(vi); } vi->if_flags = if_getflags(ifp); } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { rc = cxgbe_uninit_synchronized(vi); } end_synchronized_op(sc, 0); break; case SIOCADDMULTI: case SIOCDELMULTI: rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi"); if (rc) return (rc); if (!hw_off_limits(sc) && if_getdrvflags(ifp) & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_MCADDRS); end_synchronized_op(sc, 0); break; case SIOCSIFCAP: rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); if (rc) return (rc); mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); if (mask & IFCAP_TXCSUM) { if_togglecapenable(ifp, IFCAP_TXCSUM); if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP); if (IFCAP_TSO4 & if_getcapenable(ifp) && !(IFCAP_TXCSUM & if_getcapenable(ifp))) { mask &= ~IFCAP_TSO4; if_setcapenablebit(ifp, 0, IFCAP_TSO4); if_printf(ifp, "tso4 disabled due to -txcsum.\n"); } } if (mask & IFCAP_TXCSUM_IPV6) { if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6); if (IFCAP_TSO6 & if_getcapenable(ifp) && !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { mask &= ~IFCAP_TSO6; if_setcapenablebit(ifp, 0, IFCAP_TSO6); if_printf(ifp, "tso6 disabled due to -txcsum6.\n"); } } if (mask & IFCAP_RXCSUM) if_togglecapenable(ifp, IFCAP_RXCSUM); if (mask & IFCAP_RXCSUM_IPV6) if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); /* * Note that we leave CSUM_TSO alone (it is always set). The * kernel takes both IFCAP_TSOx and CSUM_TSO into account before * sending a TSO request our way, so it's sufficient to toggle * IFCAP_TSOx only. */ if (mask & IFCAP_TSO4) { if (!(IFCAP_TSO4 & if_getcapenable(ifp)) && !(IFCAP_TXCSUM & if_getcapenable(ifp))) { if_printf(ifp, "enable txcsum first.\n"); rc = EAGAIN; goto fail; } if_togglecapenable(ifp, IFCAP_TSO4); } if (mask & IFCAP_TSO6) { if (!(IFCAP_TSO6 & if_getcapenable(ifp)) && !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { if_printf(ifp, "enable txcsum6 first.\n"); rc = EAGAIN; goto fail; } if_togglecapenable(ifp, IFCAP_TSO6); } if (mask & IFCAP_LRO) { #if defined(INET) || defined(INET6) int i; struct sge_rxq *rxq; if_togglecapenable(ifp, IFCAP_LRO); for_each_rxq(vi, i, rxq) { if (if_getcapenable(ifp) & IFCAP_LRO) rxq->iq.flags |= IQ_LRO_ENABLED; else rxq->iq.flags &= ~IQ_LRO_ENABLED; } #endif } #ifdef TCP_OFFLOAD if (mask & IFCAP_TOE) { int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE; rc = toe_capability(vi, enable); if (rc != 0) goto fail; if_togglecapenable(ifp, mask); } #endif if (mask & IFCAP_VLAN_HWTAGGING) { if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) rc = update_mac_settings(ifp, XGMAC_VLANEX); } if (mask & IFCAP_VLAN_MTU) { if_togglecapenable(ifp, IFCAP_VLAN_MTU); /* Need to find out how to disable auto-mtu-inflation */ } if (mask & IFCAP_VLAN_HWTSO) if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); if (mask & IFCAP_VLAN_HWCSUM) if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM); #ifdef RATELIMIT if (mask & IFCAP_TXRTLMT) if_togglecapenable(ifp, IFCAP_TXRTLMT); #endif if (mask & IFCAP_HWRXTSTMP) { int i; struct sge_rxq *rxq; if_togglecapenable(ifp, IFCAP_HWRXTSTMP); for_each_rxq(vi, i, rxq) { if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP) rxq->iq.flags |= IQ_RX_TIMESTAMP; else rxq->iq.flags &= ~IQ_RX_TIMESTAMP; } } if (mask & IFCAP_MEXTPG) if_togglecapenable(ifp, IFCAP_MEXTPG); #ifdef KERN_TLS if (mask & IFCAP_TXTLS) { int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TXTLS; rc = ktls_capability(sc, enable); if (rc != 0) goto fail; if_togglecapenable(ifp, mask & IFCAP_TXTLS); } #endif if (mask & IFCAP_VXLAN_HWCSUM) { if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM); if_togglehwassist(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP | CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP); } if (mask & IFCAP_VXLAN_HWTSO) { if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO); if_togglehwassist(ifp, CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO); } #ifdef VLAN_CAPABILITIES VLAN_CAPABILITIES(ifp); #endif fail: end_synchronized_op(sc, 0); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: case SIOCGIFXMEDIA: rc = ifmedia_ioctl(ifp, ifr, &pi->media, cmd); break; case SIOCGI2C: { struct ifi2creq i2c; rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); if (rc != 0) break; if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { rc = EPERM; break; } if (i2c.len > sizeof(i2c.data)) { rc = EINVAL; break; } rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, i2c.offset, i2c.len, &i2c.data[0]); end_synchronized_op(sc, 0); if (rc == 0) rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); break; } default: rc = ether_ioctl(ifp, cmd, data); } return (rc); } static int cxgbe_transmit(if_t ifp, struct mbuf *m) { struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct adapter *sc; struct sge_txq *txq; void *items[1]; int rc; M_ASSERTPKTHDR(m); MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ #if defined(KERN_TLS) || defined(RATELIMIT) if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) MPASS(m->m_pkthdr.snd_tag->ifp == ifp); #endif if (__predict_false(pi->link_cfg.link_ok == false)) { m_freem(m); return (ENETDOWN); } rc = parse_pkt(&m, vi->flags & TX_USES_VM_WR); if (__predict_false(rc != 0)) { if (__predict_true(rc == EINPROGRESS)) { /* queued by parse_pkt */ MPASS(m != NULL); return (0); } MPASS(m == NULL); /* was freed already */ atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ return (rc); } /* Select a txq. */ sc = vi->adapter; txq = &sc->sge.txq[vi->first_txq]; if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + vi->rsrv_noflowq); items[0] = m; rc = mp_ring_enqueue(txq->r, items, 1, 256); if (__predict_false(rc != 0)) m_freem(m); return (rc); } static void cxgbe_qflush(if_t ifp) { struct vi_info *vi = if_getsoftc(ifp); struct sge_txq *txq; int i; /* queues do not exist if !VI_INIT_DONE. */ if (vi->flags & VI_INIT_DONE) { for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags |= EQ_QFLUSH; TXQ_UNLOCK(txq); while (!mp_ring_is_idle(txq->r)) { mp_ring_check_drainage(txq->r, 4096); pause("qflush", 1); } TXQ_LOCK(txq); txq->eq.flags &= ~EQ_QFLUSH; TXQ_UNLOCK(txq); } } if_qflush(ifp); } static uint64_t vi_get_counter(if_t ifp, ift_counter c) { struct vi_info *vi = if_getsoftc(ifp); struct fw_vi_stats_vf *s = &vi->stats; mtx_lock(&vi->tick_mtx); vi_refresh_stats(vi); mtx_unlock(&vi->tick_mtx); switch (c) { case IFCOUNTER_IPACKETS: return (s->rx_bcast_frames + s->rx_mcast_frames + s->rx_ucast_frames); case IFCOUNTER_IERRORS: return (s->rx_err_frames); case IFCOUNTER_OPACKETS: return (s->tx_bcast_frames + s->tx_mcast_frames + s->tx_ucast_frames + s->tx_offload_frames); case IFCOUNTER_OERRORS: return (s->tx_drop_frames); case IFCOUNTER_IBYTES: return (s->rx_bcast_bytes + s->rx_mcast_bytes + s->rx_ucast_bytes); case IFCOUNTER_OBYTES: return (s->tx_bcast_bytes + s->tx_mcast_bytes + s->tx_ucast_bytes + s->tx_offload_bytes); case IFCOUNTER_IMCASTS: return (s->rx_mcast_frames); case IFCOUNTER_OMCASTS: return (s->tx_mcast_frames); case IFCOUNTER_OQDROPS: { uint64_t drops; drops = 0; if (vi->flags & VI_INIT_DONE) { int i; struct sge_txq *txq; for_each_txq(vi, i, txq) drops += counter_u64_fetch(txq->r->dropped); } return (drops); } default: return (if_get_counter_default(ifp, c)); } } static uint64_t cxgbe_get_counter(if_t ifp, ift_counter c) { struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct port_stats *s = &pi->stats; mtx_lock(&vi->tick_mtx); cxgbe_refresh_stats(vi); mtx_unlock(&vi->tick_mtx); switch (c) { case IFCOUNTER_IPACKETS: return (s->rx_frames); case IFCOUNTER_IERRORS: return (s->rx_jabber + s->rx_runt + s->rx_too_long + s->rx_fcs_err + s->rx_len_err); case IFCOUNTER_OPACKETS: return (s->tx_frames); case IFCOUNTER_OERRORS: return (s->tx_error_frames); case IFCOUNTER_IBYTES: return (s->rx_octets); case IFCOUNTER_OBYTES: return (s->tx_octets); case IFCOUNTER_IMCASTS: return (s->rx_mcast_frames); case IFCOUNTER_OMCASTS: return (s->tx_mcast_frames); case IFCOUNTER_IQDROPS: return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + s->rx_trunc3 + pi->tnl_cong_drops); case IFCOUNTER_OQDROPS: { uint64_t drops; drops = s->tx_drop; if (vi->flags & VI_INIT_DONE) { int i; struct sge_txq *txq; for_each_txq(vi, i, txq) drops += counter_u64_fetch(txq->r->dropped); } return (drops); } default: return (if_get_counter_default(ifp, c)); } } #if defined(KERN_TLS) || defined(RATELIMIT) static int cxgbe_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **pt) { int error; switch (params->hdr.type) { #ifdef RATELIMIT case IF_SND_TAG_TYPE_RATE_LIMIT: error = cxgbe_rate_tag_alloc(ifp, params, pt); break; #endif #ifdef KERN_TLS case IF_SND_TAG_TYPE_TLS: { struct vi_info *vi = if_getsoftc(ifp); if (is_t6(vi->pi->adapter)) error = t6_tls_tag_alloc(ifp, params, pt); else error = EOPNOTSUPP; break; } #endif default: error = EOPNOTSUPP; } return (error); } #endif /* * The kernel picks a media from the list we had provided but we still validate * the requeste. */ int cxgbe_media_change(if_t ifp) { struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct ifmedia *ifm = &pi->media; struct link_config *lc = &pi->link_cfg; struct adapter *sc = pi->adapter; int rc; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec"); if (rc != 0) return (rc); PORT_LOCK(pi); if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) { /* ifconfig .. media autoselect */ if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) { rc = ENOTSUP; /* AN not supported by transceiver */ goto done; } lc->requested_aneg = AUTONEG_ENABLE; lc->requested_speed = 0; lc->requested_fc |= PAUSE_AUTONEG; } else { lc->requested_aneg = AUTONEG_DISABLE; lc->requested_speed = ifmedia_baudrate(ifm->ifm_media) / 1000000; lc->requested_fc = 0; if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE) lc->requested_fc |= PAUSE_RX; if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE) lc->requested_fc |= PAUSE_TX; } if (pi->up_vis > 0 && !hw_off_limits(sc)) { fixup_link_config(pi); rc = apply_link_config(pi); } done: PORT_UNLOCK(pi); end_synchronized_op(sc, 0); return (rc); } /* * Base media word (without ETHER, pause, link active, etc.) for the port at the * given speed. */ static int port_mword(struct port_info *pi, uint32_t speed) { MPASS(speed & M_FW_PORT_CAP32_SPEED); MPASS(powerof2(speed)); switch(pi->port_type) { case FW_PORT_TYPE_BT_SGMII: case FW_PORT_TYPE_BT_XFI: case FW_PORT_TYPE_BT_XAUI: /* BaseT */ switch (speed) { case FW_PORT_CAP32_SPEED_100M: return (IFM_100_T); case FW_PORT_CAP32_SPEED_1G: return (IFM_1000_T); case FW_PORT_CAP32_SPEED_10G: return (IFM_10G_T); } break; case FW_PORT_TYPE_KX4: if (speed == FW_PORT_CAP32_SPEED_10G) return (IFM_10G_KX4); break; case FW_PORT_TYPE_CX4: if (speed == FW_PORT_CAP32_SPEED_10G) return (IFM_10G_CX4); break; case FW_PORT_TYPE_KX: if (speed == FW_PORT_CAP32_SPEED_1G) return (IFM_1000_KX); break; case FW_PORT_TYPE_KR: case FW_PORT_TYPE_BP_AP: case FW_PORT_TYPE_BP4_AP: case FW_PORT_TYPE_BP40_BA: case FW_PORT_TYPE_KR4_100G: case FW_PORT_TYPE_KR_SFP28: case FW_PORT_TYPE_KR_XLAUI: switch (speed) { case FW_PORT_CAP32_SPEED_1G: return (IFM_1000_KX); case FW_PORT_CAP32_SPEED_10G: return (IFM_10G_KR); case FW_PORT_CAP32_SPEED_25G: return (IFM_25G_KR); case FW_PORT_CAP32_SPEED_40G: return (IFM_40G_KR4); case FW_PORT_CAP32_SPEED_50G: return (IFM_50G_KR2); case FW_PORT_CAP32_SPEED_100G: return (IFM_100G_KR4); } break; case FW_PORT_TYPE_FIBER_XFI: case FW_PORT_TYPE_FIBER_XAUI: case FW_PORT_TYPE_SFP: case FW_PORT_TYPE_QSFP_10G: case FW_PORT_TYPE_QSA: case FW_PORT_TYPE_QSFP: case FW_PORT_TYPE_CR4_QSFP: case FW_PORT_TYPE_CR_QSFP: case FW_PORT_TYPE_CR2_QSFP: case FW_PORT_TYPE_SFP28: /* Pluggable transceiver */ switch (pi->mod_type) { case FW_PORT_MOD_TYPE_LR: switch (speed) { case FW_PORT_CAP32_SPEED_1G: return (IFM_1000_LX); case FW_PORT_CAP32_SPEED_10G: return (IFM_10G_LR); case FW_PORT_CAP32_SPEED_25G: return (IFM_25G_LR); case FW_PORT_CAP32_SPEED_40G: return (IFM_40G_LR4); case FW_PORT_CAP32_SPEED_50G: return (IFM_50G_LR2); case FW_PORT_CAP32_SPEED_100G: return (IFM_100G_LR4); } break; case FW_PORT_MOD_TYPE_SR: switch (speed) { case FW_PORT_CAP32_SPEED_1G: return (IFM_1000_SX); case FW_PORT_CAP32_SPEED_10G: return (IFM_10G_SR); case FW_PORT_CAP32_SPEED_25G: return (IFM_25G_SR); case FW_PORT_CAP32_SPEED_40G: return (IFM_40G_SR4); case FW_PORT_CAP32_SPEED_50G: return (IFM_50G_SR2); case FW_PORT_CAP32_SPEED_100G: return (IFM_100G_SR4); } break; case FW_PORT_MOD_TYPE_ER: if (speed == FW_PORT_CAP32_SPEED_10G) return (IFM_10G_ER); break; case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: switch (speed) { case FW_PORT_CAP32_SPEED_1G: return (IFM_1000_CX); case FW_PORT_CAP32_SPEED_10G: return (IFM_10G_TWINAX); case FW_PORT_CAP32_SPEED_25G: return (IFM_25G_CR); case FW_PORT_CAP32_SPEED_40G: return (IFM_40G_CR4); case FW_PORT_CAP32_SPEED_50G: return (IFM_50G_CR2); case FW_PORT_CAP32_SPEED_100G: return (IFM_100G_CR4); } break; case FW_PORT_MOD_TYPE_LRM: if (speed == FW_PORT_CAP32_SPEED_10G) return (IFM_10G_LRM); break; case FW_PORT_MOD_TYPE_NA: MPASS(0); /* Not pluggable? */ /* fall throough */ case FW_PORT_MOD_TYPE_ERROR: case FW_PORT_MOD_TYPE_UNKNOWN: case FW_PORT_MOD_TYPE_NOTSUPPORTED: break; case FW_PORT_MOD_TYPE_NONE: return (IFM_NONE); } break; case FW_PORT_TYPE_NONE: return (IFM_NONE); } return (IFM_UNKNOWN); } void cxgbe_media_status(if_t ifp, struct ifmediareq *ifmr) { struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; if (begin_synchronized_op(sc, vi , SLEEP_OK | INTR_OK, "t4med") != 0) return; PORT_LOCK(pi); if (pi->up_vis == 0 && !hw_off_limits(sc)) { /* * If all the interfaces are administratively down the firmware * does not report transceiver changes. Refresh port info here * so that ifconfig displays accurate ifmedia at all times. * This is the only reason we have a synchronized op in this * function. Just PORT_LOCK would have been enough otherwise. */ t4_update_port_info(pi); build_medialist(pi); } /* ifm_status */ ifmr->ifm_status = IFM_AVALID; if (lc->link_ok == false) goto done; ifmr->ifm_status |= IFM_ACTIVE; /* ifm_active */ ifmr->ifm_active = IFM_ETHER | IFM_FDX; ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE); if (lc->fc & PAUSE_RX) ifmr->ifm_active |= IFM_ETH_RXPAUSE; if (lc->fc & PAUSE_TX) ifmr->ifm_active |= IFM_ETH_TXPAUSE; ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed)); done: PORT_UNLOCK(pi); end_synchronized_op(sc, 0); } static int vcxgbe_probe(device_t dev) { struct vi_info *vi = device_get_softc(dev); device_set_descf(dev, "port %d vi %td", vi->pi->port_id, vi - vi->pi->vi); return (BUS_PROBE_DEFAULT); } static int alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi) { int func, index, rc; uint32_t param, val; ASSERT_SYNCHRONIZED_OP(sc); index = vi - pi->vi; MPASS(index > 0); /* This function deals with _extra_ VIs only */ KASSERT(index < nitems(vi_mac_funcs), ("%s: VI %s doesn't have a MAC func", __func__, device_get_nameunit(vi->dev))); func = vi_mac_funcs[index]; rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0); if (rc < 0) { CH_ERR(vi, "failed to allocate virtual interface %d" "for port %d: %d\n", index, pi->port_id, -rc); return (-rc); } vi->viid = rc; if (vi->rss_size == 1) { /* * This VI didn't get a slice of the RSS table. Reduce the * number of VIs being created (hw.cxgbe.num_vis) or modify the * configuration file (nvi, rssnvi for this PF) if this is a * problem. */ device_printf(vi->dev, "RSS table not available.\n"); vi->rss_base = 0xffff; return (0); } param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | V_FW_PARAMS_PARAM_YZ(vi->viid); rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc) vi->rss_base = 0xffff; else { MPASS((val >> 16) == vi->rss_size); vi->rss_base = val & 0xffff; } return (0); } static int vcxgbe_attach(device_t dev) { struct vi_info *vi; struct port_info *pi; struct adapter *sc; int rc; vi = device_get_softc(dev); pi = vi->pi; sc = pi->adapter; rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via"); if (rc) return (rc); rc = alloc_extra_vi(sc, pi, vi); end_synchronized_op(sc, 0); if (rc) return (rc); cxgbe_vi_attach(dev, vi); return (0); } static int vcxgbe_detach(device_t dev) { struct vi_info *vi; struct adapter *sc; vi = device_get_softc(dev); sc = vi->adapter; begin_vi_detach(sc, vi); cxgbe_vi_detach(vi); t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); end_vi_detach(sc, vi); return (0); } static struct callout fatal_callout; static struct taskqueue *reset_tq; static void delayed_panic(void *arg) { struct adapter *sc = arg; panic("%s: panic on fatal error", device_get_nameunit(sc->dev)); } static void fatal_error_task(void *arg, int pending) { struct adapter *sc = arg; int rc; if (atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_CIM_ERR))) { dump_cim_regs(sc); dump_cimla(sc); dump_devlog(sc); } if (t4_reset_on_fatal_err) { CH_ALERT(sc, "resetting adapter after fatal error.\n"); rc = reset_adapter(sc); if (rc == 0 && t4_panic_on_fatal_err) { CH_ALERT(sc, "reset was successful, " "system will NOT panic.\n"); return; } } if (t4_panic_on_fatal_err) { CH_ALERT(sc, "panicking on fatal error (after 30s).\n"); callout_reset(&fatal_callout, hz * 30, delayed_panic, sc); } } void t4_fatal_err(struct adapter *sc, bool fw_error) { const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0; stop_adapter(sc); if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR))) return; if (fw_error) { /* * We are here because of a firmware error/timeout and not * because of a hardware interrupt. It is possible (although * not very likely) that an error interrupt was also raised but * this thread ran first and inhibited t4_intr_err. We walk the * main INT_CAUSE registers here to make sure we haven't missed * anything interesting. */ t4_slow_intr_handler(sc, verbose); atomic_set_int(&sc->error_flags, ADAP_CIM_ERR); } t4_report_fw_error(sc); log(LOG_ALERT, "%s: encountered fatal error, adapter stopped (%d).\n", device_get_nameunit(sc->dev), fw_error); taskqueue_enqueue(reset_tq, &sc->fatal_error_task); } void t4_add_adapter(struct adapter *sc) { sx_xlock(&t4_list_lock); SLIST_INSERT_HEAD(&t4_list, sc, link); sx_xunlock(&t4_list_lock); } int t4_map_bars_0_and_4(struct adapter *sc) { sc->regs_rid = PCIR_BAR(0); sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->regs_rid, RF_ACTIVE); if (sc->regs_res == NULL) { device_printf(sc->dev, "cannot map registers.\n"); return (ENXIO); } sc->bt = rman_get_bustag(sc->regs_res); sc->bh = rman_get_bushandle(sc->regs_res); sc->mmio_len = rman_get_size(sc->regs_res); setbit(&sc->doorbells, DOORBELL_KDB); sc->msix_rid = PCIR_BAR(4); sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->msix_rid, RF_ACTIVE); if (sc->msix_res == NULL) { device_printf(sc->dev, "cannot map MSI-X BAR.\n"); return (ENXIO); } return (0); } int t4_map_bar_2(struct adapter *sc) { /* * T4: only iWARP driver uses the userspace doorbells. There is no need * to map it if RDMA is disabled. */ if (is_t4(sc) && sc->rdmacaps == 0) return (0); sc->udbs_rid = PCIR_BAR(2); sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &sc->udbs_rid, RF_ACTIVE); if (sc->udbs_res == NULL) { device_printf(sc->dev, "cannot map doorbell BAR.\n"); return (ENXIO); } sc->udbs_base = rman_get_virtual(sc->udbs_res); if (chip_id(sc) >= CHELSIO_T5) { setbit(&sc->doorbells, DOORBELL_UDB); #if defined(__i386__) || defined(__amd64__) if (t5_write_combine) { int rc, mode; /* * Enable write combining on BAR2. This is the * userspace doorbell BAR and is split into 128B * (UDBS_SEG_SIZE) doorbell regions, each associated * with an egress queue. The first 64B has the doorbell * and the second 64B can be used to submit a tx work * request with an implicit doorbell. */ rc = pmap_change_attr((vm_offset_t)sc->udbs_base, rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); if (rc == 0) { clrbit(&sc->doorbells, DOORBELL_UDB); setbit(&sc->doorbells, DOORBELL_WCWR); setbit(&sc->doorbells, DOORBELL_UDBWC); } else { device_printf(sc->dev, "couldn't enable write combining: %d\n", rc); } mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0); t4_write_reg(sc, A_SGE_STAT_CFG, V_STATSOURCE_T5(7) | mode); } #endif } sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0; return (0); } int t4_adj_doorbells(struct adapter *sc) { if ((sc->doorbells & t4_doorbells_allowed) != 0) { sc->doorbells &= t4_doorbells_allowed; return (0); } CH_ERR(sc, "No usable doorbell (available = 0x%x, allowed = 0x%x).\n", sc->doorbells, t4_doorbells_allowed); return (EINVAL); } struct memwin_init { uint32_t base; uint32_t aperture; }; static const struct memwin_init t4_memwin[NUM_MEMWIN] = { { MEMWIN0_BASE, MEMWIN0_APERTURE }, { MEMWIN1_BASE, MEMWIN1_APERTURE }, { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } }; static const struct memwin_init t5_memwin[NUM_MEMWIN] = { { MEMWIN0_BASE, MEMWIN0_APERTURE }, { MEMWIN1_BASE, MEMWIN1_APERTURE }, { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, }; static void setup_memwin(struct adapter *sc) { const struct memwin_init *mw_init; struct memwin *mw; int i; uint32_t bar0; if (is_t4(sc)) { /* * Read low 32b of bar0 indirectly via the hardware backdoor * mechanism. Works from within PCI passthrough environments * too, where rman_get_start() can return a different value. We * need to program the T4 memory window decoders with the actual * addresses that will be coming across the PCIe link. */ bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; mw_init = &t4_memwin[0]; } else { /* T5+ use the relative offset inside the PCIe BAR */ bar0 = 0; mw_init = &t5_memwin[0]; } for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) { if (!rw_initialized(&mw->mw_lock)) { rw_init(&mw->mw_lock, "memory window access"); mw->mw_base = mw_init->base; mw->mw_aperture = mw_init->aperture; mw->mw_curpos = 0; } t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), (mw->mw_base + bar0) | V_BIR(0) | V_WINDOW(ilog2(mw->mw_aperture) - 10)); rw_wlock(&mw->mw_lock); position_memwin(sc, i, mw->mw_curpos); rw_wunlock(&mw->mw_lock); } /* flush */ t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); } /* * Positions the memory window at the given address in the card's address space. * There are some alignment requirements and the actual position may be at an * address prior to the requested address. mw->mw_curpos always has the actual * position of the window. */ static void position_memwin(struct adapter *sc, int idx, uint32_t addr) { struct memwin *mw; uint32_t pf; uint32_t reg; MPASS(idx >= 0 && idx < NUM_MEMWIN); mw = &sc->memwin[idx]; rw_assert(&mw->mw_lock, RA_WLOCKED); if (is_t4(sc)) { pf = 0; mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */ } else { pf = V_PFNUM(sc->pf); mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */ } reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx); t4_write_reg(sc, reg, mw->mw_curpos | pf); t4_read_reg(sc, reg); /* flush */ } int rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len, int rw) { struct memwin *mw; uint32_t mw_end, v; MPASS(idx >= 0 && idx < NUM_MEMWIN); /* Memory can only be accessed in naturally aligned 4 byte units */ if (addr & 3 || len & 3 || len <= 0) return (EINVAL); mw = &sc->memwin[idx]; while (len > 0) { rw_rlock(&mw->mw_lock); mw_end = mw->mw_curpos + mw->mw_aperture; if (addr >= mw_end || addr < mw->mw_curpos) { /* Will need to reposition the window */ if (!rw_try_upgrade(&mw->mw_lock)) { rw_runlock(&mw->mw_lock); rw_wlock(&mw->mw_lock); } rw_assert(&mw->mw_lock, RA_WLOCKED); position_memwin(sc, idx, addr); rw_downgrade(&mw->mw_lock); mw_end = mw->mw_curpos + mw->mw_aperture; } rw_assert(&mw->mw_lock, RA_RLOCKED); while (addr < mw_end && len > 0) { if (rw == 0) { v = t4_read_reg(sc, mw->mw_base + addr - mw->mw_curpos); *val++ = le32toh(v); } else { v = *val++; t4_write_reg(sc, mw->mw_base + addr - mw->mw_curpos, htole32(v)); } addr += 4; len -= 4; } rw_runlock(&mw->mw_lock); } return (0); } CTASSERT(M_TID_COOKIE == M_COOKIE); CTASSERT(MAX_ATIDS <= (M_TID_TID + 1)); static void t4_init_atid_table(struct adapter *sc) { struct tid_info *t; int i; t = &sc->tids; if (t->natids == 0) return; MPASS(t->atid_tab == NULL); t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE, M_ZERO | M_WAITOK); mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); t->afree = t->atid_tab; t->atids_in_use = 0; t->atid_alloc_stopped = false; for (i = 1; i < t->natids; i++) t->atid_tab[i - 1].next = &t->atid_tab[i]; t->atid_tab[t->natids - 1].next = NULL; } static void t4_free_atid_table(struct adapter *sc) { struct tid_info *t; t = &sc->tids; KASSERT(t->atids_in_use == 0, ("%s: %d atids still in use.", __func__, t->atids_in_use)); if (mtx_initialized(&t->atid_lock)) mtx_destroy(&t->atid_lock); free(t->atid_tab, M_CXGBE); t->atid_tab = NULL; } static void stop_atid_allocator(struct adapter *sc) { struct tid_info *t = &sc->tids; mtx_lock(&t->atid_lock); t->atid_alloc_stopped = true; mtx_unlock(&t->atid_lock); } static void restart_atid_allocator(struct adapter *sc) { struct tid_info *t = &sc->tids; mtx_lock(&t->atid_lock); KASSERT(t->atids_in_use == 0, ("%s: %d atids still in use.", __func__, t->atids_in_use)); t->atid_alloc_stopped = false; mtx_unlock(&t->atid_lock); } int alloc_atid(struct adapter *sc, void *ctx) { struct tid_info *t = &sc->tids; int atid = -1; mtx_lock(&t->atid_lock); if (t->afree && !t->atid_alloc_stopped) { union aopen_entry *p = t->afree; atid = p - t->atid_tab; MPASS(atid <= M_TID_TID); t->afree = p->next; p->data = ctx; t->atids_in_use++; } mtx_unlock(&t->atid_lock); return (atid); } void * lookup_atid(struct adapter *sc, int atid) { struct tid_info *t = &sc->tids; return (t->atid_tab[atid].data); } void free_atid(struct adapter *sc, int atid) { struct tid_info *t = &sc->tids; union aopen_entry *p = &t->atid_tab[atid]; mtx_lock(&t->atid_lock); p->next = t->afree; t->afree = p; t->atids_in_use--; mtx_unlock(&t->atid_lock); } static void queue_tid_release(struct adapter *sc, int tid) { CXGBE_UNIMPLEMENTED("deferred tid release"); } void release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) { struct wrqe *wr; struct cpl_tid_release *req; wr = alloc_wrqe(sizeof(*req), ctrlq); if (wr == NULL) { queue_tid_release(sc, tid); /* defer */ return; } req = wrtod(wr); INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); t4_wrq_tx(sc, wr); } static int t4_range_cmp(const void *a, const void *b) { return ((const struct t4_range *)a)->start - ((const struct t4_range *)b)->start; } /* * Verify that the memory range specified by the addr/len pair is valid within * the card's address space. */ static int validate_mem_range(struct adapter *sc, uint32_t addr, uint32_t len) { struct t4_range mem_ranges[4], *r, *next; uint32_t em, addr_len; int i, n, remaining; /* Memory can only be accessed in naturally aligned 4 byte units */ if (addr & 3 || len & 3 || len == 0) return (EINVAL); /* Enabled memories */ em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); r = &mem_ranges[0]; n = 0; bzero(r, sizeof(mem_ranges)); if (em & F_EDRAM0_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); r->size = G_EDRAM0_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EDRAM0_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (em & F_EDRAM1_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); r->size = G_EDRAM1_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EDRAM1_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (em & F_EXT_MEM_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); r->size = G_EXT_MEM_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EXT_MEM_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) { addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); r->size = G_EXT_MEM1_SIZE(addr_len) << 20; if (r->size > 0) { r->start = G_EXT_MEM1_BASE(addr_len) << 20; if (addr >= r->start && addr + len <= r->start + r->size) return (0); r++; n++; } } MPASS(n <= nitems(mem_ranges)); if (n > 1) { /* Sort and merge the ranges. */ qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp); /* Start from index 0 and examine the next n - 1 entries. */ r = &mem_ranges[0]; for (remaining = n - 1; remaining > 0; remaining--, r++) { MPASS(r->size > 0); /* r is a valid entry. */ next = r + 1; MPASS(next->size > 0); /* and so is the next one. */ while (r->start + r->size >= next->start) { /* Merge the next one into the current entry. */ r->size = max(r->start + r->size, next->start + next->size) - r->start; n--; /* One fewer entry in total. */ if (--remaining == 0) goto done; /* short circuit */ next++; } if (next != r + 1) { /* * Some entries were merged into r and next * points to the first valid entry that couldn't * be merged. */ MPASS(next->size > 0); /* must be valid */ memcpy(r + 1, next, remaining * sizeof(*r)); #ifdef INVARIANTS /* * This so that the foo->size assertion in the * next iteration of the loop do the right * thing for entries that were pulled up and are * no longer valid. */ MPASS(n < nitems(mem_ranges)); bzero(&mem_ranges[n], (nitems(mem_ranges) - n) * sizeof(struct t4_range)); #endif } } done: /* Done merging the ranges. */ MPASS(n > 0); r = &mem_ranges[0]; for (i = 0; i < n; i++, r++) { if (addr >= r->start && addr + len <= r->start + r->size) return (0); } } return (EFAULT); } static int fwmtype_to_hwmtype(int mtype) { switch (mtype) { case FW_MEMTYPE_EDC0: return (MEM_EDC0); case FW_MEMTYPE_EDC1: return (MEM_EDC1); case FW_MEMTYPE_EXTMEM: return (MEM_MC0); case FW_MEMTYPE_EXTMEM1: return (MEM_MC1); default: panic("%s: cannot translate fw mtype %d.", __func__, mtype); } } /* * Verify that the memory range specified by the memtype/offset/len pair is * valid and lies entirely within the memtype specified. The global address of * the start of the range is returned in addr. */ static int validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len, uint32_t *addr) { uint32_t em, addr_len, maddr; /* Memory can only be accessed in naturally aligned 4 byte units */ if (off & 3 || len & 3 || len == 0) return (EINVAL); em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); switch (fwmtype_to_hwmtype(mtype)) { case MEM_EDC0: if (!(em & F_EDRAM0_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); maddr = G_EDRAM0_BASE(addr_len) << 20; break; case MEM_EDC1: if (!(em & F_EDRAM1_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); maddr = G_EDRAM1_BASE(addr_len) << 20; break; case MEM_MC: if (!(em & F_EXT_MEM_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); maddr = G_EXT_MEM_BASE(addr_len) << 20; break; case MEM_MC1: if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE)) return (EINVAL); addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); maddr = G_EXT_MEM1_BASE(addr_len) << 20; break; default: return (EINVAL); } *addr = maddr + off; /* global address */ return (validate_mem_range(sc, *addr, len)); } static int fixup_devlog_params(struct adapter *sc) { struct devlog_params *dparams = &sc->params.devlog; int rc; rc = validate_mt_off_len(sc, dparams->memtype, dparams->start, dparams->size, &dparams->addr); return (rc); } static void update_nirq(struct intrs_and_queues *iaq, int nports) { iaq->nirq = T4_EXTRA_INTR; iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq); iaq->nirq += nports * iaq->nofldrxq; iaq->nirq += nports * (iaq->num_vis - 1) * max(iaq->nrxq_vi, iaq->nnmrxq_vi); iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi; } /* * Adjust requirements to fit the number of interrupts available. */ static void calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype, int navail) { int old_nirq; const int nports = sc->params.nports; MPASS(nports > 0); MPASS(navail > 0); bzero(iaq, sizeof(*iaq)); iaq->intr_type = itype; iaq->num_vis = t4_num_vis; iaq->ntxq = t4_ntxq; iaq->ntxq_vi = t4_ntxq_vi; iaq->nrxq = t4_nrxq; iaq->nrxq_vi = t4_nrxq_vi; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) if (is_offload(sc) || is_ethoffload(sc)) { iaq->nofldtxq = t4_nofldtxq; iaq->nofldtxq_vi = t4_nofldtxq_vi; } #endif #ifdef TCP_OFFLOAD if (is_offload(sc)) { iaq->nofldrxq = t4_nofldrxq; iaq->nofldrxq_vi = t4_nofldrxq_vi; } #endif #ifdef DEV_NETMAP if (t4_native_netmap & NN_MAIN_VI) { iaq->nnmtxq = t4_nnmtxq; iaq->nnmrxq = t4_nnmrxq; } if (t4_native_netmap & NN_EXTRA_VI) { iaq->nnmtxq_vi = t4_nnmtxq_vi; iaq->nnmrxq_vi = t4_nnmrxq_vi; } #endif update_nirq(iaq, nports); if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) { /* * This is the normal case -- there are enough interrupts for * everything. */ goto done; } /* * If extra VIs have been configured try reducing their count and see if * that works. */ while (iaq->num_vis > 1) { iaq->num_vis--; update_nirq(iaq, nports); if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) { device_printf(sc->dev, "virtual interfaces per port " "reduced to %d from %d. nrxq=%u, nofldrxq=%u, " "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. " "itype %d, navail %u, nirq %d.\n", iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi, itype, navail, iaq->nirq); goto done; } } /* * Extra VIs will not be created. Log a message if they were requested. */ MPASS(iaq->num_vis == 1); iaq->ntxq_vi = iaq->nrxq_vi = 0; iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; if (iaq->num_vis != t4_num_vis) { device_printf(sc->dev, "extra virtual interfaces disabled. " "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, " "nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n", iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi, itype, navail, iaq->nirq); } /* * Keep reducing the number of NIC rx queues to the next lower power of * 2 (for even RSS distribution) and halving the TOE rx queues and see * if that works. */ do { if (iaq->nrxq > 1) { iaq->nrxq = rounddown_pow_of_two(iaq->nrxq - 1); if (iaq->nnmrxq > iaq->nrxq) iaq->nnmrxq = iaq->nrxq; } if (iaq->nofldrxq > 1) iaq->nofldrxq >>= 1; old_nirq = iaq->nirq; update_nirq(iaq, nports); if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) { device_printf(sc->dev, "running with reduced number of " "rx queues because of shortage of interrupts. " "nrxq=%u, nofldrxq=%u. " "itype %d, navail %u, nirq %d.\n", iaq->nrxq, iaq->nofldrxq, itype, navail, iaq->nirq); goto done; } } while (old_nirq != iaq->nirq); /* One interrupt for everything. Ugh. */ device_printf(sc->dev, "running with minimal number of queues. " "itype %d, navail %u.\n", itype, navail); iaq->nirq = 1; iaq->nrxq = 1; iaq->ntxq = 1; if (iaq->nofldrxq > 0) { iaq->nofldrxq = 1; iaq->nofldtxq = 1; } iaq->nnmtxq = 0; iaq->nnmrxq = 0; done: MPASS(iaq->num_vis > 0); if (iaq->num_vis > 1) { MPASS(iaq->nrxq_vi > 0); MPASS(iaq->ntxq_vi > 0); } MPASS(iaq->nirq > 0); MPASS(iaq->nrxq > 0); MPASS(iaq->ntxq > 0); if (itype == INTR_MSI) { MPASS(powerof2(iaq->nirq)); } } static int cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) { int rc, itype, navail, nalloc; for (itype = INTR_MSIX; itype; itype >>= 1) { if ((itype & t4_intr_types) == 0) continue; /* not allowed */ if (itype == INTR_MSIX) navail = pci_msix_count(sc->dev); else if (itype == INTR_MSI) navail = pci_msi_count(sc->dev); else navail = 1; restart: if (navail == 0) continue; calculate_iaq(sc, iaq, itype, navail); nalloc = iaq->nirq; rc = 0; if (itype == INTR_MSIX) rc = pci_alloc_msix(sc->dev, &nalloc); else if (itype == INTR_MSI) rc = pci_alloc_msi(sc->dev, &nalloc); if (rc == 0 && nalloc > 0) { if (nalloc == iaq->nirq) return (0); /* * Didn't get the number requested. Use whatever number * the kernel is willing to allocate. */ device_printf(sc->dev, "fewer vectors than requested, " "type=%d, req=%d, rcvd=%d; will downshift req.\n", itype, iaq->nirq, nalloc); pci_release_msi(sc->dev); navail = nalloc; goto restart; } device_printf(sc->dev, "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", itype, rc, iaq->nirq, nalloc); } device_printf(sc->dev, "failed to find a usable interrupt type. " "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, pci_msix_count(sc->dev), pci_msi_count(sc->dev)); return (ENXIO); } #define FW_VERSION(chip) ( \ V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) /* Just enough of fw_hdr to cover all version info. */ struct fw_h { __u8 ver; __u8 chip; __be16 len512; __be32 fw_ver; __be32 tp_microcode_ver; __u8 intfver_nic; __u8 intfver_vnic; __u8 intfver_ofld; __u8 intfver_ri; __u8 intfver_iscsipdu; __u8 intfver_iscsi; __u8 intfver_fcoepdu; __u8 intfver_fcoe; }; /* Spot check a couple of fields. */ CTASSERT(offsetof(struct fw_h, fw_ver) == offsetof(struct fw_hdr, fw_ver)); CTASSERT(offsetof(struct fw_h, intfver_nic) == offsetof(struct fw_hdr, intfver_nic)); CTASSERT(offsetof(struct fw_h, intfver_fcoe) == offsetof(struct fw_hdr, intfver_fcoe)); struct fw_info { uint8_t chip; char *kld_name; char *fw_mod_name; struct fw_h fw_h; } fw_info[] = { { .chip = CHELSIO_T4, .kld_name = "t4fw_cfg", .fw_mod_name = "t4fw", .fw_h = { .chip = FW_HDR_CHIP_T4, .fw_ver = htobe32(FW_VERSION(T4)), .intfver_nic = FW_INTFVER(T4, NIC), .intfver_vnic = FW_INTFVER(T4, VNIC), .intfver_ofld = FW_INTFVER(T4, OFLD), .intfver_ri = FW_INTFVER(T4, RI), .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T4, ISCSI), .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), .intfver_fcoe = FW_INTFVER(T4, FCOE), }, }, { .chip = CHELSIO_T5, .kld_name = "t5fw_cfg", .fw_mod_name = "t5fw", .fw_h = { .chip = FW_HDR_CHIP_T5, .fw_ver = htobe32(FW_VERSION(T5)), .intfver_nic = FW_INTFVER(T5, NIC), .intfver_vnic = FW_INTFVER(T5, VNIC), .intfver_ofld = FW_INTFVER(T5, OFLD), .intfver_ri = FW_INTFVER(T5, RI), .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T5, ISCSI), .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), .intfver_fcoe = FW_INTFVER(T5, FCOE), }, }, { .chip = CHELSIO_T6, .kld_name = "t6fw_cfg", .fw_mod_name = "t6fw", .fw_h = { .chip = FW_HDR_CHIP_T6, .fw_ver = htobe32(FW_VERSION(T6)), .intfver_nic = FW_INTFVER(T6, NIC), .intfver_vnic = FW_INTFVER(T6, VNIC), .intfver_ofld = FW_INTFVER(T6, OFLD), .intfver_ri = FW_INTFVER(T6, RI), .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), .intfver_iscsi = FW_INTFVER(T6, ISCSI), .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), .intfver_fcoe = FW_INTFVER(T6, FCOE), }, } }; static struct fw_info * find_fw_info(int chip) { int i; for (i = 0; i < nitems(fw_info); i++) { if (fw_info[i].chip == chip) return (&fw_info[i]); } return (NULL); } /* * Is the given firmware API compatible with the one the driver was compiled * with? */ static int fw_compatible(const struct fw_h *hdr1, const struct fw_h *hdr2) { /* short circuit if it's the exact same firmware version */ if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) return (1); /* * XXX: Is this too conservative? Perhaps I should limit this to the * features that are supported in the driver. */ #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) return (1); #undef SAME_INTF return (0); } static int load_fw_module(struct adapter *sc, const struct firmware **dcfg, const struct firmware **fw) { struct fw_info *fw_info; *dcfg = NULL; if (fw != NULL) *fw = NULL; fw_info = find_fw_info(chip_id(sc)); if (fw_info == NULL) { device_printf(sc->dev, "unable to look up firmware information for chip %d.\n", chip_id(sc)); return (EINVAL); } *dcfg = firmware_get(fw_info->kld_name); if (*dcfg != NULL) { if (fw != NULL) *fw = firmware_get(fw_info->fw_mod_name); return (0); } return (ENOENT); } static void unload_fw_module(struct adapter *sc, const struct firmware *dcfg, const struct firmware *fw) { if (fw != NULL) firmware_put(fw, FIRMWARE_UNLOAD); if (dcfg != NULL) firmware_put(dcfg, FIRMWARE_UNLOAD); } /* * Return values: * 0 means no firmware install attempted. * ERESTART means a firmware install was attempted and was successful. * +ve errno means a firmware install was attempted but failed. */ static int install_kld_firmware(struct adapter *sc, struct fw_h *card_fw, const struct fw_h *drv_fw, const char *reason, int *already) { const struct firmware *cfg, *fw; const uint32_t c = be32toh(card_fw->fw_ver); uint32_t d, k; int rc, fw_install; struct fw_h bundled_fw; bool load_attempted; cfg = fw = NULL; load_attempted = false; fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install; memcpy(&bundled_fw, drv_fw, sizeof(bundled_fw)); if (t4_fw_install < 0) { rc = load_fw_module(sc, &cfg, &fw); if (rc != 0 || fw == NULL) { device_printf(sc->dev, "failed to load firmware module: %d. cfg %p, fw %p;" " will use compiled-in firmware version for" "hw.cxgbe.fw_install checks.\n", rc, cfg, fw); } else { memcpy(&bundled_fw, fw->data, sizeof(bundled_fw)); } load_attempted = true; } d = be32toh(bundled_fw.fw_ver); if (reason != NULL) goto install; if ((sc->flags & FW_OK) == 0) { if (c == 0xffffffff) { reason = "missing"; goto install; } rc = 0; goto done; } if (!fw_compatible(card_fw, &bundled_fw)) { reason = "incompatible or unusable"; goto install; } if (d > c) { reason = "older than the version bundled with this driver"; goto install; } if (fw_install == 2 && d != c) { reason = "different than the version bundled with this driver"; goto install; } /* No reason to do anything to the firmware already on the card. */ rc = 0; goto done; install: rc = 0; if ((*already)++) goto done; if (fw_install == 0) { device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " "but the driver is prohibited from installing a firmware " "on the card.\n", G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); goto done; } /* * We'll attempt to install a firmware. Load the module first (if it * hasn't been loaded already). */ if (!load_attempted) { rc = load_fw_module(sc, &cfg, &fw); if (rc != 0 || fw == NULL) { device_printf(sc->dev, "failed to load firmware module: %d. cfg %p, fw %p\n", rc, cfg, fw); /* carry on */ } } if (fw == NULL) { device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " "but the driver cannot take corrective action because it " "is unable to load the firmware module.\n", G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); rc = sc->flags & FW_OK ? 0 : ENOENT; goto done; } k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver); if (k != d) { MPASS(t4_fw_install > 0); device_printf(sc->dev, "firmware in KLD (%u.%u.%u.%u) is not what the driver was " "expecting (%u.%u.%u.%u) and will not be used.\n", G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k), G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); rc = sc->flags & FW_OK ? 0 : EINVAL; goto done; } device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " "installing firmware %u.%u.%u.%u on card.\n", G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d)); rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); if (rc != 0) { device_printf(sc->dev, "failed to install firmware: %d\n", rc); } else { /* Installed successfully, update the cached header too. */ rc = ERESTART; memcpy(card_fw, fw->data, sizeof(*card_fw)); } done: unload_fw_module(sc, cfg, fw); return (rc); } /* * Establish contact with the firmware and attempt to become the master driver. * * A firmware will be installed to the card if needed (if the driver is allowed * to do so). */ static int contact_firmware(struct adapter *sc) { int rc, already = 0; enum dev_state state; struct fw_info *fw_info; struct fw_hdr *card_fw; /* fw on the card */ const struct fw_h *drv_fw; fw_info = find_fw_info(chip_id(sc)); if (fw_info == NULL) { device_printf(sc->dev, "unable to look up firmware information for chip %d.\n", chip_id(sc)); return (EINVAL); } drv_fw = &fw_info->fw_h; /* Read the header of the firmware on the card */ card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); restart: rc = -t4_get_fw_hdr(sc, card_fw); if (rc != 0) { device_printf(sc->dev, "unable to read firmware header from card's flash: %d\n", rc); goto done; } rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL, &already); if (rc == ERESTART) goto restart; if (rc != 0) goto done; rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); if (rc < 0 || state == DEV_STATE_ERR) { rc = -rc; device_printf(sc->dev, "failed to connect to the firmware: %d, %d. " "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); #if 0 if (install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, "not responding properly to HELLO", &already) == ERESTART) goto restart; #endif goto done; } MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT); sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */ if (rc == sc->pf) { sc->flags |= MASTER_PF; rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL, &already); if (rc == ERESTART) rc = 0; else if (rc != 0) goto done; } else if (state == DEV_STATE_UNINIT) { /* * We didn't get to be the master so we definitely won't be * configuring the chip. It's a bug if someone else hasn't * configured it already. */ device_printf(sc->dev, "couldn't be master(%d), " "device not already initialized either(%d). " "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); rc = EPROTO; goto done; } else { /* * Some other PF is the master and has configured the chip. * This is allowed but untested. */ device_printf(sc->dev, "PF%d is master, device state %d. " "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW)); snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc); sc->cfcsum = 0; rc = 0; } done: if (rc != 0 && sc->flags & FW_OK) { t4_fw_bye(sc, sc->mbox); sc->flags &= ~FW_OK; } free(card_fw, M_CXGBE); return (rc); } static int copy_cfg_file_to_card(struct adapter *sc, char *cfg_file, uint32_t mtype, uint32_t moff) { struct fw_info *fw_info; const struct firmware *dcfg, *rcfg = NULL; const uint32_t *cfdata; uint32_t cflen, addr; int rc; load_fw_module(sc, &dcfg, NULL); /* Card specific interpretation of "default". */ if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { if (pci_get_device(sc->dev) == 0x440a) snprintf(cfg_file, sizeof(t4_cfg_file), UWIRE_CF); if (is_fpga(sc)) snprintf(cfg_file, sizeof(t4_cfg_file), FPGA_CF); } if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { if (dcfg == NULL) { device_printf(sc->dev, "KLD with default config is not available.\n"); rc = ENOENT; goto done; } cfdata = dcfg->data; cflen = dcfg->datasize & ~3; } else { char s[32]; fw_info = find_fw_info(chip_id(sc)); if (fw_info == NULL) { device_printf(sc->dev, "unable to look up firmware information for chip %d.\n", chip_id(sc)); rc = EINVAL; goto done; } snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file); rcfg = firmware_get(s); if (rcfg == NULL) { device_printf(sc->dev, "unable to load module \"%s\" for configuration " "profile \"%s\".\n", s, cfg_file); rc = ENOENT; goto done; } cfdata = rcfg->data; cflen = rcfg->datasize & ~3; } if (cflen > FLASH_CFG_MAX_SIZE) { device_printf(sc->dev, "config file too long (%d, max allowed is %d).\n", cflen, FLASH_CFG_MAX_SIZE); rc = EINVAL; goto done; } rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); if (rc != 0) { device_printf(sc->dev, "%s: addr (%d/0x%x) or len %d is not valid: %d.\n", __func__, mtype, moff, cflen, rc); rc = EINVAL; goto done; } write_via_memwin(sc, 2, addr, cfdata, cflen); done: if (rcfg != NULL) firmware_put(rcfg, FIRMWARE_UNLOAD); unload_fw_module(sc, dcfg, NULL); return (rc); } struct caps_allowed { uint16_t nbmcaps; uint16_t linkcaps; uint16_t switchcaps; uint16_t niccaps; uint16_t toecaps; uint16_t rdmacaps; uint16_t cryptocaps; uint16_t iscsicaps; uint16_t fcoecaps; }; #define FW_PARAM_DEV(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) #define FW_PARAM_PFVF(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) /* * Provide a configuration profile to the firmware and have it initialize the * chip accordingly. This may involve uploading a configuration file to the * card. */ static int apply_cfg_and_initialize(struct adapter *sc, char *cfg_file, const struct caps_allowed *caps_allowed) { int rc; struct fw_caps_config_cmd caps; uint32_t mtype, moff, finicsum, cfcsum, param, val; rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); if (rc != 0) { device_printf(sc->dev, "firmware reset failed: %d.\n", rc); return (rc); } bzero(&caps, sizeof(caps)); caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); if (strncmp(cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) { mtype = 0; moff = 0; caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); } else if (strncmp(cfg_file, FLASH_CF, sizeof(t4_cfg_file)) == 0) { mtype = FW_MEMTYPE_FLASH; moff = t4_flash_cfg_addr(sc); caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); } else { /* * Ask the firmware where it wants us to upload the config file. */ param = FW_PARAM_DEV(CF); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc != 0) { /* No support for config file? Shouldn't happen. */ device_printf(sc->dev, "failed to query config file location: %d.\n", rc); goto done; } mtype = G_FW_PARAMS_PARAM_Y(val); moff = G_FW_PARAMS_PARAM_Z(val) << 16; caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); rc = copy_cfg_file_to_card(sc, cfg_file, mtype, moff); if (rc != 0) { device_printf(sc->dev, "failed to upload config file to card: %d.\n", rc); goto done; } } rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); if (rc != 0) { device_printf(sc->dev, "failed to pre-process config file: %d " "(mtype %d, moff 0x%x).\n", rc, mtype, moff); goto done; } finicsum = be32toh(caps.finicsum); cfcsum = be32toh(caps.cfcsum); /* actual */ if (finicsum != cfcsum) { device_printf(sc->dev, "WARNING: config file checksum mismatch: %08x %08x\n", finicsum, cfcsum); } sc->cfcsum = cfcsum; snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file); /* * Let the firmware know what features will (not) be used so it can tune * things accordingly. */ #define LIMIT_CAPS(x) do { \ caps.x##caps &= htobe16(caps_allowed->x##caps); \ } while (0) LIMIT_CAPS(nbm); LIMIT_CAPS(link); LIMIT_CAPS(switch); LIMIT_CAPS(nic); LIMIT_CAPS(toe); LIMIT_CAPS(rdma); LIMIT_CAPS(crypto); LIMIT_CAPS(iscsi); LIMIT_CAPS(fcoe); #undef LIMIT_CAPS if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) { /* * TOE and hashfilters are mutually exclusive. It is a config * file or firmware bug if both are reported as available. Try * to cope with the situation in non-debug builds by disabling * TOE. */ MPASS(caps.toecaps == 0); caps.toecaps = 0; caps.rdmacaps = 0; caps.iscsicaps = 0; } caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE); caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); if (rc != 0) { device_printf(sc->dev, "failed to process config file: %d.\n", rc); goto done; } t4_tweak_chip_settings(sc); set_params__pre_init(sc); /* get basic stuff going */ rc = -t4_fw_initialize(sc, sc->mbox); if (rc != 0) { device_printf(sc->dev, "fw_initialize failed: %d.\n", rc); goto done; } done: return (rc); } /* * Partition chip resources for use between various PFs, VFs, etc. */ static int partition_resources(struct adapter *sc) { char cfg_file[sizeof(t4_cfg_file)]; struct caps_allowed caps_allowed; int rc; bool fallback; /* Only the master driver gets to configure the chip resources. */ MPASS(sc->flags & MASTER_PF); #define COPY_CAPS(x) do { \ caps_allowed.x##caps = t4_##x##caps_allowed; \ } while (0) bzero(&caps_allowed, sizeof(caps_allowed)); COPY_CAPS(nbm); COPY_CAPS(link); COPY_CAPS(switch); COPY_CAPS(nic); COPY_CAPS(toe); COPY_CAPS(rdma); COPY_CAPS(crypto); COPY_CAPS(iscsi); COPY_CAPS(fcoe); fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true; snprintf(cfg_file, sizeof(cfg_file), "%s", t4_cfg_file); retry: rc = apply_cfg_and_initialize(sc, cfg_file, &caps_allowed); if (rc != 0 && fallback) { dump_devlog(sc); device_printf(sc->dev, "failed (%d) to configure card with \"%s\" profile, " "will fall back to a basic configuration and retry.\n", rc, cfg_file); snprintf(cfg_file, sizeof(cfg_file), "%s", BUILTIN_CF); bzero(&caps_allowed, sizeof(caps_allowed)); COPY_CAPS(switch); caps_allowed.niccaps = FW_CAPS_CONFIG_NIC; fallback = false; goto retry; } #undef COPY_CAPS return (rc); } /* * Retrieve parameters that are needed (or nice to have) very early. */ static int get_params__pre_init(struct adapter *sc) { int rc; uint32_t param[2], val[2]; t4_get_version_info(sc); snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers), G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers), G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers), G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers)); snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers), G_FW_HDR_FW_VER_MINOR(sc->params.er_vers), G_FW_HDR_FW_VER_MICRO(sc->params.er_vers), G_FW_HDR_FW_VER_BUILD(sc->params.er_vers)); param[0] = FW_PARAM_DEV(PORTVEC); param[1] = FW_PARAM_DEV(CCLK); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (pre_init): %d.\n", rc); return (rc); } sc->params.portvec = val[0]; sc->params.nports = bitcount32(val[0]); sc->params.vpd.cclk = val[1]; /* Read device log parameters. */ rc = -t4_init_devlog_params(sc, 1); if (rc == 0) fixup_devlog_params(sc); else { device_printf(sc->dev, "failed to get devlog parameters: %d.\n", rc); rc = 0; /* devlog isn't critical for device operation */ } return (rc); } /* * Any params that need to be set before FW_INITIALIZE. */ static int set_params__pre_init(struct adapter *sc) { int rc = 0; uint32_t param, val; if (chip_id(sc) >= CHELSIO_T6) { param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT); val = 1; rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); /* firmwares < 1.20.1.0 do not have this param. */ if (rc == FW_EINVAL && sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) { rc = 0; } if (rc != 0) { device_printf(sc->dev, "failed to enable high priority filters :%d.\n", rc); } param = FW_PARAM_DEV(PPOD_EDRAM); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc == 0 && val == 1) { rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc != 0) { device_printf(sc->dev, "failed to set PPOD_EDRAM: %d.\n", rc); } } } /* Enable opaque VIIDs with firmwares that support it. */ param = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN); val = 1; rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); if (rc == 0 && val == 1) sc->params.viid_smt_extn_support = true; else sc->params.viid_smt_extn_support = false; return (rc); } /* * Retrieve various parameters that are of interest to the driver. The device * has been initialized by the firmware at this point. */ static int get_params__post_init(struct adapter *sc) { int rc; uint32_t param[7], val[7]; struct fw_caps_config_cmd caps; param[0] = FW_PARAM_PFVF(IQFLINT_START); param[1] = FW_PARAM_PFVF(EQ_START); param[2] = FW_PARAM_PFVF(FILTER_START); param[3] = FW_PARAM_PFVF(FILTER_END); param[4] = FW_PARAM_PFVF(L2T_START); param[5] = FW_PARAM_PFVF(L2T_END); param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (post_init): %d.\n", rc); return (rc); } sc->sge.iq_start = val[0]; sc->sge.eq_start = val[1]; if ((int)val[3] > (int)val[2]) { sc->tids.ftid_base = val[2]; sc->tids.ftid_end = val[3]; sc->tids.nftids = val[3] - val[2] + 1; } sc->vres.l2t.start = val[4]; sc->vres.l2t.size = val[5] - val[4] + 1; /* val[5] is the last hwidx and it must not collide with F_SYNC_WR */ if (sc->vres.l2t.size > 0) MPASS(fls(val[5]) <= S_SYNC_WR); sc->params.core_vdd = val[6]; param[0] = FW_PARAM_PFVF(IQFLINT_END); param[1] = FW_PARAM_PFVF(EQ_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (post_init2): %d.\n", rc); return (rc); } MPASS((int)val[0] >= sc->sge.iq_start); sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1; MPASS((int)val[1] >= sc->sge.eq_start); sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1; if (chip_id(sc) >= CHELSIO_T6) { sc->tids.tid_base = t4_read_reg(sc, A_LE_DB_ACTIVE_TABLE_START_INDEX); param[0] = FW_PARAM_PFVF(HPFILTER_START); param[1] = FW_PARAM_PFVF(HPFILTER_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query hpfilter parameters: %d.\n", rc); return (rc); } if ((int)val[1] > (int)val[0]) { sc->tids.hpftid_base = val[0]; sc->tids.hpftid_end = val[1]; sc->tids.nhpftids = val[1] - val[0] + 1; /* * These should go off if the layout changes and the * driver needs to catch up. */ MPASS(sc->tids.hpftid_base == 0); MPASS(sc->tids.tid_base == sc->tids.nhpftids); } param[0] = FW_PARAM_PFVF(RAWF_START); param[1] = FW_PARAM_PFVF(RAWF_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query rawf parameters: %d.\n", rc); return (rc); } if ((int)val[1] > (int)val[0]) { sc->rawf_base = val[0]; sc->nrawf = val[1] - val[0] + 1; } } /* * The parameters that follow may not be available on all firmwares. We * query them individually rather than in a compound query because old * firmwares fail the entire query if an unknown parameter is queried. */ /* * MPS buffer group configuration. */ param[0] = FW_PARAM_DEV(MPSBGMAP); val[0] = 0; rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.mps_bg_map = val[0]; else sc->params.mps_bg_map = UINT32_MAX; /* Not a legal value. */ param[0] = FW_PARAM_DEV(TPCHMAP); val[0] = 0; rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.tp_ch_map = val[0]; else sc->params.tp_ch_map = UINT32_MAX; /* Not a legal value. */ /* * Determine whether the firmware supports the filter2 work request. */ param[0] = FW_PARAM_DEV(FILTER2_WR); val[0] = 0; rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.filter2_wr_support = val[0] != 0; else sc->params.filter2_wr_support = 0; /* * Find out whether we're allowed to use the ULPTX MEMWRITE DSGL. */ param[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); val[0] = 0; rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.ulptx_memwrite_dsgl = val[0] != 0; else sc->params.ulptx_memwrite_dsgl = false; /* FW_RI_FR_NSMR_TPTE_WR support */ param[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.fr_nsmr_tpte_wr_support = val[0] != 0; else sc->params.fr_nsmr_tpte_wr_support = false; /* Support for 512 SGL entries per FR MR. */ param[0] = FW_PARAM_DEV(DEV_512SGL_MR); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.dev_512sgl_mr = val[0] != 0; else sc->params.dev_512sgl_mr = false; param[0] = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) sc->params.max_pkts_per_eth_tx_pkts_wr = val[0]; else sc->params.max_pkts_per_eth_tx_pkts_wr = 15; param[0] = FW_PARAM_DEV(NUM_TM_CLASS); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc == 0) { MPASS(val[0] > 0 && val[0] < 256); /* nsched_cls is 8b */ sc->params.nsched_cls = val[0]; } else sc->params.nsched_cls = sc->chip_params->nsched_cls; /* get capabilites */ bzero(&caps, sizeof(caps)); caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ); caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); if (rc != 0) { device_printf(sc->dev, "failed to get card capabilities: %d.\n", rc); return (rc); } #define READ_CAPS(x) do { \ sc->x = htobe16(caps.x); \ } while (0) READ_CAPS(nbmcaps); READ_CAPS(linkcaps); READ_CAPS(switchcaps); READ_CAPS(niccaps); READ_CAPS(toecaps); READ_CAPS(rdmacaps); READ_CAPS(cryptocaps); READ_CAPS(iscsicaps); READ_CAPS(fcoecaps); if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) { MPASS(chip_id(sc) > CHELSIO_T4); MPASS(sc->toecaps == 0); sc->toecaps = 0; param[0] = FW_PARAM_DEV(NTID); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query HASHFILTER parameters: %d.\n", rc); return (rc); } sc->tids.ntids = val[0]; if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) { MPASS(sc->tids.ntids >= sc->tids.nhpftids); sc->tids.ntids -= sc->tids.nhpftids; } sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); sc->params.hash_filter = 1; } if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { param[0] = FW_PARAM_PFVF(ETHOFLD_START); param[1] = FW_PARAM_PFVF(ETHOFLD_END); param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query NIC parameters: %d.\n", rc); return (rc); } if ((int)val[1] > (int)val[0]) { sc->tids.etid_base = val[0]; sc->tids.etid_end = val[1]; sc->tids.netids = val[1] - val[0] + 1; sc->params.eo_wr_cred = val[2]; sc->params.ethoffload = 1; } } if (sc->toecaps) { /* query offload-related parameters */ param[0] = FW_PARAM_DEV(NTID); param[1] = FW_PARAM_PFVF(SERVER_START); param[2] = FW_PARAM_PFVF(SERVER_END); param[3] = FW_PARAM_PFVF(TDDP_START); param[4] = FW_PARAM_PFVF(TDDP_END); param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query TOE parameters: %d.\n", rc); return (rc); } sc->tids.ntids = val[0]; if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) { MPASS(sc->tids.ntids >= sc->tids.nhpftids); sc->tids.ntids -= sc->tids.nhpftids; } sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); if ((int)val[2] > (int)val[1]) { sc->tids.stid_base = val[1]; sc->tids.nstids = val[2] - val[1] + 1; } sc->vres.ddp.start = val[3]; sc->vres.ddp.size = val[4] - val[3] + 1; sc->params.ofldq_wr_cred = val[5]; sc->params.offload = 1; } else { /* * The firmware attempts memfree TOE configuration for -SO cards * and will report toecaps=0 if it runs out of resources (this * depends on the config file). It may not report 0 for other * capabilities dependent on the TOE in this case. Set them to * 0 here so that the driver doesn't bother tracking resources * that will never be used. */ sc->iscsicaps = 0; sc->rdmacaps = 0; } if (sc->rdmacaps) { param[0] = FW_PARAM_PFVF(STAG_START); param[1] = FW_PARAM_PFVF(STAG_END); param[2] = FW_PARAM_PFVF(RQ_START); param[3] = FW_PARAM_PFVF(RQ_END); param[4] = FW_PARAM_PFVF(PBL_START); param[5] = FW_PARAM_PFVF(PBL_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(1): %d.\n", rc); return (rc); } sc->vres.stag.start = val[0]; sc->vres.stag.size = val[1] - val[0] + 1; sc->vres.rq.start = val[2]; sc->vres.rq.size = val[3] - val[2] + 1; sc->vres.pbl.start = val[4]; sc->vres.pbl.size = val[5] - val[4] + 1; param[0] = FW_PARAM_PFVF(SQRQ_START); param[1] = FW_PARAM_PFVF(SQRQ_END); param[2] = FW_PARAM_PFVF(CQ_START); param[3] = FW_PARAM_PFVF(CQ_END); param[4] = FW_PARAM_PFVF(OCQ_START); param[5] = FW_PARAM_PFVF(OCQ_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(2): %d.\n", rc); return (rc); } sc->vres.qp.start = val[0]; sc->vres.qp.size = val[1] - val[0] + 1; sc->vres.cq.start = val[2]; sc->vres.cq.size = val[3] - val[2] + 1; sc->vres.ocq.start = val[4]; sc->vres.ocq.size = val[5] - val[4] + 1; param[0] = FW_PARAM_PFVF(SRQ_START); param[1] = FW_PARAM_PFVF(SRQ_END); param[2] = FW_PARAM_DEV(MAXORDIRD_QP); param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query RDMA parameters(3): %d.\n", rc); return (rc); } sc->vres.srq.start = val[0]; sc->vres.srq.size = val[1] - val[0] + 1; sc->params.max_ordird_qp = val[2]; sc->params.max_ird_adapter = val[3]; } if (sc->iscsicaps) { param[0] = FW_PARAM_PFVF(ISCSI_START); param[1] = FW_PARAM_PFVF(ISCSI_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query iSCSI parameters: %d.\n", rc); return (rc); } sc->vres.iscsi.start = val[0]; sc->vres.iscsi.size = val[1] - val[0] + 1; } if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) { param[0] = FW_PARAM_PFVF(TLS_START); param[1] = FW_PARAM_PFVF(TLS_END); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); if (rc != 0) { device_printf(sc->dev, "failed to query TLS parameters: %d.\n", rc); return (rc); } sc->vres.key.start = val[0]; sc->vres.key.size = val[1] - val[0] + 1; } /* * We've got the params we wanted to query directly from the firmware. * Grab some others via other means. */ t4_init_sge_params(sc); t4_init_tp_params(sc); t4_read_mtu_tbl(sc, sc->params.mtus, NULL); t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); rc = t4_verify_chip_settings(sc); if (rc != 0) return (rc); t4_init_rx_buf_info(sc); return (rc); } #ifdef KERN_TLS static void ktls_tick(void *arg) { struct adapter *sc; uint32_t tstamp; sc = arg; tstamp = tcp_ts_getticks(); t4_write_reg(sc, A_TP_SYNC_TIME_HI, tstamp >> 1); t4_write_reg(sc, A_TP_SYNC_TIME_LO, tstamp << 31); callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK); } static int t6_config_kern_tls(struct adapter *sc, bool enable) { int rc; uint32_t param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_KTLS_HW) | V_FW_PARAMS_PARAM_Y(enable ? 1 : 0) | V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_KTLS_HW_USER_ENABLE); rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, ¶m); if (rc != 0) { CH_ERR(sc, "failed to %s NIC TLS: %d\n", enable ? "enable" : "disable", rc); return (rc); } if (enable) { sc->flags |= KERN_TLS_ON; callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc, C_HARDCLOCK); } else { sc->flags &= ~KERN_TLS_ON; callout_stop(&sc->ktls_tick); } return (rc); } #endif static int set_params__post_init(struct adapter *sc) { uint32_t mask, param, val; #ifdef TCP_OFFLOAD int i, v, shift; #endif /* ask for encapsulated CPLs */ param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); val = 1; (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); /* Enable 32b port caps if the firmware supports it. */ param = FW_PARAM_PFVF(PORT_CAPS32); val = 1; if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0) sc->params.port_caps32 = 1; /* Let filter + maskhash steer to a part of the VI's RSS region. */ val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1); t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER), V_MASKFILTER(val - 1)); mask = F_DROPERRORANY | F_DROPERRORMAC | F_DROPERRORIPVER | F_DROPERRORFRAG | F_DROPERRORATTACK | F_DROPERRORETHHDRLEN | F_DROPERRORIPHDRLEN | F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN | F_DROPERRORTCPOPT | F_DROPERRORCSUMIP | F_DROPERRORCSUM; val = 0; if (chip_id(sc) < CHELSIO_T6 && t4_attack_filter != 0) { t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_ATTACKFILTERENABLE, F_ATTACKFILTERENABLE); val |= F_DROPERRORATTACK; } if (t4_drop_ip_fragments != 0) { t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_FRAGMENTDROP, F_FRAGMENTDROP); val |= F_DROPERRORFRAG; } if (t4_drop_pkts_with_l2_errors != 0) val |= F_DROPERRORMAC | F_DROPERRORETHHDRLEN; if (t4_drop_pkts_with_l3_errors != 0) { val |= F_DROPERRORIPVER | F_DROPERRORIPHDRLEN | F_DROPERRORCSUMIP; } if (t4_drop_pkts_with_l4_errors != 0) { val |= F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN | F_DROPERRORTCPOPT | F_DROPERRORCSUM; } t4_set_reg_field(sc, A_TP_ERR_CONFIG, mask, val); #ifdef TCP_OFFLOAD /* * Override the TOE timers with user provided tunables. This is not the * recommended way to change the timers (the firmware config file is) so * these tunables are not documented. * * All the timer tunables are in microseconds. */ if (t4_toe_keepalive_idle != 0) { v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle); v &= M_KEEPALIVEIDLE; t4_set_reg_field(sc, A_TP_KEEP_IDLE, V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v)); } if (t4_toe_keepalive_interval != 0) { v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval); v &= M_KEEPALIVEINTVL; t4_set_reg_field(sc, A_TP_KEEP_INTVL, V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v)); } if (t4_toe_keepalive_count != 0) { v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2; t4_set_reg_field(sc, A_TP_SHIFT_CNT, V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) | V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2), V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v)); } if (t4_toe_rexmt_min != 0) { v = us_to_tcp_ticks(sc, t4_toe_rexmt_min); v &= M_RXTMIN; t4_set_reg_field(sc, A_TP_RXT_MIN, V_RXTMIN(M_RXTMIN), V_RXTMIN(v)); } if (t4_toe_rexmt_max != 0) { v = us_to_tcp_ticks(sc, t4_toe_rexmt_max); v &= M_RXTMAX; t4_set_reg_field(sc, A_TP_RXT_MAX, V_RXTMAX(M_RXTMAX), V_RXTMAX(v)); } if (t4_toe_rexmt_count != 0) { v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2; t4_set_reg_field(sc, A_TP_SHIFT_CNT, V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) | V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2), V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v)); } for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) { if (t4_toe_rexmt_backoff[i] != -1) { v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0; shift = (i & 3) << 3; t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3), M_TIMERBACKOFFINDEX0 << shift, v << shift); } } #endif /* * Limit TOE connections to 2 reassembly "islands". This is * required to permit migrating TOE connections to either * ULP_MODE_TCPDDP or UPL_MODE_TLS. */ t4_tp_wr_bits_indirect(sc, A_TP_FRAG_CONFIG, V_PASSMODE(M_PASSMODE), V_PASSMODE(2)); #ifdef KERN_TLS if (is_ktls(sc)) { sc->tlst.inline_keys = t4_tls_inline_keys; sc->tlst.combo_wrs = t4_tls_combo_wrs; if (t4_kern_tls != 0 && is_t6(sc)) t6_config_kern_tls(sc, true); } #endif return (0); } #undef FW_PARAM_PFVF #undef FW_PARAM_DEV static void t4_set_desc(struct adapter *sc) { struct adapter_params *p = &sc->params; device_set_descf(sc->dev, "Chelsio %s", p->vpd.id); } static inline void ifmedia_add4(struct ifmedia *ifm, int m) { ifmedia_add(ifm, m, 0, NULL); ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL); ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL); ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL); } /* * This is the selected media, which is not quite the same as the active media. * The media line in ifconfig is "media: Ethernet selected (active)" if selected * and active are not the same, and "media: Ethernet selected" otherwise. */ static void set_current_media(struct port_info *pi) { struct link_config *lc; struct ifmedia *ifm; int mword; u_int speed; PORT_LOCK_ASSERT_OWNED(pi); /* Leave current media alone if it's already set to IFM_NONE. */ ifm = &pi->media; if (ifm->ifm_cur != NULL && IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE) return; lc = &pi->link_cfg; if (lc->requested_aneg != AUTONEG_DISABLE && lc->pcaps & FW_PORT_CAP32_ANEG) { ifmedia_set(ifm, IFM_ETHER | IFM_AUTO); return; } mword = IFM_ETHER | IFM_FDX; if (lc->requested_fc & PAUSE_TX) mword |= IFM_ETH_TXPAUSE; if (lc->requested_fc & PAUSE_RX) mword |= IFM_ETH_RXPAUSE; if (lc->requested_speed == 0) speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */ else speed = lc->requested_speed; mword |= port_mword(pi, speed_to_fwcap(speed)); ifmedia_set(ifm, mword); } /* * Returns true if the ifmedia list for the port cannot change. */ static bool fixed_ifmedia(struct port_info *pi) { return (pi->port_type == FW_PORT_TYPE_BT_SGMII || pi->port_type == FW_PORT_TYPE_BT_XFI || pi->port_type == FW_PORT_TYPE_BT_XAUI || pi->port_type == FW_PORT_TYPE_KX4 || pi->port_type == FW_PORT_TYPE_KX || pi->port_type == FW_PORT_TYPE_KR || pi->port_type == FW_PORT_TYPE_BP_AP || pi->port_type == FW_PORT_TYPE_BP4_AP || pi->port_type == FW_PORT_TYPE_BP40_BA || pi->port_type == FW_PORT_TYPE_KR4_100G || pi->port_type == FW_PORT_TYPE_KR_SFP28 || pi->port_type == FW_PORT_TYPE_KR_XLAUI); } static void build_medialist(struct port_info *pi) { uint32_t ss, speed; int unknown, mword, bit; struct link_config *lc; struct ifmedia *ifm; PORT_LOCK_ASSERT_OWNED(pi); if (pi->flags & FIXED_IFMEDIA) return; /* * Rebuild the ifmedia list. */ ifm = &pi->media; ifmedia_removeall(ifm); lc = &pi->link_cfg; ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */ if (__predict_false(ss == 0)) { /* not supposed to happen. */ MPASS(ss != 0); no_media: MPASS(LIST_EMPTY(&ifm->ifm_list)); ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL); ifmedia_set(ifm, IFM_ETHER | IFM_NONE); return; } unknown = 0; for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) { speed = 1 << bit; MPASS(speed & M_FW_PORT_CAP32_SPEED); if (ss & speed) { mword = port_mword(pi, speed); if (mword == IFM_NONE) { goto no_media; } else if (mword == IFM_UNKNOWN) unknown++; else ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword); } } if (unknown > 0) /* Add one unknown for all unknown media types. */ ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN); if (lc->pcaps & FW_PORT_CAP32_ANEG) ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL); set_current_media(pi); } /* * Initialize the requested fields in the link config based on driver tunables. */ static void init_link_config(struct port_info *pi) { struct link_config *lc = &pi->link_cfg; PORT_LOCK_ASSERT_OWNED(pi); lc->requested_caps = 0; lc->requested_speed = 0; if (t4_autoneg == 0) lc->requested_aneg = AUTONEG_DISABLE; else if (t4_autoneg == 1) lc->requested_aneg = AUTONEG_ENABLE; else lc->requested_aneg = AUTONEG_AUTO; lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG); if (t4_fec & FEC_AUTO) lc->requested_fec = FEC_AUTO; else if (t4_fec == 0) lc->requested_fec = FEC_NONE; else { /* -1 is handled by the FEC_AUTO block above and not here. */ lc->requested_fec = t4_fec & (FEC_RS | FEC_BASER_RS | FEC_NONE | FEC_MODULE); if (lc->requested_fec == 0) lc->requested_fec = FEC_AUTO; } if (t4_force_fec < 0) lc->force_fec = -1; else if (t4_force_fec > 0) lc->force_fec = 1; else lc->force_fec = 0; } /* * Makes sure that all requested settings comply with what's supported by the * port. Returns the number of settings that were invalid and had to be fixed. */ static int fixup_link_config(struct port_info *pi) { int n = 0; struct link_config *lc = &pi->link_cfg; uint32_t fwspeed; PORT_LOCK_ASSERT_OWNED(pi); /* Speed (when not autonegotiating) */ if (lc->requested_speed != 0) { fwspeed = speed_to_fwcap(lc->requested_speed); if ((fwspeed & lc->pcaps) == 0) { n++; lc->requested_speed = 0; } } /* Link autonegotiation */ MPASS(lc->requested_aneg == AUTONEG_ENABLE || lc->requested_aneg == AUTONEG_DISABLE || lc->requested_aneg == AUTONEG_AUTO); if (lc->requested_aneg == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) { n++; lc->requested_aneg = AUTONEG_AUTO; } /* Flow control */ MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0); if (lc->requested_fc & PAUSE_TX && !(lc->pcaps & FW_PORT_CAP32_FC_TX)) { n++; lc->requested_fc &= ~PAUSE_TX; } if (lc->requested_fc & PAUSE_RX && !(lc->pcaps & FW_PORT_CAP32_FC_RX)) { n++; lc->requested_fc &= ~PAUSE_RX; } if (!(lc->requested_fc & PAUSE_AUTONEG) && !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) { n++; lc->requested_fc |= PAUSE_AUTONEG; } /* FEC */ if ((lc->requested_fec & FEC_RS && !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) || (lc->requested_fec & FEC_BASER_RS && !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) { n++; lc->requested_fec = FEC_AUTO; } return (n); } /* * Apply the requested L1 settings, which are expected to be valid, to the * hardware. */ static int apply_link_config(struct port_info *pi) { struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; #ifdef INVARIANTS ASSERT_SYNCHRONIZED_OP(sc); PORT_LOCK_ASSERT_OWNED(pi); if (lc->requested_aneg == AUTONEG_ENABLE) MPASS(lc->pcaps & FW_PORT_CAP32_ANEG); if (!(lc->requested_fc & PAUSE_AUTONEG)) MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE); if (lc->requested_fc & PAUSE_TX) MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX); if (lc->requested_fc & PAUSE_RX) MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX); if (lc->requested_fec & FEC_RS) MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS); if (lc->requested_fec & FEC_BASER_RS) MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS); #endif if (!(sc->flags & IS_VF)) { rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); if (rc != 0) { device_printf(pi->dev, "l1cfg failed: %d\n", rc); return (rc); } } /* * An L1_CFG will almost always result in a link-change event if the * link is up, and the driver will refresh the actual fec/fc/etc. when * the notification is processed. If the link is down then the actual * settings are meaningless. * * This takes care of the case where a change in the L1 settings may not * result in a notification. */ if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG)) lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX); return (0); } #define FW_MAC_EXACT_CHUNK 7 struct mcaddr_ctx { if_t ifp; const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; uint64_t hash; int i; int del; int rc; }; static u_int add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) { struct mcaddr_ctx *ctx = arg; struct vi_info *vi = if_getsoftc(ctx->ifp); struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; if (ctx->rc < 0) return (0); ctx->mcaddr[ctx->i] = LLADDR(sdl); MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i])); ctx->i++; if (ctx->i == FW_MAC_EXACT_CHUNK) { ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del, ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0); if (ctx->rc < 0) { int j; for (j = 0; j < ctx->i; j++) { if_printf(ctx->ifp, "failed to add mc address" " %02x:%02x:%02x:" "%02x:%02x:%02x rc=%d\n", ctx->mcaddr[j][0], ctx->mcaddr[j][1], ctx->mcaddr[j][2], ctx->mcaddr[j][3], ctx->mcaddr[j][4], ctx->mcaddr[j][5], -ctx->rc); } return (0); } ctx->del = 0; ctx->i = 0; } return (1); } /* * Program the port's XGMAC based on parameters in ifnet. The caller also * indicates which parameters should be programmed (the rest are left alone). */ int update_mac_settings(if_t ifp, int flags) { int rc = 0; struct vi_info *vi = if_getsoftc(ifp); struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; uint8_t match_all_mac[ETHER_ADDR_LEN] = {0}; ASSERT_SYNCHRONIZED_OP(sc); KASSERT(flags, ("%s: not told what to update.", __func__)); if (flags & XGMAC_MTU) mtu = if_getmtu(ifp); if (flags & XGMAC_PROMISC) promisc = if_getflags(ifp) & IFF_PROMISC ? 1 : 0; if (flags & XGMAC_ALLMULTI) allmulti = if_getflags(ifp) & IFF_ALLMULTI ? 1 : 0; if (flags & XGMAC_VLANEX) vlanex = if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING ? 1 : 0; if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, allmulti, 1, vlanex, false); if (rc) { if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); return (rc); } } if (flags & XGMAC_UCADDR) { uint8_t ucaddr[ETHER_ADDR_LEN]; bcopy(if_getlladdr(ifp), ucaddr, sizeof(ucaddr)); rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, ucaddr, true, &vi->smt_idx); if (rc < 0) { rc = -rc; if_printf(ifp, "change_mac failed: %d\n", rc); return (rc); } else { vi->xact_addr_filt = rc; rc = 0; } } if (flags & XGMAC_MCADDRS) { struct epoch_tracker et; struct mcaddr_ctx ctx; int j; ctx.ifp = ifp; ctx.hash = 0; ctx.i = 0; ctx.del = 1; ctx.rc = 0; /* * Unlike other drivers, we accumulate list of pointers into * interface address lists and we need to keep it safe even * after if_foreach_llmaddr() returns, thus we must enter the * network epoch. */ NET_EPOCH_ENTER(et); if_foreach_llmaddr(ifp, add_maddr, &ctx); if (ctx.rc < 0) { NET_EPOCH_EXIT(et); rc = -ctx.rc; return (rc); } if (ctx.i > 0) { rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx.del, ctx.i, ctx.mcaddr, NULL, &ctx.hash, 0); NET_EPOCH_EXIT(et); if (rc < 0) { rc = -rc; for (j = 0; j < ctx.i; j++) { if_printf(ifp, "failed to add mcast address" " %02x:%02x:%02x:" "%02x:%02x:%02x rc=%d\n", ctx.mcaddr[j][0], ctx.mcaddr[j][1], ctx.mcaddr[j][2], ctx.mcaddr[j][3], ctx.mcaddr[j][4], ctx.mcaddr[j][5], rc); } return (rc); } ctx.del = 0; } else NET_EPOCH_EXIT(et); rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0); if (rc != 0) if_printf(ifp, "failed to set mcast address hash: %d\n", rc); if (ctx.del == 0) { /* We clobbered the VXLAN entry if there was one. */ pi->vxlan_tcam_entry = false; } } if (IS_MAIN_VI(vi) && sc->vxlan_refcount > 0 && pi->vxlan_tcam_entry == false) { rc = t4_alloc_raw_mac_filt(sc, vi->viid, match_all_mac, match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id, true); if (rc < 0) { rc = -rc; if_printf(ifp, "failed to add VXLAN TCAM entry: %d.\n", rc); } else { MPASS(rc == sc->rawf_base + pi->port_id); rc = 0; pi->vxlan_tcam_entry = true; } } return (rc); } /* * {begin|end}_synchronized_op must be called from the same thread. */ int begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, char *wmesg) { int rc, pri; #ifdef WITNESS /* the caller thinks it's ok to sleep, but is it really? */ if (flags & SLEEP_OK) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "begin_synchronized_op"); #endif if (INTR_OK) pri = PCATCH; else pri = 0; ADAPTER_LOCK(sc); for (;;) { if (vi && IS_DETACHING(vi)) { rc = ENXIO; goto done; } if (!IS_BUSY(sc)) { rc = 0; break; } if (!(flags & SLEEP_OK)) { rc = EBUSY; goto done; } if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { rc = EINTR; goto done; } } KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); SET_BUSY(sc); #ifdef INVARIANTS sc->last_op = wmesg; sc->last_op_thr = curthread; sc->last_op_flags = flags; #endif done: if (!(flags & HOLD_LOCK) || rc) ADAPTER_UNLOCK(sc); return (rc); } /* * Tell if_ioctl and if_init that the VI is going away. This is * special variant of begin_synchronized_op and must be paired with a * call to end_vi_detach. */ void begin_vi_detach(struct adapter *sc, struct vi_info *vi) { ADAPTER_LOCK(sc); SET_DETACHING(vi); wakeup(&sc->flags); while (IS_BUSY(sc)) mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); SET_BUSY(sc); #ifdef INVARIANTS sc->last_op = "t4detach"; sc->last_op_thr = curthread; sc->last_op_flags = 0; #endif ADAPTER_UNLOCK(sc); } void end_vi_detach(struct adapter *sc, struct vi_info *vi) { ADAPTER_LOCK(sc); KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); CLR_BUSY(sc); CLR_DETACHING(vi); wakeup(&sc->flags); ADAPTER_UNLOCK(sc); } /* * {begin|end}_synchronized_op must be called from the same thread. */ void end_synchronized_op(struct adapter *sc, int flags) { if (flags & LOCK_HELD) ADAPTER_LOCK_ASSERT_OWNED(sc); else ADAPTER_LOCK(sc); KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); CLR_BUSY(sc); wakeup(&sc->flags); ADAPTER_UNLOCK(sc); } static int cxgbe_init_synchronized(struct vi_info *vi) { struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; if_t ifp = vi->ifp; int rc = 0, i; struct sge_txq *txq; ASSERT_SYNCHRONIZED_OP(sc); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) return (0); /* already running */ if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0)) return (rc); /* error message displayed already */ if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0)) return (rc); /* error message displayed already */ rc = update_mac_settings(ifp, XGMAC_ALL); if (rc) goto done; /* error message displayed already */ PORT_LOCK(pi); if (pi->up_vis == 0) { t4_update_port_info(pi); fixup_link_config(pi); build_medialist(pi); apply_link_config(pi); } rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); if (rc != 0) { if_printf(ifp, "enable_vi failed: %d\n", rc); PORT_UNLOCK(pi); goto done; } /* * Can't fail from this point onwards. Review cxgbe_uninit_synchronized * if this changes. */ for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags |= EQ_ENABLED; TXQ_UNLOCK(txq); } /* * The first iq of the first port to come up is used for tracing. */ if (sc->traceq < 0 && IS_MAIN_VI(vi)) { sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | V_QUEUENUMBER(sc->traceq)); pi->flags |= HAS_TRACEQ; } /* all ok */ pi->up_vis++; if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); if (pi->link_cfg.link_ok) t4_os_link_changed(pi); PORT_UNLOCK(pi); mtx_lock(&vi->tick_mtx); if (vi->pi->nvi > 1 || sc->flags & IS_VF) callout_reset(&vi->tick, hz, vi_tick, vi); else callout_reset(&vi->tick, hz, cxgbe_tick, vi); mtx_unlock(&vi->tick_mtx); done: if (rc != 0) cxgbe_uninit_synchronized(vi); return (rc); } /* * Idempotent. */ static int cxgbe_uninit_synchronized(struct vi_info *vi) { struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; if_t ifp = vi->ifp; int rc, i; struct sge_txq *txq; ASSERT_SYNCHRONIZED_OP(sc); if (!(vi->flags & VI_INIT_DONE)) { if (__predict_false(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { KASSERT(0, ("uninited VI is running")); if_printf(ifp, "uninited VI with running ifnet. " "vi->flags 0x%016lx, if_flags 0x%08x, " "if_drv_flags 0x%08x\n", vi->flags, if_getflags(ifp), if_getdrvflags(ifp)); } return (0); } /* * Disable the VI so that all its data in either direction is discarded * by the MPS. Leave everything else (the queues, interrupts, and 1Hz * tick) intact as the TP can deliver negative advice or data that it's * holding in its RAM (for an offloaded connection) even after the VI is * disabled. */ rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); if (rc) { if_printf(ifp, "disable_vi failed: %d\n", rc); return (rc); } for_each_txq(vi, i, txq) { TXQ_LOCK(txq); txq->eq.flags &= ~EQ_ENABLED; TXQ_UNLOCK(txq); } mtx_lock(&vi->tick_mtx); callout_stop(&vi->tick); mtx_unlock(&vi->tick_mtx); PORT_LOCK(pi); if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { PORT_UNLOCK(pi); return (0); } if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); pi->up_vis--; if (pi->up_vis > 0) { PORT_UNLOCK(pi); return (0); } pi->link_cfg.link_ok = false; pi->link_cfg.speed = 0; pi->link_cfg.link_down_rc = 255; t4_os_link_changed(pi); PORT_UNLOCK(pi); return (0); } /* * It is ok for this function to fail midway and return right away. t4_detach * will walk the entire sc->irq list and clean up whatever is valid. */ int t4_setup_intr_handlers(struct adapter *sc) { int rc, rid, p, q, v; char s[8]; struct irq *irq; struct port_info *pi; struct vi_info *vi; struct sge *sge = &sc->sge; struct sge_rxq *rxq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif #ifdef DEV_NETMAP struct sge_nm_rxq *nm_rxq; #endif #ifdef RSS int nbuckets = rss_getnumbuckets(); #endif /* * Setup interrupts. */ irq = &sc->irq[0]; rid = sc->intr_type == INTR_INTX ? 0 : 1; if (forwarding_intr_to_fwq(sc)) return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); /* Multiple interrupts. */ if (sc->flags & IS_VF) KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports, ("%s: too few intr.", __func__)); else KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, ("%s: too few intr.", __func__)); /* The first one is always error intr on PFs */ if (!(sc->flags & IS_VF)) { rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); if (rc != 0) return (rc); irq++; rid++; } /* The second one is always the firmware event queue (first on VFs) */ rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); if (rc != 0) return (rc); irq++; rid++; for_each_port(sc, p) { pi = sc->port[p]; for_each_vi(pi, v, vi) { vi->first_intr = rid - 1; if (vi->nnmrxq > 0) { int n = max(vi->nrxq, vi->nnmrxq); rxq = &sge->rxq[vi->first_rxq]; #ifdef DEV_NETMAP nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; #endif for (q = 0; q < n; q++) { snprintf(s, sizeof(s), "%x%c%x", p, 'a' + v, q); if (q < vi->nrxq) irq->rxq = rxq++; #ifdef DEV_NETMAP if (q < vi->nnmrxq) irq->nm_rxq = nm_rxq++; if (irq->nm_rxq != NULL && irq->rxq == NULL) { /* Netmap rx only */ rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, irq->nm_rxq, s); } if (irq->nm_rxq != NULL && irq->rxq != NULL) { /* NIC and Netmap rx */ rc = t4_alloc_irq(sc, irq, rid, t4_vi_intr, irq, s); } #endif if (irq->rxq != NULL && irq->nm_rxq == NULL) { /* NIC rx only */ rc = t4_alloc_irq(sc, irq, rid, t4_intr, irq->rxq, s); } if (rc != 0) return (rc); #ifdef RSS if (q < vi->nrxq) { bus_bind_intr(sc->dev, irq->res, rss_getcpu(q % nbuckets)); } #endif irq++; rid++; vi->nintr++; } } else { for_each_rxq(vi, q, rxq) { snprintf(s, sizeof(s), "%x%c%x", p, 'a' + v, q); rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, s); if (rc != 0) return (rc); #ifdef RSS bus_bind_intr(sc->dev, irq->res, rss_getcpu(q % nbuckets)); #endif irq++; rid++; vi->nintr++; } } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, q, ofld_rxq) { snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q); rc = t4_alloc_irq(sc, irq, rid, t4_intr, ofld_rxq, s); if (rc != 0) return (rc); irq++; rid++; vi->nintr++; } #endif } } MPASS(irq == &sc->irq[sc->intr_count]); return (0); } static void write_global_rss_key(struct adapter *sc) { #ifdef RSS int i; uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; CTASSERT(RSS_KEYSIZE == 40); rss_getkey((void *)&raw_rss_key[0]); for (i = 0; i < nitems(rss_key); i++) { rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); } t4_write_rss_key(sc, &rss_key[0], -1, 1); #endif } /* * Idempotent. */ static int adapter_full_init(struct adapter *sc) { int rc, i; ASSERT_SYNCHRONIZED_OP(sc); /* * queues that belong to the adapter (not any particular port). */ rc = t4_setup_adapter_queues(sc); if (rc != 0) return (rc); MPASS(sc->params.nports <= nitems(sc->tq)); for (i = 0; i < sc->params.nports; i++) { if (sc->tq[i] != NULL) continue; sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, taskqueue_thread_enqueue, &sc->tq[i]); if (sc->tq[i] == NULL) { CH_ERR(sc, "failed to allocate task queue %d\n", i); return (ENOMEM); } taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", device_get_nameunit(sc->dev), i); } if (!(sc->flags & IS_VF)) { write_global_rss_key(sc); t4_intr_enable(sc); } return (0); } int adapter_init(struct adapter *sc) { int rc; ASSERT_SYNCHRONIZED_OP(sc); ADAPTER_LOCK_ASSERT_NOTOWNED(sc); KASSERT((sc->flags & FULL_INIT_DONE) == 0, ("%s: FULL_INIT_DONE already", __func__)); rc = adapter_full_init(sc); if (rc != 0) adapter_full_uninit(sc); else sc->flags |= FULL_INIT_DONE; return (rc); } /* * Idempotent. */ static void adapter_full_uninit(struct adapter *sc) { int i; t4_teardown_adapter_queues(sc); for (i = 0; i < nitems(sc->tq); i++) { if (sc->tq[i] == NULL) continue; taskqueue_free(sc->tq[i]); sc->tq[i] = NULL; } sc->flags &= ~FULL_INIT_DONE; } #ifdef RSS #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ RSS_HASHTYPE_RSS_UDP_IPV6) /* Translates kernel hash types to hardware. */ static int hashconfig_to_hashen(int hashconfig) { int hashen = 0; if (hashconfig & RSS_HASHTYPE_RSS_IPV4) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_IPV6) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; } if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; } if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; return (hashen); } /* Translates hardware hash types to kernel. */ static int hashen_to_hashconfig(int hashen) { int hashconfig = 0; if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { /* * If UDP hashing was enabled it must have been enabled for * either IPv4 or IPv6 (inclusive or). Enabling UDP without * enabling any 4-tuple hash is nonsense configuration. */ MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; } if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) hashconfig |= RSS_HASHTYPE_RSS_IPV4; if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) hashconfig |= RSS_HASHTYPE_RSS_IPV6; return (hashconfig); } #endif /* * Idempotent. */ static int vi_full_init(struct vi_info *vi) { struct adapter *sc = vi->adapter; struct sge_rxq *rxq; int rc, i, j; #ifdef RSS int nbuckets = rss_getnumbuckets(); int hashconfig = rss_gethashconfig(); int extra; #endif ASSERT_SYNCHRONIZED_OP(sc); /* * Allocate tx/rx/fl queues for this VI. */ rc = t4_setup_vi_queues(vi); if (rc != 0) return (rc); /* * Setup RSS for this VI. Save a copy of the RSS table for later use. */ if (vi->nrxq > vi->rss_size) { CH_ALERT(vi, "nrxq (%d) > hw RSS table size (%d); " "some queues will never receive traffic.\n", vi->nrxq, vi->rss_size); } else if (vi->rss_size % vi->nrxq) { CH_ALERT(vi, "nrxq (%d), hw RSS table size (%d); " "expect uneven traffic distribution.\n", vi->nrxq, vi->rss_size); } #ifdef RSS if (vi->nrxq != nbuckets) { CH_ALERT(vi, "nrxq (%d) != kernel RSS buckets (%d);" "performance will be impacted.\n", vi->nrxq, nbuckets); } #endif if (vi->rss == NULL) vi->rss = malloc(vi->rss_size * sizeof (*vi->rss), M_CXGBE, M_ZERO | M_WAITOK); for (i = 0; i < vi->rss_size;) { #ifdef RSS j = rss_get_indirection_to_bucket(i); j %= vi->nrxq; rxq = &sc->sge.rxq[vi->first_rxq + j]; vi->rss[i++] = rxq->iq.abs_id; #else for_each_rxq(vi, j, rxq) { vi->rss[i++] = rxq->iq.abs_id; if (i == vi->rss_size) break; } #endif } rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, vi->rss, vi->rss_size); if (rc != 0) { CH_ERR(vi, "rss_config failed: %d\n", rc); return (rc); } #ifdef RSS vi->hashen = hashconfig_to_hashen(hashconfig); /* * We may have had to enable some hashes even though the global config * wants them disabled. This is a potential problem that must be * reported to the user. */ extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig; /* * If we consider only the supported hash types, then the enabled hashes * are a superset of the requested hashes. In other words, there cannot * be any supported hash that was requested but not enabled, but there * can be hashes that were not requested but had to be enabled. */ extra &= SUPPORTED_RSS_HASHTYPES; MPASS((extra & hashconfig) == 0); if (extra) { CH_ALERT(vi, "global RSS config (0x%x) cannot be accommodated.\n", hashconfig); } if (extra & RSS_HASHTYPE_RSS_IPV4) CH_ALERT(vi, "IPv4 2-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) CH_ALERT(vi, "TCP/IPv4 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_IPV6) CH_ALERT(vi, "IPv6 2-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) CH_ALERT(vi, "TCP/IPv6 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n"); if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n"); #else vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; #endif rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0], 0, 0); if (rc != 0) { CH_ERR(vi, "rss hash/defaultq config failed: %d\n", rc); return (rc); } return (0); } int vi_init(struct vi_info *vi) { int rc; ASSERT_SYNCHRONIZED_OP(vi->adapter); KASSERT((vi->flags & VI_INIT_DONE) == 0, ("%s: VI_INIT_DONE already", __func__)); rc = vi_full_init(vi); if (rc != 0) vi_full_uninit(vi); else vi->flags |= VI_INIT_DONE; return (rc); } /* * Idempotent. */ static void vi_full_uninit(struct vi_info *vi) { if (vi->flags & VI_INIT_DONE) { quiesce_vi(vi); free(vi->rss, M_CXGBE); free(vi->nm_rss, M_CXGBE); } t4_teardown_vi_queues(vi); vi->flags &= ~VI_INIT_DONE; } static void quiesce_txq(struct sge_txq *txq) { struct sge_eq *eq = &txq->eq; struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; MPASS(eq->flags & EQ_SW_ALLOCATED); MPASS(!(eq->flags & EQ_ENABLED)); /* Wait for the mp_ring to empty. */ while (!mp_ring_is_idle(txq->r)) { mp_ring_check_drainage(txq->r, 4096); pause("rquiesce", 1); } MPASS(txq->txp.npkt == 0); if (eq->flags & EQ_HW_ALLOCATED) { /* * Hardware is alive and working normally. Wait for it to * finish and then wait for the driver to catch up and reclaim * all descriptors. */ while (spg->cidx != htobe16(eq->pidx)) pause("equiesce", 1); while (eq->cidx != eq->pidx) pause("dquiesce", 1); } else { /* * Hardware is unavailable. Discard all pending tx and reclaim * descriptors directly. */ TXQ_LOCK(txq); while (eq->cidx != eq->pidx) { struct mbuf *m, *nextpkt; struct tx_sdesc *txsd; txsd = &txq->sdesc[eq->cidx]; for (m = txsd->m; m != NULL; m = nextpkt) { nextpkt = m->m_nextpkt; m->m_nextpkt = NULL; m_freem(m); } IDXINCR(eq->cidx, txsd->desc_used, eq->sidx); } spg->pidx = spg->cidx = htobe16(eq->cidx); TXQ_UNLOCK(txq); } } static void quiesce_wrq(struct sge_wrq *wrq) { struct wrqe *wr; TXQ_LOCK(wrq); while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) { STAILQ_REMOVE_HEAD(&wrq->wr_list, link); #ifdef INVARIANTS wrq->nwr_pending--; wrq->ndesc_needed -= howmany(wr->wr_len, EQ_ESIZE); #endif free(wr, M_CXGBE); } MPASS(wrq->nwr_pending == 0); MPASS(wrq->ndesc_needed == 0); wrq->nwr_pending = 0; wrq->ndesc_needed = 0; TXQ_UNLOCK(wrq); } static void quiesce_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) { /* Synchronize with the interrupt handler */ while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) pause("iqfree", 1); if (fl != NULL) { MPASS(iq->flags & IQ_HAS_FL); mtx_lock(&sc->sfl_lock); FL_LOCK(fl); fl->flags |= FL_DOOMED; FL_UNLOCK(fl); callout_stop(&sc->sfl_callout); mtx_unlock(&sc->sfl_lock); KASSERT((fl->flags & FL_STARVING) == 0, ("%s: still starving", __func__)); /* Release all buffers if hardware is no longer available. */ if (!(iq->flags & IQ_HW_ALLOCATED)) free_fl_buffers(sc, fl); } } /* * Wait for all activity on all the queues of the VI to complete. It is assumed * that no new work is being enqueued by the hardware or the driver. That part * should be arranged before calling this function. */ static void quiesce_vi(struct vi_info *vi) { int i; struct adapter *sc = vi->adapter; struct sge_rxq *rxq; struct sge_txq *txq; #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif #if defined(TCP_OFFLOAD) || defined(RATELIMIT) struct sge_ofld_txq *ofld_txq; #endif if (!(vi->flags & VI_INIT_DONE)) return; for_each_txq(vi, i, txq) { quiesce_txq(txq); } #if defined(TCP_OFFLOAD) || defined(RATELIMIT) for_each_ofld_txq(vi, i, ofld_txq) { quiesce_wrq(&ofld_txq->wrq); } #endif for_each_rxq(vi, i, rxq) { quiesce_iq_fl(sc, &rxq->iq, &rxq->fl); } #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { quiesce_iq_fl(sc, &ofld_rxq->iq, &ofld_rxq->fl); } #endif } static int t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, driver_intr_t *handler, void *arg, char *name) { int rc; irq->rid = rid; irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, RF_SHAREABLE | RF_ACTIVE); if (irq->res == NULL) { device_printf(sc->dev, "failed to allocate IRQ for rid %d, name %s.\n", rid, name); return (ENOMEM); } rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, NULL, handler, arg, &irq->tag); if (rc != 0) { device_printf(sc->dev, "failed to setup interrupt for rid %d, name %s: %d\n", rid, name, rc); } else if (name) bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name); return (rc); } static int t4_free_irq(struct adapter *sc, struct irq *irq) { if (irq->tag) bus_teardown_intr(sc->dev, irq->res, irq->tag); if (irq->res) bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); bzero(irq, sizeof(*irq)); return (0); } static void get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) { regs->version = chip_id(sc) | chip_rev(sc) << 10; t4_get_regs(sc, buf, regs->len); } #define A_PL_INDIR_CMD 0x1f8 #define S_PL_AUTOINC 31 #define M_PL_AUTOINC 0x1U #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) #define S_PL_VFID 20 #define M_PL_VFID 0xffU #define V_PL_VFID(x) ((x) << S_PL_VFID) #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) #define S_PL_ADDR 0 #define M_PL_ADDR 0xfffffU #define V_PL_ADDR(x) ((x) << S_PL_ADDR) #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) #define A_PL_INDIR_DATA 0x1fc static uint64_t read_vf_stat(struct adapter *sc, u_int vin, int reg) { u32 stats[2]; if (sc->flags & IS_VF) { stats[0] = t4_read_reg(sc, VF_MPS_REG(reg)); stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4)); } else { mtx_assert(&sc->reg_lock, MA_OWNED); t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(vin) | V_PL_ADDR(VF_MPS_REG(reg))); stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); } return (((uint64_t)stats[1]) << 32 | stats[0]); } static void t4_get_vi_stats(struct adapter *sc, u_int vin, struct fw_vi_stats_vf *stats) { #define GET_STAT(name) \ read_vf_stat(sc, vin, A_MPS_VF_STAT_##name##_L) if (!(sc->flags & IS_VF)) mtx_lock(&sc->reg_lock); stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); if (!(sc->flags & IS_VF)) mtx_unlock(&sc->reg_lock); #undef GET_STAT } static void t4_clr_vi_stats(struct adapter *sc, u_int vin) { int reg; t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(vin) | V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) t4_write_reg(sc, A_PL_INDIR_DATA, 0); } static void vi_refresh_stats(struct vi_info *vi) { struct timeval tv; const struct timeval interval = {0, 250000}; /* 250ms */ mtx_assert(&vi->tick_mtx, MA_OWNED); if (vi->flags & VI_SKIP_STATS) return; getmicrotime(&tv); timevalsub(&tv, &interval); if (timevalcmp(&tv, &vi->last_refreshed, <)) return; t4_get_vi_stats(vi->adapter, vi->vin, &vi->stats); getmicrotime(&vi->last_refreshed); } static void cxgbe_refresh_stats(struct vi_info *vi) { u_int i, v, tnl_cong_drops, chan_map; struct timeval tv; const struct timeval interval = {0, 250000}; /* 250ms */ struct port_info *pi; struct adapter *sc; mtx_assert(&vi->tick_mtx, MA_OWNED); if (vi->flags & VI_SKIP_STATS) return; getmicrotime(&tv); timevalsub(&tv, &interval); if (timevalcmp(&tv, &vi->last_refreshed, <)) return; pi = vi->pi; sc = vi->adapter; tnl_cong_drops = 0; t4_get_port_stats(sc, pi->port_id, &pi->stats); chan_map = pi->rx_e_chan_map; while (chan_map) { i = ffs(chan_map) - 1; mtx_lock(&sc->reg_lock); t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1, A_TP_MIB_TNL_CNG_DROP_0 + i); mtx_unlock(&sc->reg_lock); tnl_cong_drops += v; chan_map &= ~(1 << i); } pi->tnl_cong_drops = tnl_cong_drops; getmicrotime(&vi->last_refreshed); } static void cxgbe_tick(void *arg) { struct vi_info *vi = arg; MPASS(IS_MAIN_VI(vi)); mtx_assert(&vi->tick_mtx, MA_OWNED); cxgbe_refresh_stats(vi); callout_schedule(&vi->tick, hz); } static void vi_tick(void *arg) { struct vi_info *vi = arg; mtx_assert(&vi->tick_mtx, MA_OWNED); vi_refresh_stats(vi); callout_schedule(&vi->tick, hz); } /* * Should match fw_caps_config_ enums in t4fw_interface.h */ static char *caps_decoder[] = { "\20\001IPMI\002NCSI", /* 0: NBM */ "\20\001PPP\002QFC\003DCBX", /* 1: link */ "\20\001INGRESS\002EGRESS", /* 2: switch */ "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */ "\006HASHFILTER\007ETHOFLD", "\20\001TOE", /* 4: TOE */ "\20\001RDDP\002RDMAC", /* 5: RDMA */ "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */ "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD" "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD" "\007T10DIF" "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD", "\20\001LOOKASIDE\002TLSKEYS\003IPSEC_INLINE" /* 7: Crypto */ "\004TLS_HW", "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */ "\004PO_INITIATOR\005PO_TARGET", }; void t4_sysctls(struct adapter *sc) { struct sysctl_ctx_list *ctx = &sc->ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children, *c0; static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; /* * dev.t4nex.X. */ oid = device_get_sysctl_tree(sc->dev); c0 = children = SYSCTL_CHILDREN(oid); sc->sc_do_rxcopy = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, sc->params.nports, "# of ports"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, doorbells, (uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A", "available doorbells"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, sc->params.vpd.cclk, "core clock frequency (in KHz)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val), sysctl_int_array, "A", "interrupt holdoff timer values (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val), sysctl_int_array, "A", "interrupt holdoff packet counter values"); t4_sge_sysctls(sc, ctx, children); sc->lro_timeout = 100; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW, &sc->debug_flags, 0, "flags to enable runtime debugging"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version", CTLFLAG_RD, sc->tp_version, 0, "TP microcode version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", CTLFLAG_RD, sc->fw_version, 0, "firmware version"); if (sc->flags & IS_VF) return; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, NULL, chip_rev(sc), "chip hardware revision"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn", CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn", CTLFLAG_RD, sc->params.vpd.pn, 0, "part number"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec", CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version", CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na", CTLFLAG_RD, sc->params.vpd.na, 0, "network address"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD, sc->er_version, 0, "expansion ROM version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD, sc->bs_version, 0, "bootstrap firmware version"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD, NULL, sc->params.scfg_vers, "serial config version"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD, NULL, sc->params.vpd_vers, "VPD version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, sc->cfcsum, "config file checksum"); #define SYSCTL_CAP(name, n, text) \ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, caps_decoder[n], \ (uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \ "available " text " capabilities") SYSCTL_CAP(nbmcaps, 0, "NBM"); SYSCTL_CAP(linkcaps, 1, "link"); SYSCTL_CAP(switchcaps, 2, "switch"); SYSCTL_CAP(niccaps, 3, "NIC"); SYSCTL_CAP(toecaps, 4, "TCP offload"); SYSCTL_CAP(rdmacaps, 5, "RDMA"); SYSCTL_CAP(iscsicaps, 6, "iSCSI"); SYSCTL_CAP(cryptocaps, 7, "crypto"); SYSCTL_CAP(fcoecaps, 8, "FCoE"); #undef SYSCTL_CAP SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, NULL, sc->tids.nftids, "number of filters"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_temperature, "I", "chip temperature (in Celsius)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset_sensor", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, sysctl_reset_sensor, "I", "reset the chip's temperature sensor."); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_loadavg, "A", "microprocessor load averages (debug firmwares only)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "core_vdd", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_vdd, "I", "core Vdd (in mV)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, LOCAL_CPUS, sysctl_cpus, "A", "local CPUs"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, INTR_CPUS, sysctl_cpus, "A", "preferred CPUs for interrupts"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "swintr", CTLFLAG_RW, &sc->swintr, 0, "software triggered interrupts"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset", CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_reset, "I", "1 = reset adapter, 0 = zero reset counter"); /* * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. */ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, "logs and miscellaneous information"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cctrl, "A", "congestion control"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1, sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2, sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 3, sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 4, sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 5, sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cim_la, "A", "CIM logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 3 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 4 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 5 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); if (chip_id(sc) > CHELSIO_T4) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 6 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 7 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); } SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cim_qcfg, "A", "CIM queue configuration"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_cpl_stats, "A", "CPL statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_ddp_stats, "A", "non-TCP DDP statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tid_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tid_stats, "A", "tid stats"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_devlog, "A", "firmware's device log"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_fcoe_stats, "A", "FCoE statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_hw_sched, "A", "hardware scheduler "); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_l2t, "A", "hardware L2 table"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_smt, "A", "hardware source MAC table"); #ifdef INET6 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "clip", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_clip, "A", "active CLIP table entries"); #endif SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_lb_stats, "A", "loopback statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_meminfo, "A", "memory regions"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6, "A", "MPS TCAM entries"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_path_mtus, "A", "path MTUs"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_pm_stats, "A", "PM statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_rdma_stats, "A", "RDMA statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tcp_stats, "A", "TCP statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tids, "A", "TID information"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tp_err_stats, "A", "TP error statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tnl_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tnl_stats, "A", "TP tunnel statistics"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, sysctl_tp_la_mask, "I", "TP logic analyzer event capture mask"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tp_la, "A", "TP logic analyzer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tx_rate, "A", "Tx rate"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_ulprx_la, "A", "ULPRX logic analyzer"); if (chip_id(sc) >= CHELSIO_T5) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_wcwr_stats, "A", "write combined work requests"); } #ifdef KERN_TLS if (is_ktls(sc)) { /* * dev.t4nex.0.tls. */ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "tls", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "KERN_TLS parameters"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "inline_keys", CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS " "keys in work requests (1) or attempt to store TLS keys " "in card memory."); if (is_t6(sc)) SYSCTL_ADD_INT(ctx, children, OID_AUTO, "combo_wrs", CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to " "combine TCB field updates with TLS record work " "requests."); } #endif #ifdef TCP_OFFLOAD if (is_offload(sc)) { int i; char s[4]; /* * dev.t4nex.X.toe. */ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE parameters"); children = SYSCTL_CHILDREN(oid); sc->tt.cong_algorithm = -1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm", CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control " "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, " "3 = highspeed)"); sc->tt.sndbuf = -1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, &sc->tt.sndbuf, 0, "hardware send buffer"); sc->tt.ddp = 0; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW | CTLFLAG_SKIP, &sc->tt.ddp, 0, ""); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_zcopy", CTLFLAG_RW, &sc->tt.ddp, 0, "Enable zero-copy aio_read(2)"); sc->tt.rx_coalesce = -1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); sc->tt.tls = 1; SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, sysctl_tls, "I", "Inline TLS allowed"); sc->tt.tx_align = -1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); sc->tt.tx_zcopy = 0; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy", CTLFLAG_RW, &sc->tt.tx_zcopy, 0, "Enable zero-copy aio_write(2)"); sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cop_managed_offloading", CTLFLAG_RW, &sc->tt.cop_managed_offloading, 0, "COP (Connection Offload Policy) controls all TOE offload"); sc->tt.autorcvbuf_inc = 16 * 1024; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "autorcvbuf_inc", CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0, "autorcvbuf increment"); sc->tt.update_hc_on_pmtu_change = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "update_hc_on_pmtu_change", CTLFLAG_RW, &sc->tt.update_hc_on_pmtu_change, 0, "Update hostcache entry if the PMTU changes"); sc->tt.iso = 1; SYSCTL_ADD_INT(ctx, children, OID_AUTO, "iso", CTLFLAG_RW, &sc->tt.iso, 0, "Enable iSCSI segmentation offload"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tp_tick, "A", "TP timer tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1, sysctl_tp_tick, "A", "TCP timestamp tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2, sysctl_tp_tick, "A", "DACK tick (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_tp_dack_timer, "IU", "DACK timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_RXT_MIN, sysctl_tp_timer, "LU", "Minimum retransmit interval (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_RXT_MAX, sysctl_tp_timer, "LU", "Maximum retransmit interval (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_PERS_MIN, sysctl_tp_timer, "LU", "Persist timer min (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_PERS_MAX, sysctl_tp_timer, "LU", "Persist timer max (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_KEEP_IDLE, sysctl_tp_timer, "LU", "Keepalive idle timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_KEEP_INTVL, sysctl_tp_timer, "LU", "Keepalive interval timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_INIT_SRTT, sysctl_tp_timer, "LU", "Initial SRTT (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer", CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, A_TP_FINWAIT2_TIMER, sysctl_tp_timer, "LU", "FINWAIT2 timer (us)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, S_SYNSHIFTMAX, sysctl_tp_shift_cnt, "IU", "Number of SYN retransmissions before abort"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, S_RXTSHIFTMAXR2, sysctl_tp_shift_cnt, "IU", "Number of retransmissions before abort"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, S_KEEPALIVEMAXR2, sysctl_tp_shift_cnt, "IU", "Number of keepalive probes before abort"); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE retransmit backoffs"); children = SYSCTL_CHILDREN(oid); for (i = 0; i < 16; i++) { snprintf(s, sizeof(s), "%u", i); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s, CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, i, sysctl_tp_backoff, "IU", "TOE retransmit backoff"); } } #endif } void vi_sysctls(struct vi_info *vi) { struct sysctl_ctx_list *ctx = &vi->ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; /* * dev.v?(cxgbe|cxl).X. */ oid = device_get_sysctl_tree(vi->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, vi->viid, "VI identifer"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, &vi->nrxq, 0, "# of rx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, &vi->ntxq, 0, "# of tx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, &vi->first_rxq, 0, "index of first rx queue"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, &vi->first_txq, 0, "index of first tx queue"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL, vi->rss_base, "start of RSS indirection table"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL, vi->rss_size, "size of RSS indirection table"); if (IS_MAIN_VI(vi)) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_noflowq, "IU", "Reserve queue 0 for non-flowid packets"); } if (vi->adapter->flags & IS_VF) { MPASS(vi->flags & TX_USES_VM_WR); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_vm_wr", CTLFLAG_RD, NULL, 1, "use VM work requests for transmit"); } else { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_vm_wr", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_tx_vm_wr, "I", "use VM work requestes for transmit"); } #ifdef TCP_OFFLOAD if (vi->nofldrxq != 0) { SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, &vi->nofldrxq, 0, "# of rx queues for offloaded TCP connections"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", CTLFLAG_RD, &vi->first_ofld_rxq, 0, "index of first TOE rx queue"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_holdoff_tmr_idx_ofld, "I", "holdoff timer index for TOE queues"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_holdoff_pktc_idx_ofld, "I", "holdoff packet counter index for TOE queues"); } #endif #if defined(TCP_OFFLOAD) || defined(RATELIMIT) if (vi->nofldtxq != 0) { SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, &vi->nofldtxq, 0, "# of tx queues for TOE/ETHOFLD"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", CTLFLAG_RD, &vi->first_ofld_txq, 0, "index of first TOE/ETHOFLD tx queue"); } #endif #ifdef DEV_NETMAP if (vi->nnmrxq != 0) { SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, &vi->nnmrxq, 0, "# of netmap rx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, &vi->nnmtxq, 0, "# of netmap tx queues"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", CTLFLAG_RD, &vi->first_nm_rxq, 0, "index of first netmap rx queue"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", CTLFLAG_RD, &vi->first_nm_txq, 0, "index of first netmap tx queue"); } #endif SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_holdoff_tmr_idx, "I", "holdoff timer index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_holdoff_pktc_idx, "I", "holdoff packet counter index"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_qsize_rxq, "I", "rx queue size"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0, sysctl_qsize_txq, "I", "tx queue size"); } static void cxgbe_sysctls(struct port_info *pi) { struct sysctl_ctx_list *ctx = &pi->ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children, *children2; struct adapter *sc = pi->adapter; int i; char name[16]; static char *tc_flags = {"\20\1USER"}; /* * dev.cxgbe.X. */ oid = device_get_sysctl_tree(pi->dev); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0, sysctl_btphy, "I", "PHY temperature (in Celsius)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 1, sysctl_btphy, "I", "PHY firmware version"); } SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, sysctl_pause_settings, "A", "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "link_fec", CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_link_fec, "A", "FEC in use on the link"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "requested_fec", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, sysctl_requested_fec, "A", "FECs to use (bit 0 = RS, 1 = FC, 2 = none, 5 = auto, 6 = module)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "module_fec", CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_module_fec, "A", "FEC recommended by the cable/transceiver"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, sysctl_autoneg, "I", "autonegotiation (-1 = not supported)"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "force_fec", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0, sysctl_force_fec, "I", "when to use FORCE_FEC bit for link config"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rcaps", CTLFLAG_RD, &pi->link_cfg.requested_caps, 0, "L1 config requested by driver"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcaps", CTLFLAG_RD, &pi->link_cfg.pcaps, 0, "port capabilities"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "acaps", CTLFLAG_RD, &pi->link_cfg.acaps, 0, "advertised capabilities"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpacaps", CTLFLAG_RD, &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, port_top_speed(pi), "max speed (in Gbps)"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL, pi->mps_bg_map, "MPS buffer group map"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD, NULL, pi->rx_e_chan_map, "TP rx e-channel map"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_chan", CTLFLAG_RD, NULL, pi->tx_chan, "TP tx c-channel"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_chan", CTLFLAG_RD, NULL, pi->rx_chan, "TP rx c-channel"); if (sc->flags & IS_VF) return; /* * dev.(cxgbe|cxl).X.tc. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Tx scheduler traffic classes (cl_rl)"); children2 = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize", CTLFLAG_RW, &pi->sched_params->pktsize, 0, "pktsize for per-flow cl-rl (0 means up to the driver )"); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize", CTLFLAG_RW, &pi->sched_params->burstsize, 0, "burstsize for per-flow cl-rl (0 means up to the driver)"); for (i = 0; i < sc->params.nsched_cls; i++) { struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i]; snprintf(name, sizeof(name), "%d", i); children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "traffic class")); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "state", CTLFLAG_RD, &tc->state, 0, "current state"); SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, tc_flags, (uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags"); SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount", CTLFLAG_RD, &tc->refcount, 0, "references to this class"); SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, (pi->port_id << 16) | i, sysctl_tc_params, "A", "traffic class parameters"); } /* * dev.cxgbe.X.stats. */ oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "port statistics"); children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, &pi->tx_parse_error, 0, "# of tx packets with invalid length or # of segments"); #define T4_REGSTAT(name, stat, desc) \ SYSCTL_ADD_OID(ctx, children, OID_AUTO, #name, \ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, \ t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_##stat##_L), \ sysctl_handle_t4_reg64, "QU", desc) /* We get these from port_stats and they may be stale by up to 1s */ #define T4_PORTSTAT(name, desc) \ SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ &pi->stats.name, desc) T4_REGSTAT(tx_octets, TX_PORT_BYTES, "# of octets in good frames"); T4_REGSTAT(tx_frames, TX_PORT_FRAMES, "total # of good frames"); T4_REGSTAT(tx_bcast_frames, TX_PORT_BCAST, "# of broadcast frames"); T4_REGSTAT(tx_mcast_frames, TX_PORT_MCAST, "# of multicast frames"); T4_REGSTAT(tx_ucast_frames, TX_PORT_UCAST, "# of unicast frames"); T4_REGSTAT(tx_error_frames, TX_PORT_ERROR, "# of error frames"); T4_REGSTAT(tx_frames_64, TX_PORT_64B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_65_127, TX_PORT_65B_127B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_128_255, TX_PORT_128B_255B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_256_511, TX_PORT_256B_511B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_512_1023, TX_PORT_512B_1023B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_1024_1518, TX_PORT_1024B_1518B, "# of tx frames in this range"); T4_REGSTAT(tx_frames_1519_max, TX_PORT_1519B_MAX, "# of tx frames in this range"); T4_REGSTAT(tx_drop, TX_PORT_DROP, "# of dropped tx frames"); T4_REGSTAT(tx_pause, TX_PORT_PAUSE, "# of pause frames transmitted"); T4_REGSTAT(tx_ppp0, TX_PORT_PPP0, "# of PPP prio 0 frames transmitted"); T4_REGSTAT(tx_ppp1, TX_PORT_PPP1, "# of PPP prio 1 frames transmitted"); T4_REGSTAT(tx_ppp2, TX_PORT_PPP2, "# of PPP prio 2 frames transmitted"); T4_REGSTAT(tx_ppp3, TX_PORT_PPP3, "# of PPP prio 3 frames transmitted"); T4_REGSTAT(tx_ppp4, TX_PORT_PPP4, "# of PPP prio 4 frames transmitted"); T4_REGSTAT(tx_ppp5, TX_PORT_PPP5, "# of PPP prio 5 frames transmitted"); T4_REGSTAT(tx_ppp6, TX_PORT_PPP6, "# of PPP prio 6 frames transmitted"); T4_REGSTAT(tx_ppp7, TX_PORT_PPP7, "# of PPP prio 7 frames transmitted"); T4_REGSTAT(rx_octets, RX_PORT_BYTES, "# of octets in good frames"); T4_REGSTAT(rx_frames, RX_PORT_FRAMES, "total # of good frames"); T4_REGSTAT(rx_bcast_frames, RX_PORT_BCAST, "# of broadcast frames"); T4_REGSTAT(rx_mcast_frames, RX_PORT_MCAST, "# of multicast frames"); T4_REGSTAT(rx_ucast_frames, RX_PORT_UCAST, "# of unicast frames"); T4_REGSTAT(rx_too_long, RX_PORT_MTU_ERROR, "# of frames exceeding MTU"); T4_REGSTAT(rx_jabber, RX_PORT_MTU_CRC_ERROR, "# of jabber frames"); if (is_t6(sc)) { T4_PORTSTAT(rx_fcs_err, "# of frames received with bad FCS since last link up"); } else { T4_REGSTAT(rx_fcs_err, RX_PORT_CRC_ERROR, "# of frames received with bad FCS"); } T4_REGSTAT(rx_len_err, RX_PORT_LEN_ERROR, "# of frames received with length error"); T4_REGSTAT(rx_symbol_err, RX_PORT_SYM_ERROR, "symbol errors"); T4_REGSTAT(rx_runt, RX_PORT_LESS_64B, "# of short frames received"); T4_REGSTAT(rx_frames_64, RX_PORT_64B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_65_127, RX_PORT_65B_127B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_128_255, RX_PORT_128B_255B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_256_511, RX_PORT_256B_511B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_512_1023, RX_PORT_512B_1023B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_1024_1518, RX_PORT_1024B_1518B, "# of rx frames in this range"); T4_REGSTAT(rx_frames_1519_max, RX_PORT_1519B_MAX, "# of rx frames in this range"); T4_REGSTAT(rx_pause, RX_PORT_PAUSE, "# of pause frames received"); T4_REGSTAT(rx_ppp0, RX_PORT_PPP0, "# of PPP prio 0 frames received"); T4_REGSTAT(rx_ppp1, RX_PORT_PPP1, "# of PPP prio 1 frames received"); T4_REGSTAT(rx_ppp2, RX_PORT_PPP2, "# of PPP prio 2 frames received"); T4_REGSTAT(rx_ppp3, RX_PORT_PPP3, "# of PPP prio 3 frames received"); T4_REGSTAT(rx_ppp4, RX_PORT_PPP4, "# of PPP prio 4 frames received"); T4_REGSTAT(rx_ppp5, RX_PORT_PPP5, "# of PPP prio 5 frames received"); T4_REGSTAT(rx_ppp6, RX_PORT_PPP6, "# of PPP prio 6 frames received"); T4_REGSTAT(rx_ppp7, RX_PORT_PPP7, "# of PPP prio 7 frames received"); T4_PORTSTAT(rx_ovflow0, "# drops due to buffer-group 0 overflows"); T4_PORTSTAT(rx_ovflow1, "# drops due to buffer-group 1 overflows"); T4_PORTSTAT(rx_ovflow2, "# drops due to buffer-group 2 overflows"); T4_PORTSTAT(rx_ovflow3, "# drops due to buffer-group 3 overflows"); T4_PORTSTAT(rx_trunc0, "# of buffer-group 0 truncated packets"); T4_PORTSTAT(rx_trunc1, "# of buffer-group 1 truncated packets"); T4_PORTSTAT(rx_trunc2, "# of buffer-group 2 truncated packets"); T4_PORTSTAT(rx_trunc3, "# of buffer-group 3 truncated packets"); #undef T4_REGSTAT #undef T4_PORTSTAT } static int sysctl_int_array(SYSCTL_HANDLER_ARGS) { int rc, *i, space = 0; struct sbuf sb; sbuf_new_for_sysctl(&sb, NULL, 64, req); for (i = arg1; arg2; arg2 -= sizeof(int), i++) { if (space) sbuf_printf(&sb, " "); sbuf_printf(&sb, "%d", *i); space = 1; } rc = sbuf_finish(&sb); sbuf_delete(&sb); return (rc); } static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS) { int rc; struct sbuf *sb; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS) { int rc; struct sbuf *sb; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_btphy(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; int op = arg2; struct adapter *sc = pi->adapter; u_int v; int rc; rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { /* XXX: magic numbers */ rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, &v); } end_synchronized_op(sc, 0); if (rc) return (rc); if (op == 0) v /= 256; rc = sysctl_handle_int(oidp, &v, 0, req); return (rc); } static int sysctl_noflowq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; int rc, val; val = vi->rsrv_noflowq; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if ((val >= 1) && (vi->ntxq > 1)) vi->rsrv_noflowq = 1; else vi->rsrv_noflowq = 0; return (rc); } static int sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int rc, val, i; MPASS(!(sc->flags & IS_VF)); val = vi->flags & TX_USES_VM_WR ? 1 : 0; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (val != 0 && val != 1) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4txvm"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else if (if_getdrvflags(vi->ifp) & IFF_DRV_RUNNING) { /* * We don't want parse_pkt to run with one setting (VF or PF) * and then eth_tx to see a different setting but still use * stale information calculated by parse_pkt. */ rc = EBUSY; } else { struct port_info *pi = vi->pi; struct sge_txq *txq; uint32_t ctrl0; uint8_t npkt = sc->params.max_pkts_per_eth_tx_pkts_wr; if (val) { vi->flags |= TX_USES_VM_WR; if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO); ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | V_TXPKT_INTF(pi->tx_chan)); if (!(sc->flags & IS_VF)) npkt--; } else { vi->flags &= ~TX_USES_VM_WR; if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO); ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); } for_each_txq(vi, i, txq) { txq->cpl_ctrl0 = ctrl0; txq->txp.max_npkt = npkt; } } end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int idx, rc, i; struct sge_rxq *rxq; uint8_t v; idx = vi->tmr_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < 0 || idx >= SGE_NTIMERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4tmr"); if (rc) return (rc); v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); for_each_rxq(vi, i, rxq) { #ifdef atomic_store_rel_8 atomic_store_rel_8(&rxq->iq.intr_params, v); #else rxq->iq.intr_params = v; #endif } vi->tmr_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (0); } static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int idx, rc; idx = vi->pktc_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < -1 || idx >= SGE_NCOUNTERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4pktc"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->pktc_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int qsize, rc; qsize = vi->qsize_rxq; rc = sysctl_handle_int(oidp, &qsize, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (qsize < 128 || (qsize & 7)) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4rxqs"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->qsize_rxq = qsize; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int qsize, rc; qsize = vi->qsize_txq; rc = sysctl_handle_int(oidp, &qsize, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (qsize < 128 || qsize > 65536) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4txqs"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->qsize_txq = qsize; end_synchronized_op(sc, LOCK_HELD); return (rc); } static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; if (req->newptr == NULL) { struct sbuf *sb; static char *bits = "\20\1RX\2TX\3AUTO"; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); if (lc->link_ok) { sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) | (lc->requested_fc & PAUSE_AUTONEG), bits); } else { sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG), bits); } rc = sbuf_finish(sb); sbuf_delete(sb); } else { char s[2]; int n; s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)); s[1] = 0; rc = sysctl_handle_string(oidp, s, sizeof(s), req); if (rc != 0) return(rc); if (s[1] != 0) return (EINVAL); if (s[0] < '0' || s[0] > '9') return (EINVAL); /* not a number */ n = s[0] - '0'; if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) return (EINVAL); /* some other bit is set too */ rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4PAUSE"); if (rc) return (rc); if (!hw_off_limits(sc)) { PORT_LOCK(pi); lc->requested_fc = n; fixup_link_config(pi); if (pi->up_vis > 0) rc = apply_link_config(pi); set_current_media(pi); PORT_UNLOCK(pi); } end_synchronized_op(sc, 0); } return (rc); } static int sysctl_link_fec(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct link_config *lc = &pi->link_cfg; int rc; struct sbuf *sb; static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD1\5RSVD2"; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); if (lc->link_ok) sbuf_printf(sb, "%b", lc->fec, bits); else sbuf_printf(sb, "no link"); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_requested_fec(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; int8_t old; if (req->newptr == NULL) { struct sbuf *sb; static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2" "\5RSVD3\6auto\7module"; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%b", lc->requested_fec, bits); rc = sbuf_finish(sb); sbuf_delete(sb); } else { char s[8]; int n; snprintf(s, sizeof(s), "%d", lc->requested_fec == FEC_AUTO ? -1 : lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE)); rc = sysctl_handle_string(oidp, s, sizeof(s), req); if (rc != 0) return(rc); n = strtol(&s[0], NULL, 0); if (n < 0 || n & FEC_AUTO) n = FEC_AUTO; else if (n & ~(M_FW_PORT_CAP32_FEC | FEC_MODULE)) return (EINVAL);/* some other bit is set too */ rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4reqf"); if (rc) return (rc); PORT_LOCK(pi); old = lc->requested_fec; if (n == FEC_AUTO) lc->requested_fec = FEC_AUTO; else if (n == 0 || n == FEC_NONE) lc->requested_fec = FEC_NONE; else { if ((lc->pcaps | V_FW_PORT_CAP32_FEC(n & M_FW_PORT_CAP32_FEC)) != lc->pcaps) { rc = ENOTSUP; goto done; } lc->requested_fec = n & (M_FW_PORT_CAP32_FEC | FEC_MODULE); } if (!hw_off_limits(sc)) { fixup_link_config(pi); if (pi->up_vis > 0) { rc = apply_link_config(pi); if (rc != 0) { lc->requested_fec = old; if (rc == FW_EPROTO) rc = ENOTSUP; } } } done: PORT_UNLOCK(pi); end_synchronized_op(sc, 0); } return (rc); } static int sysctl_module_fec(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc; int8_t fec; struct sbuf *sb; static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2\5RSVD3"; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) return (ENOMEM); if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mfec") != 0) { rc = EBUSY; goto done; } if (hw_off_limits(sc)) { rc = ENXIO; goto done; } PORT_LOCK(pi); if (pi->up_vis == 0) { /* * If all the interfaces are administratively down the firmware * does not report transceiver changes. Refresh port info here. * This is the only reason we have a synchronized op in this * function. Just PORT_LOCK would have been enough otherwise. */ t4_update_port_info(pi); } fec = lc->fec_hint; if (pi->mod_type == FW_PORT_MOD_TYPE_NONE || !fec_supported(lc->pcaps)) { PORT_UNLOCK(pi); sbuf_printf(sb, "n/a"); } else { if (fec == 0) fec = FEC_NONE; PORT_UNLOCK(pi); sbuf_printf(sb, "%b", fec & M_FW_PORT_CAP32_FEC, bits); } rc = sbuf_finish(sb); done: sbuf_delete(sb); end_synchronized_op(sc, 0); return (rc); } static int sysctl_autoneg(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc, val; if (lc->pcaps & FW_PORT_CAP32_ANEG) val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1; else val = -1; rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (val == 0) val = AUTONEG_DISABLE; else if (val == 1) val = AUTONEG_ENABLE; else val = AUTONEG_AUTO; rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4aneg"); if (rc) return (rc); PORT_LOCK(pi); if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) { rc = ENOTSUP; goto done; } lc->requested_aneg = val; if (!hw_off_limits(sc)) { fixup_link_config(pi); if (pi->up_vis > 0) rc = apply_link_config(pi); set_current_media(pi); } done: PORT_UNLOCK(pi); end_synchronized_op(sc, 0); return (rc); } static int sysctl_force_fec(SYSCTL_HANDLER_ARGS) { struct port_info *pi = arg1; struct adapter *sc = pi->adapter; struct link_config *lc = &pi->link_cfg; int rc, val; val = lc->force_fec; MPASS(val >= -1 && val <= 1); rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC)) return (ENOTSUP); if (val < -1 || val > 1) return (EINVAL); rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff"); if (rc) return (rc); PORT_LOCK(pi); lc->force_fec = val; if (!hw_off_limits(sc)) { fixup_link_config(pi); if (pi->up_vis > 0) rc = apply_link_config(pi); } PORT_UNLOCK(pi); end_synchronized_op(sc, 0); return (rc); } static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, reg = arg2; uint64_t val; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = 0; val = t4_read_reg64(sc, reg); } mtx_unlock(&sc->reg_lock); if (rc == 0) rc = sysctl_handle_64(oidp, &val, 0, req); return (rc); } static int sysctl_temperature(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, t; uint32_t param, val; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); } end_synchronized_op(sc, 0); if (rc) return (rc); /* unknown is returned as 0 but we display -1 in that case */ t = val == 0 ? -1 : val; rc = sysctl_handle_int(oidp, &t, 0, req); return (rc); } static int sysctl_vdd(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc; uint32_t param, val; if (sc->params.core_vdd == 0) { rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vdd"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); } end_synchronized_op(sc, 0); if (rc) return (rc); sc->params.core_vdd = val; } return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req)); } static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, v; uint32_t param, val; v = sc->sensor_resets; rc = sysctl_handle_int(oidp, &v, 0, req); if (rc != 0 || req->newptr == NULL || v <= 0) return (rc); if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) || chip_id(sc) < CHELSIO_T5) return (ENOTSUP); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4srst"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_RESET_TMP_SENSOR)); val = 1; rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); } end_synchronized_op(sc, 0); if (rc == 0) sc->sensor_resets++; return (rc); } static int sysctl_loadavg(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; uint32_t param, val; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD); rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); } end_synchronized_op(sc, 0); if (rc) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); if (val == 0xffffffff) { /* Only debug and custom firmwares report load averages. */ sbuf_printf(sb, "not available"); } else { sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff, (val >> 16) & 0xff); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_cctrl(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint16_t incr[NMTUS][NCCTRL_WIN]; static const char *dec_fac[] = { "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", "0.9375" }; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_read_cong_tbl(sc, incr); mtx_unlock(&sc->reg_lock); if (rc) goto done; for (i = 0; i < NCCTRL_WIN; ++i) { sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], incr[5][i], incr[6][i], incr[7][i]); sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", incr[8][i], incr[9][i], incr[10][i], incr[11][i], incr[12][i], incr[13][i], incr[14][i], incr[15][i], sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); } rc = sbuf_finish(sb); done: sbuf_delete(sb); return (rc); } static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ }; static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, n, qid = arg2; uint32_t *buf, *p; char *qtype; u_int cim_num_obq = sc->chip_params->cim_num_obq; KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, ("%s: bad qid %d\n", __func__, qid)); if (qid < CIM_NUM_IBQ) { /* inbound queue */ qtype = "IBQ"; n = 4 * CIM_IBQ_SIZE; buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = -ENXIO; else rc = t4_read_cim_ibq(sc, qid, buf, n); mtx_unlock(&sc->reg_lock); } else { /* outbound queue */ qtype = "OBQ"; qid -= CIM_NUM_IBQ; n = 4 * cim_num_obq * CIM_OBQ_SIZE; buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = -ENXIO; else rc = t4_read_cim_obq(sc, qid, buf, n); mtx_unlock(&sc->reg_lock); } if (rc < 0) { rc = -rc; goto done; } n = rc * sizeof(uint32_t); /* rc has # of words actually read */ sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); if (sb == NULL) { rc = ENOMEM; goto done; } sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); for (i = 0, p = buf; i < n; i += 16, p += 4) sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], p[2], p[3]); rc = sbuf_finish(sb); sbuf_delete(sb); done: free(buf, M_CXGBE); return (rc); } static void sbuf_cim_la4(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg) { uint32_t *p; sbuf_printf(sb, "Status Data PC%s", cfg & F_UPDBGLACAPTPCONLY ? "" : " LS0Stat LS0Addr LS0Data"); for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { if (cfg & F_UPDBGLACAPTPCONLY) { sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, p[6], p[7]); sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, p[4] & 0xff, p[5] >> 8); sbuf_printf(sb, "\n %02x %x%07x %x%07x", (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, p[1] & 0xf, p[2] >> 4); } else { sbuf_printf(sb, "\n %02x %x%07x %x%07x %08x %08x " "%08x%08x%08x%08x", (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], p[6], p[7]); } } } static void sbuf_cim_la6(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg) { uint32_t *p; sbuf_printf(sb, "Status Inst Data PC%s", cfg & F_UPDBGLACAPTPCONLY ? "" : " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { if (cfg & F_UPDBGLACAPTPCONLY) { sbuf_printf(sb, "\n %02x %08x %08x %08x", p[3] & 0xff, p[2], p[1], p[0]); sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, p[6] >> 16); } else { sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " "%08x %08x %08x %08x %08x %08x", (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, p[6] >> 16, p[2], p[1], p[0], p[5], p[4], p[3]); } } } static int sbuf_cim_la(struct adapter *sc, struct sbuf *sb, int flags) { uint32_t cfg, *buf; int rc; MPASS(flags == M_WAITOK || flags == M_NOWAIT); buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, M_ZERO | flags); if (buf == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); if (rc == 0) rc = -t4_cim_read_la(sc, buf, NULL); } mtx_unlock(&sc->reg_lock); if (rc == 0) { if (chip_id(sc) < CHELSIO_T6) sbuf_cim_la4(sc, sb, buf, cfg); else sbuf_cim_la6(sc, sb, buf, cfg); } free(buf, M_CXGBE); return (rc); } static int sysctl_cim_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); rc = sbuf_cim_la(sc, sb, M_WAITOK); if (rc == 0) rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static void dump_cim_regs(struct adapter *sc) { log(LOG_DEBUG, "%s: CIM debug regs1 %08x %08x %08x %08x %08x\n", device_get_nameunit(sc->dev), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA2), t4_read_reg(sc, A_EDC_H_BIST_DATA_PATTERN), t4_read_reg(sc, A_EDC_H_BIST_STATUS_RDATA)); log(LOG_DEBUG, "%s: CIM debug regs2 %08x %08x %08x %08x %08x\n", device_get_nameunit(sc->dev), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0 + 0x800), t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1 + 0x800), t4_read_reg(sc, A_EDC_H_BIST_CMD_LEN)); } static void dump_cimla(struct adapter *sc) { struct sbuf sb; int rc; if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) { log(LOG_DEBUG, "%s: failed to generate CIM LA dump.\n", device_get_nameunit(sc->dev)); return; } rc = sbuf_cim_la(sc, &sb, M_WAITOK); if (rc == 0) { rc = sbuf_finish(&sb); if (rc == 0) { log(LOG_DEBUG, "%s: CIM LA dump follows.\n%s\n", device_get_nameunit(sc->dev), sbuf_data(&sb)); } } sbuf_delete(&sb); } void t4_os_cim_err(struct adapter *sc) { atomic_set_int(&sc->error_flags, ADAP_CIM_ERR); } static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int i; struct sbuf *sb; uint32_t *buf, *p; int rc; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); mtx_unlock(&sc->reg_lock); if (rc) goto done; p = buf; for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], p[1], p[0]); } sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, (p[1] >> 2) | ((p[2] & 3) << 30), (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, p[0] & 1); } rc = sbuf_finish(sb); done: sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int i; struct sbuf *sb; uint32_t *buf, *p; int rc; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); mtx_unlock(&sc->reg_lock); if (rc) goto done; p = buf; sbuf_printf(sb, "Cntl ID DataBE Addr Data"); for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, p[4], p[3], p[2], p[1], p[0]); } sbuf_printf(sb, "\n\nCntl ID Data"); for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); } rc = sbuf_finish(sb); done: sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; uint16_t thres[CIM_NUM_IBQ]; uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; cim_num_obq = sc->chip_params->cim_num_obq; if (is_t4(sc)) { ibq_rdaddr = A_UP_IBQ_0_RDADDR; obq_rdaddr = A_UP_OBQ_0_REALADDR; } else { ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; } nq = CIM_NUM_IBQ + cim_num_obq; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); if (rc == 0) { rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); if (rc == 0) t4_read_cimq_cfg(sc, base, size, thres); } } mtx_unlock(&sc->reg_lock); if (rc) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), G_QUEREMFLITS(p[2]) * 16); for ( ; i < nq; i++, p += 4, wr += 2) sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), G_QUEREMFLITS(p[2]) * 16); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_cpl_stats stats; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_cpl_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); if (rc) goto done; if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3"); sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", stats.req[0], stats.req[1], stats.req[2], stats.req[3]); sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); } else { sbuf_printf(sb, " channel 0 channel 1"); sbuf_printf(sb, "\nCPL requests: %10u %10u", stats.req[0], stats.req[1]); sbuf_printf(sb, "\nCPL responses: %10u %10u", stats.rsp[0], stats.rsp[1]); } rc = sbuf_finish(sb); done: sbuf_delete(sb); return (rc); } static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_usm_stats stats; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_get_usm_stats(sc, &stats, 1); mtx_unlock(&sc->reg_lock); if (rc == 0) { sbuf_printf(sb, "Frames: %u\n", stats.frames); sbuf_printf(sb, "Octets: %ju\n", stats.octets); sbuf_printf(sb, "Drops: %u", stats.drops); rc = sbuf_finish(sb); } sbuf_delete(sb); return (rc); } static int sysctl_tid_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_tid_stats stats; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_tid_stats(sc, &stats, 1); mtx_unlock(&sc->reg_lock); if (rc == 0) { sbuf_printf(sb, "Delete: %u\n", stats.del); sbuf_printf(sb, "Invalidate: %u\n", stats.inv); sbuf_printf(sb, "Active: %u\n", stats.act); sbuf_printf(sb, "Passive: %u", stats.pas); rc = sbuf_finish(sb); } sbuf_delete(sb); return (rc); } static const char * const devlog_level_strings[] = { [FW_DEVLOG_LEVEL_EMERG] = "EMERG", [FW_DEVLOG_LEVEL_CRIT] = "CRIT", [FW_DEVLOG_LEVEL_ERR] = "ERR", [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", [FW_DEVLOG_LEVEL_INFO] = "INFO", [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" }; static const char * const devlog_facility_strings[] = { [FW_DEVLOG_FACILITY_CORE] = "CORE", [FW_DEVLOG_FACILITY_CF] = "CF", [FW_DEVLOG_FACILITY_SCHED] = "SCHED", [FW_DEVLOG_FACILITY_TIMER] = "TIMER", [FW_DEVLOG_FACILITY_RES] = "RES", [FW_DEVLOG_FACILITY_HW] = "HW", [FW_DEVLOG_FACILITY_FLR] = "FLR", [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", [FW_DEVLOG_FACILITY_PHY] = "PHY", [FW_DEVLOG_FACILITY_MAC] = "MAC", [FW_DEVLOG_FACILITY_PORT] = "PORT", [FW_DEVLOG_FACILITY_VI] = "VI", [FW_DEVLOG_FACILITY_FILTER] = "FILTER", [FW_DEVLOG_FACILITY_ACL] = "ACL", [FW_DEVLOG_FACILITY_TM] = "TM", [FW_DEVLOG_FACILITY_QFC] = "QFC", [FW_DEVLOG_FACILITY_DCB] = "DCB", [FW_DEVLOG_FACILITY_ETH] = "ETH", [FW_DEVLOG_FACILITY_OFLD] = "OFLD", [FW_DEVLOG_FACILITY_RI] = "RI", [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", [FW_DEVLOG_FACILITY_FCOE] = "FCOE", [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE", [FW_DEVLOG_FACILITY_CHNET] = "CHNET", }; static int sbuf_devlog(struct adapter *sc, struct sbuf *sb, int flags) { int i, j, rc, nentries, first = 0; struct devlog_params *dparams = &sc->params.devlog; struct fw_devlog_e *buf, *e; uint64_t ftstamp = UINT64_MAX; if (dparams->addr == 0) return (ENXIO); MPASS(flags == M_WAITOK || flags == M_NOWAIT); buf = malloc(dparams->size, M_CXGBE, M_ZERO | flags); if (buf == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size); mtx_unlock(&sc->reg_lock); if (rc != 0) goto done; nentries = dparams->size / sizeof(struct fw_devlog_e); for (i = 0; i < nentries; i++) { e = &buf[i]; if (e->timestamp == 0) break; /* end */ e->timestamp = be64toh(e->timestamp); e->seqno = be32toh(e->seqno); for (j = 0; j < 8; j++) e->params[j] = be32toh(e->params[j]); if (e->timestamp < ftstamp) { ftstamp = e->timestamp; first = i; } } if (buf[first].timestamp == 0) goto done; /* nothing in the log */ sbuf_printf(sb, "%10s %15s %8s %8s %s\n", "Seq#", "Tstamp", "Level", "Facility", "Message"); i = first; do { e = &buf[i]; if (e->timestamp == 0) break; /* end */ sbuf_printf(sb, "%10d %15ju %8s %8s ", e->seqno, e->timestamp, (e->level < nitems(devlog_level_strings) ? devlog_level_strings[e->level] : "UNKNOWN"), (e->facility < nitems(devlog_facility_strings) ? devlog_facility_strings[e->facility] : "UNKNOWN")); sbuf_printf(sb, e->fmt, e->params[0], e->params[1], e->params[2], e->params[3], e->params[4], e->params[5], e->params[6], e->params[7]); if (++i == nentries) i = 0; } while (i != first); done: free(buf, M_CXGBE); return (rc); } static int sysctl_devlog(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc; struct sbuf *sb; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); rc = sbuf_devlog(sc, sb, M_WAITOK); if (rc == 0) rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static void dump_devlog(struct adapter *sc) { int rc; struct sbuf sb; if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) { log(LOG_DEBUG, "%s: failed to generate devlog dump.\n", device_get_nameunit(sc->dev)); return; } rc = sbuf_devlog(sc, &sb, M_WAITOK); if (rc == 0) { rc = sbuf_finish(&sb); if (rc == 0) { log(LOG_DEBUG, "%s: device log follows.\n%s", device_get_nameunit(sc->dev), sbuf_data(&sb)); } } sbuf_delete(&sb); } static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_fcoe_stats stats[MAX_NCHAN]; int i, nchan = sc->chip_params->nchan; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { for (i = 0; i < nchan; i++) t4_get_fcoe_stats(sc, i, &stats[i], 1); } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3"); sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", stats[0].octets_ddp, stats[1].octets_ddp, stats[2].octets_ddp, stats[3].octets_ddp); sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", stats[0].frames_ddp, stats[1].frames_ddp, stats[2].frames_ddp, stats[3].frames_ddp); sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", stats[0].frames_drop, stats[1].frames_drop, stats[2].frames_drop, stats[3].frames_drop); } else { sbuf_printf(sb, " channel 0 channel 1"); sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", stats[0].octets_ddp, stats[1].octets_ddp); sbuf_printf(sb, "\nframesDDP: %16u %16u", stats[0].frames_ddp, stats[1].frames_ddp); sbuf_printf(sb, "\nframesDrop: %16u %16u", stats[0].frames_drop, stats[1].frames_drop); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; unsigned int map, kbps, ipg, mode; unsigned int pace_tab[NTX_SCHED]; sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); if (sb == NULL) return (ENOMEM); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) { mtx_unlock(&sc->reg_lock); rc = ENXIO; goto done; } map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); t4_read_pace_tbl(sc, pace_tab); mtx_unlock(&sc->reg_lock); sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " "Class IPG (0.1 ns) Flow IPG (us)"); for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { t4_get_tx_sched(sc, i, &kbps, &ipg, 1); sbuf_printf(sb, "\n %u %-5s %u ", i, (mode & (1 << i)) ? "flow" : "class", map & 3); if (kbps) sbuf_printf(sb, "%9u ", kbps); else sbuf_printf(sb, " disabled "); if (ipg) sbuf_printf(sb, "%13u ", ipg); else sbuf_printf(sb, " disabled "); if (pace_tab[i]) sbuf_printf(sb, "%10u", pace_tab[i]); else sbuf_printf(sb, " disabled"); } rc = sbuf_finish(sb); done: sbuf_delete(sb); return (rc); } static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, j; uint64_t *p0, *p1; struct lb_port_stats s[2]; static const char *stat_name[] = { "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", "Frames128To255:", "Frames256To511:", "Frames512To1023:", "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", "BG2FramesTrunc:", "BG3FramesTrunc:" }; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); memset(s, 0, sizeof(s)); rc = 0; for (i = 0; i < sc->chip_params->nchan; i += 2) { mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { t4_get_lb_stats(sc, i, &s[0]); t4_get_lb_stats(sc, i + 1, &s[1]); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; p0 = &s[0].octets; p1 = &s[1].octets; sbuf_printf(sb, "%s Loopback %u" " Loopback %u", i == 0 ? "" : "\n", i, i + 1); for (j = 0; j < nitems(stat_name); j++) sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], *p0++, *p1++); } if (rc == 0) rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) { int rc = 0; struct port_info *pi = arg1; struct link_config *lc = &pi->link_cfg; struct sbuf *sb; sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); if (sb == NULL) return (ENOMEM); if (lc->link_ok || lc->link_down_rc == 255) sbuf_printf(sb, "n/a"); else sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc)); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } struct mem_desc { u_int base; u_int limit; u_int idx; }; static int mem_desc_cmp(const void *a, const void *b) { const u_int v1 = ((const struct mem_desc *)a)->base; const u_int v2 = ((const struct mem_desc *)b)->base; if (v1 < v2) return (-1); else if (v1 > v2) return (1); return (0); } static void mem_region_show(struct sbuf *sb, const char *name, unsigned int from, unsigned int to) { unsigned int size; if (from == to) return; size = to - from + 1; if (size == 0) return; /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); } static int sysctl_meminfo(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i, n; uint32_t lo, hi, used, free, alloc; static const char *memory[] = { "EDC0:", "EDC1:", "MC:", "MC0:", "MC1:", "HMA:" }; static const char *region[] = { "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", "TDDP region:", "TPT region:", "STAG region:", "RQ region:", "RQUDP region:", "PBL region:", "TXPBL region:", "TLSKey region:", "DBVFIFO region:", "ULPRX state:", "ULPTX state:", "On-chip queues:", }; struct mem_desc avail[4]; struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ struct mem_desc *md = mem; rc = sysctl_wire_old_buffer(req, 0); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); for (i = 0; i < nitems(mem); i++) { mem[i].limit = 0; mem[i].idx = i; } mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } /* Find and sort the populated memory ranges */ i = 0; lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); if (lo & F_EDRAM0_ENABLE) { hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); avail[i].base = G_EDRAM0_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); avail[i].idx = 0; i++; } if (lo & F_EDRAM1_ENABLE) { hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); avail[i].base = G_EDRAM1_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); avail[i].idx = 1; i++; } if (lo & F_EXT_MEM_ENABLE) { hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); avail[i].base = G_EXT_MEM_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */ i++; } if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) { hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); avail[i].base = G_EXT_MEM1_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EXT_MEM1_SIZE(hi) << 20); avail[i].idx = 4; i++; } if (is_t6(sc) && lo & F_HMA_MUX) { hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); avail[i].base = G_EXT_MEM1_BASE(hi) << 20; avail[i].limit = avail[i].base + (G_EXT_MEM1_SIZE(hi) << 20); avail[i].idx = 5; i++; } MPASS(i <= nitems(avail)); if (!i) /* no memory available */ goto done; qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); /* the next few have explicit upper bounds */ md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); md->limit = md->base - 1 + t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); md++; md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); md->limit = md->base - 1 + t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); md++; if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { if (chip_id(sc) <= CHELSIO_T5) md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); else md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); md->limit = 0; } else { md->base = 0; md->idx = nitems(region); /* hide it */ } md++; #define ulp_region(reg) \ md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) ulp_region(RX_ISCSI); ulp_region(RX_TDDP); ulp_region(TX_TPT); ulp_region(RX_STAG); ulp_region(RX_RQ); ulp_region(RX_RQUDP); ulp_region(RX_PBL); ulp_region(TX_PBL); if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) { ulp_region(RX_TLS_KEY); } #undef ulp_region md->base = 0; if (is_t4(sc)) md->idx = nitems(region); else { uint32_t size = 0; uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); if (is_t5(sc)) { if (sge_ctrl & F_VFIFO_ENABLE) size = fifo_size << 2; } else size = G_T6_DBVFIFO_SIZE(fifo_size) << 6; if (size) { md->base = t4_read_reg(sc, A_SGE_DBVFIFO_BADDR); md->limit = md->base + size - 1; } else md->idx = nitems(region); } md++; md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); md->limit = 0; md++; md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); md->limit = 0; md++; md->base = sc->vres.ocq.start; if (sc->vres.ocq.size) md->limit = md->base + sc->vres.ocq.size - 1; else md->idx = nitems(region); /* hide it */ md++; /* add any address-space holes, there can be up to 3 */ for (n = 0; n < i - 1; n++) if (avail[n].limit < avail[n + 1].base) (md++)->base = avail[n].limit; if (avail[n].limit) (md++)->base = avail[n].limit; n = md - mem; MPASS(n <= nitems(mem)); qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); for (lo = 0; lo < i; lo++) mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, avail[lo].limit - 1); sbuf_printf(sb, "\n"); for (i = 0; i < n; i++) { if (mem[i].idx >= nitems(region)) continue; /* skip holes */ if (!mem[i].limit) mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; mem_region_show(sb, region[mem[i].idx], mem[i].base, mem[i].limit); } sbuf_printf(sb, "\n"); lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; mem_region_show(sb, "uP RAM:", lo, hi); lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; mem_region_show(sb, "uP Extmem2:", lo, hi); lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); for (i = 0, free = 0; i < 2; i++) free += G_FREERXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_RX_CNT)); sbuf_printf(sb, "\n%u Rx pages (%u free) of size %uKiB for %u channels\n", G_PMRXMAXPAGE(lo), free, t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, (lo & F_PMRXNUMCHN) ? 2 : 1); lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); for (i = 0, free = 0; i < 4; i++) free += G_FREETXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_TX_CNT)); sbuf_printf(sb, "%u Tx pages (%u free) of size %u%ciB for %u channels\n", G_PMTXMAXPAGE(lo), free, hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); sbuf_printf(sb, "%u p-structs (%u free)\n", t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT), G_FREEPSTRUCTCOUNT(t4_read_reg(sc, A_TP_FLM_FREE_PS_CNT))); for (i = 0; i < 4; i++) { if (chip_id(sc) > CHELSIO_T5) lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); else lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); if (is_t5(sc)) { used = G_T5_USED(lo); alloc = G_T5_ALLOC(lo); } else { used = G_USED(lo); alloc = G_ALLOC(lo); } /* For T6 these are MAC buffer groups */ sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", i, used, alloc); } for (i = 0; i < sc->chip_params->nchan; i++) { if (chip_id(sc) > CHELSIO_T5) lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); else lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); if (is_t5(sc)) { used = G_T5_USED(lo); alloc = G_T5_ALLOC(lo); } else { used = G_USED(lo); alloc = G_ALLOC(lo); } /* For T6 these are MAC buffer groups */ sbuf_printf(sb, "\nLoopback %d using %u pages out of %u allocated", i, used, alloc); } done: mtx_unlock(&sc->reg_lock); if (rc == 0) rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static inline void tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) { *mask = x | y; y = htobe64(y); memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); } static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; MPASS(chip_id(sc) <= CHELSIO_T5); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "Idx Ethernet address Mask Vld Ports PF" " VF Replication P0 P1 P2 P3 ML"); rc = 0; for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { uint64_t tcamx, tcamy, mask; uint32_t cls_lo, cls_hi; uint8_t addr[ETHER_ADDR_LEN]; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; if (tcamx & tcamy) continue; tcamxy2valmask(tcamx, tcamy, addr, &mask); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask, (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', G_PORTMAP(cls_hi), G_PF(cls_lo), (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); if (cls_lo & F_REPLICATE) { struct fw_ldst_cmd ldst_cmd; memset(&ldst_cmd, 0, sizeof(ldst_cmd)); ldst_cmd.op_to_addrspace = htobe32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); ldst_cmd.u.mps.rplc.fid_idx = htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | V_FW_LDST_CMD_IDX(i)); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mps"); if (rc) break; if (hw_off_limits(sc)) rc = ENXIO; else rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, sizeof(ldst_cmd), &ldst_cmd); end_synchronized_op(sc, 0); if (rc != 0) break; else { sbuf_printf(sb, " %08x %08x %08x %08x", be32toh(ldst_cmd.u.mps.rplc.rplc127_96), be32toh(ldst_cmd.u.mps.rplc.rplc95_64), be32toh(ldst_cmd.u.mps.rplc.rplc63_32), be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); } } else sbuf_printf(sb, "%36s", ""); sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); } if (rc) (void) sbuf_finish(sb); else rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; MPASS(chip_id(sc) > CHELSIO_T5); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" " Replication" " P0 P1 P2 P3 ML\n"); rc = 0; for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { uint8_t dip_hit, vlan_vld, lookup_type, port_num; uint16_t ivlan; uint64_t tcamx, tcamy, val, mask; uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; uint8_t addr[ETHER_ADDR_LEN]; ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); if (i < 256) ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); else ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); tcamy = G_DMACH(val) << 32; tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; lookup_type = G_DATALKPTYPE(data2); port_num = G_DATAPORTNUM(data2); if (lookup_type && lookup_type != M_DATALKPTYPE) { /* Inner header VNI */ vniy = ((data2 & F_DATAVIDH2) << 23) | (G_DATAVIDH1(data2) << 16) | G_VIDL(val); dip_hit = data2 & F_DATADIPHIT; vlan_vld = 0; } else { vniy = 0; dip_hit = 0; vlan_vld = data2 & F_DATAVIDH2; ivlan = G_VIDL(val); } ctl |= V_CTLXYBITSEL(1); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); tcamx = G_DMACH(val) << 32; tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; if (lookup_type && lookup_type != M_DATALKPTYPE) { /* Inner header VNI mask */ vnix = ((data2 & F_DATAVIDH2) << 23) | (G_DATAVIDH1(data2) << 16) | G_VIDL(val); } else vnix = 0; if (tcamx & tcamy) continue; tcamxy2valmask(tcamx, tcamy, addr, &mask); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); } mtx_unlock(&sc->reg_lock); if (rc != 0) break; if (lookup_type && lookup_type != M_DATALKPTYPE) { sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " "%012jx %06x %06x - - %3c" " I %4x %3c %#x%4u%4d", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', G_PORTMAP(cls_hi), G_T6_PF(cls_lo), cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); } else { sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " "%012jx - - ", i, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], (uintmax_t)mask); if (vlan_vld) sbuf_printf(sb, "%4u Y ", ivlan); else sbuf_printf(sb, " - N "); sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", lookup_type ? 'I' : 'O', port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', G_PORTMAP(cls_hi), G_T6_PF(cls_lo), cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); } if (cls_lo & F_T6_REPLICATE) { struct fw_ldst_cmd ldst_cmd; memset(&ldst_cmd, 0, sizeof(ldst_cmd)); ldst_cmd.op_to_addrspace = htobe32(V_FW_CMD_OP(FW_LDST_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); ldst_cmd.u.mps.rplc.fid_idx = htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | V_FW_LDST_CMD_IDX(i)); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t6mps"); if (rc) break; if (hw_off_limits(sc)) rc = ENXIO; else rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, sizeof(ldst_cmd), &ldst_cmd); end_synchronized_op(sc, 0); if (rc != 0) break; else { sbuf_printf(sb, " %08x %08x %08x %08x" " %08x %08x %08x %08x", be32toh(ldst_cmd.u.mps.rplc.rplc255_224), be32toh(ldst_cmd.u.mps.rplc.rplc223_192), be32toh(ldst_cmd.u.mps.rplc.rplc191_160), be32toh(ldst_cmd.u.mps.rplc.rplc159_128), be32toh(ldst_cmd.u.mps.rplc.rplc127_96), be32toh(ldst_cmd.u.mps.rplc.rplc95_64), be32toh(ldst_cmd.u.mps.rplc.rplc63_32), be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); } } else sbuf_printf(sb, "%72s", ""); sbuf_printf(sb, "%4u%3u%3u%3u %#x", G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), (cls_lo >> S_T6_MULTILISTEN0) & 0xf); } if (rc) (void) sbuf_finish(sb); else rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; uint16_t mtus[NMTUS]; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_read_mtu_tbl(sc, mtus, NULL); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], mtus[14], mtus[15]); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc, i; uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; static const char *tx_stats[MAX_PM_NSTATS] = { "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", "Tx FIFO wait", NULL, "Tx latency" }; static const char *rx_stats[MAX_PM_NSTATS] = { "Read:", "Write bypass:", "Write mem:", "Flush:", "Rx FIFO wait", NULL, "Rx latency" }; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, " Tx pcmds Tx bytes"); for (i = 0; i < 4; i++) { sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); } sbuf_printf(sb, "\n Rx pcmds Rx bytes"); for (i = 0; i < 4; i++) { sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); } if (chip_id(sc) > CHELSIO_T5) { sbuf_printf(sb, "\n Total wait Total occupancy"); sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); i += 2; MPASS(i < nitems(tx_stats)); sbuf_printf(sb, "\n Reads Total wait"); sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], tx_cyc[i]); sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], rx_cyc[i]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_rdma_stats stats; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_rdma_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_tcp_stats v4, v6; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_tcp_stats(sc, &v4, &v6, 0); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, " IP IPv6\n"); sbuf_printf(sb, "OutRsts: %20u %20u\n", v4.tcp_out_rsts, v6.tcp_out_rsts); sbuf_printf(sb, "InSegs: %20ju %20ju\n", v4.tcp_in_segs, v6.tcp_in_segs); sbuf_printf(sb, "OutSegs: %20ju %20ju\n", v4.tcp_out_segs, v6.tcp_out_segs); sbuf_printf(sb, "RetransSegs: %20ju %20ju", v4.tcp_retrans_segs, v6.tcp_retrans_segs); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tids(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; uint32_t x, y; struct tid_info *t = &sc->tids; rc = 0; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (t->natids) { sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, t->atids_in_use); } if (t->nhpftids) { sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n", t->hpftid_base, t->hpftid_end, t->hpftids_in_use); } if (t->ntids) { bool hashen = false; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { hashen = true; if (chip_id(sc) <= CHELSIO_T5) { x = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; y = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; } else { x = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX); y = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE); } } mtx_unlock(&sc->reg_lock); if (rc != 0) goto done; sbuf_printf(sb, "TID range: "); if (hashen) { if (x) sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1); sbuf_printf(sb, "%u-%u", y, t->ntids - 1); } else { sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base + t->ntids - 1); } sbuf_printf(sb, ", in use: %u\n", atomic_load_acq_int(&t->tids_in_use)); } if (t->nstids) { sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, t->stid_base + t->nstids - 1, t->stids_in_use); } if (t->nftids) { sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base, t->ftid_end, t->ftids_in_use); } if (t->netids) { sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base, t->etid_base + t->netids - 1, t->etids_in_use); } mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { x = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4); y = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6); } mtx_unlock(&sc->reg_lock); if (rc != 0) goto done; sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", x, y); done: if (rc == 0) rc = sbuf_finish(sb); else (void)sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_err_stats stats; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_err_stats(sc, &stats, 0); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3\n"); sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", stats.mac_in_errs[0], stats.mac_in_errs[1], stats.mac_in_errs[2], stats.mac_in_errs[3]); sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", stats.hdr_in_errs[0], stats.hdr_in_errs[1], stats.hdr_in_errs[2], stats.hdr_in_errs[3]); sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", stats.tcp_in_errs[0], stats.tcp_in_errs[1], stats.tcp_in_errs[2], stats.tcp_in_errs[3]); sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); } else { sbuf_printf(sb, " channel 0 channel 1\n"); sbuf_printf(sb, "macInErrs: %10u %10u\n", stats.mac_in_errs[0], stats.mac_in_errs[1]); sbuf_printf(sb, "hdrInErrs: %10u %10u\n", stats.hdr_in_errs[0], stats.hdr_in_errs[1]); sbuf_printf(sb, "tcpInErrs: %10u %10u\n", stats.tcp_in_errs[0], stats.tcp_in_errs[1]); sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); } sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", stats.ofld_no_neigh, stats.ofld_cong_defer); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tnl_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; struct tp_tnl_stats stats; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_tp_get_tnl_stats(sc, &stats, 1); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3\n"); sbuf_printf(sb, "OutPkts: %10u %10u %10u %10u\n", stats.out_pkt[0], stats.out_pkt[1], stats.out_pkt[2], stats.out_pkt[3]); sbuf_printf(sb, "InPkts: %10u %10u %10u %10u", stats.in_pkt[0], stats.in_pkt[1], stats.in_pkt[2], stats.in_pkt[3]); } else { sbuf_printf(sb, " channel 0 channel 1\n"); sbuf_printf(sb, "OutPkts: %10u %10u\n", stats.out_pkt[0], stats.out_pkt[1]); sbuf_printf(sb, "InPkts: %10u %10u", stats.in_pkt[0], stats.in_pkt[1]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct tp_params *tpp = &sc->params.tp; u_int mask; int rc; mask = tpp->la_mask >> 16; rc = sysctl_handle_int(oidp, &mask, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (mask > 0xffff) return (EINVAL); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { tpp->la_mask = mask << 16; t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask); } mtx_unlock(&sc->reg_lock); return (rc); } struct field_desc { const char *name; u_int start; u_int width; }; static void field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) { char buf[32]; int line_size = 0; while (f->name) { uint64_t mask = (1ULL << f->width) - 1; int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, ((uintmax_t)v >> f->start) & mask); if (line_size + len >= 79) { line_size = 8; sbuf_printf(sb, "\n "); } sbuf_printf(sb, "%s ", buf); line_size += len + 1; f++; } sbuf_printf(sb, "\n"); } static const struct field_desc tp_la0[] = { { "RcfOpCodeOut", 60, 4 }, { "State", 56, 4 }, { "WcfState", 52, 4 }, { "RcfOpcSrcOut", 50, 2 }, { "CRxError", 49, 1 }, { "ERxError", 48, 1 }, { "SanityFailed", 47, 1 }, { "SpuriousMsg", 46, 1 }, { "FlushInputMsg", 45, 1 }, { "FlushInputCpl", 44, 1 }, { "RssUpBit", 43, 1 }, { "RssFilterHit", 42, 1 }, { "Tid", 32, 10 }, { "InitTcb", 31, 1 }, { "LineNumber", 24, 7 }, { "Emsg", 23, 1 }, { "EdataOut", 22, 1 }, { "Cmsg", 21, 1 }, { "CdataOut", 20, 1 }, { "EreadPdu", 19, 1 }, { "CreadPdu", 18, 1 }, { "TunnelPkt", 17, 1 }, { "RcfPeerFin", 16, 1 }, { "RcfReasonOut", 12, 4 }, { "TxCchannel", 10, 2 }, { "RcfTxChannel", 8, 2 }, { "RxEchannel", 6, 2 }, { "RcfRxChannel", 5, 1 }, { "RcfDataOutSrdy", 4, 1 }, { "RxDvld", 3, 1 }, { "RxOoDvld", 2, 1 }, { "RxCongestion", 1, 1 }, { "TxCongestion", 0, 1 }, { NULL } }; static const struct field_desc tp_la1[] = { { "CplCmdIn", 56, 8 }, { "CplCmdOut", 48, 8 }, { "ESynOut", 47, 1 }, { "EAckOut", 46, 1 }, { "EFinOut", 45, 1 }, { "ERstOut", 44, 1 }, { "SynIn", 43, 1 }, { "AckIn", 42, 1 }, { "FinIn", 41, 1 }, { "RstIn", 40, 1 }, { "DataIn", 39, 1 }, { "DataInVld", 38, 1 }, { "PadIn", 37, 1 }, { "RxBufEmpty", 36, 1 }, { "RxDdp", 35, 1 }, { "RxFbCongestion", 34, 1 }, { "TxFbCongestion", 33, 1 }, { "TxPktSumSrdy", 32, 1 }, { "RcfUlpType", 28, 4 }, { "Eread", 27, 1 }, { "Ebypass", 26, 1 }, { "Esave", 25, 1 }, { "Static0", 24, 1 }, { "Cread", 23, 1 }, { "Cbypass", 22, 1 }, { "Csave", 21, 1 }, { "CPktOut", 20, 1 }, { "RxPagePoolFull", 18, 2 }, { "RxLpbkPkt", 17, 1 }, { "TxLpbkPkt", 16, 1 }, { "RxVfValid", 15, 1 }, { "SynLearned", 14, 1 }, { "SetDelEntry", 13, 1 }, { "SetInvEntry", 12, 1 }, { "CpcmdDvld", 11, 1 }, { "CpcmdSave", 10, 1 }, { "RxPstructsFull", 8, 2 }, { "EpcmdDvld", 7, 1 }, { "EpcmdFlush", 6, 1 }, { "EpcmdTrimPrefix", 5, 1 }, { "EpcmdTrimPostfix", 4, 1 }, { "ERssIp4Pkt", 3, 1 }, { "ERssIp6Pkt", 2, 1 }, { "ERssTcpUdpPkt", 1, 1 }, { "ERssFceFipPkt", 0, 1 }, { NULL } }; static const struct field_desc tp_la2[] = { { "CplCmdIn", 56, 8 }, { "MpsVfVld", 55, 1 }, { "MpsPf", 52, 3 }, { "MpsVf", 44, 8 }, { "SynIn", 43, 1 }, { "AckIn", 42, 1 }, { "FinIn", 41, 1 }, { "RstIn", 40, 1 }, { "DataIn", 39, 1 }, { "DataInVld", 38, 1 }, { "PadIn", 37, 1 }, { "RxBufEmpty", 36, 1 }, { "RxDdp", 35, 1 }, { "RxFbCongestion", 34, 1 }, { "TxFbCongestion", 33, 1 }, { "TxPktSumSrdy", 32, 1 }, { "RcfUlpType", 28, 4 }, { "Eread", 27, 1 }, { "Ebypass", 26, 1 }, { "Esave", 25, 1 }, { "Static0", 24, 1 }, { "Cread", 23, 1 }, { "Cbypass", 22, 1 }, { "Csave", 21, 1 }, { "CPktOut", 20, 1 }, { "RxPagePoolFull", 18, 2 }, { "RxLpbkPkt", 17, 1 }, { "TxLpbkPkt", 16, 1 }, { "RxVfValid", 15, 1 }, { "SynLearned", 14, 1 }, { "SetDelEntry", 13, 1 }, { "SetInvEntry", 12, 1 }, { "CpcmdDvld", 11, 1 }, { "CpcmdSave", 10, 1 }, { "RxPstructsFull", 8, 2 }, { "EpcmdDvld", 7, 1 }, { "EpcmdFlush", 6, 1 }, { "EpcmdTrimPrefix", 5, 1 }, { "EpcmdTrimPostfix", 4, 1 }, { "ERssIp4Pkt", 3, 1 }, { "ERssIp6Pkt", 2, 1 }, { "ERssTcpUdpPkt", 1, 1 }, { "ERssFceFipPkt", 0, 1 }, { NULL } }; static void tp_la_show(struct sbuf *sb, uint64_t *p, int idx) { field_desc_show(sb, *p, tp_la0); } static void tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) { if (idx) sbuf_printf(sb, "\n"); field_desc_show(sb, p[0], tp_la0); if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) field_desc_show(sb, p[1], tp_la0); } static void tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) { if (idx) sbuf_printf(sb, "\n"); field_desc_show(sb, p[0], tp_la0); if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); } static int sysctl_tp_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; uint64_t *buf, *p; int rc; u_int i, inc; void (*show_func)(struct sbuf *, uint64_t *, int); rc = 0; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { t4_tp_read_la(sc, buf, NULL); switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { case 2: inc = 2; show_func = tp_la_show2; break; case 3: inc = 2; show_func = tp_la_show3; break; default: inc = 1; show_func = tp_la_show; } } mtx_unlock(&sc->reg_lock); if (rc != 0) goto done; p = buf; for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) (*show_func)(sb, p, i); rc = sbuf_finish(sb); done: sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_get_chan_txrate(sc, nrate, orate); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); if (sc->chip_params->nchan > 2) { sbuf_printf(sb, " channel 0 channel 1" " channel 2 channel 3\n"); sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", nrate[0], nrate[1], nrate[2], nrate[3]); sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", orate[0], orate[1], orate[2], orate[3]); } else { sbuf_printf(sb, " channel 0 channel 1\n"); sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", nrate[0], nrate[1]); sbuf_printf(sb, "Offload B/s: %10ju %10ju", orate[0], orate[1]); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; uint32_t *buf, *p; int rc, i; rc = 0; sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else t4_ulprx_read_la(sc, buf); mtx_unlock(&sc->reg_lock); if (rc != 0) goto done; p = buf; sbuf_printf(sb, " Pcmd Type Message" " Data"); for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); } rc = sbuf_finish(sb); done: sbuf_delete(sb); free(buf, M_CXGBE); return (rc); } static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; struct sbuf *sb; int rc; uint32_t cfg, s1, s2; MPASS(chip_id(sc) >= CHELSIO_T5); rc = 0; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { cfg = t4_read_reg(sc, A_SGE_STAT_CFG); s1 = t4_read_reg(sc, A_SGE_STAT_TOTAL); s2 = t4_read_reg(sc, A_SGE_STAT_MATCH); } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); if (G_STATSOURCE_T5(cfg) == 7) { int mode; mode = is_t5(sc) ? G_STATMODE(cfg) : G_T6_STATMODE(cfg); if (mode == 0) sbuf_printf(sb, "total %d, incomplete %d", s1, s2); else if (mode == 1) sbuf_printf(sb, "total %d, data overflow %d", s1, s2); else sbuf_printf(sb, "unknown mode %d", mode); } rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_cpus(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; enum cpu_sets op = arg2; cpuset_t cpuset; struct sbuf *sb; int i, rc; MPASS(op == LOCAL_CPUS || op == INTR_CPUS); CPU_ZERO(&cpuset); rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset); if (rc != 0) return (rc); sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); if (sb == NULL) return (ENOMEM); CPU_FOREACH(i) sbuf_printf(sb, "%d ", i); rc = sbuf_finish(sb); sbuf_delete(sb); return (rc); } static int sysctl_reset(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; u_int val; int rc; val = atomic_load_int(&sc->num_resets); rc = sysctl_handle_int(oidp, &val, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (val == 0) { /* Zero out the counter that tracks reset. */ atomic_store_int(&sc->num_resets, 0); return (0); } if (val != 1) return (EINVAL); /* 0 or 1 are the only legal values */ if (hw_off_limits(sc)) /* harmless race */ return (EALREADY); taskqueue_enqueue(reset_tq, &sc->reset_task); return (0); } #ifdef TCP_OFFLOAD static int sysctl_tls(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int i, j, v, rc; struct vi_info *vi; v = sc->tt.tls; rc = sysctl_handle_int(oidp, &v, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS)) return (ENOTSUP); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4stls"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else { sc->tt.tls = !!v; for_each_port(sc, i) { for_each_vi(sc->port[i], j, vi) { if (vi->flags & VI_INIT_DONE) t4_update_fl_bufsize(vi->ifp); } } } end_synchronized_op(sc, 0); return (rc); } static void unit_conv(char *buf, size_t len, u_int val, u_int factor) { u_int rem = val % factor; if (rem == 0) snprintf(buf, len, "%u", val / factor); else { while (rem % 10 == 0) rem /= 10; snprintf(buf, len, "%u.%u", val / factor, rem); } } static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; char buf[16]; u_int res, re; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) res = (u_int)-1; else res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); mtx_unlock(&sc->reg_lock); if (res == (u_int)-1) return (ENXIO); switch (arg2) { case 0: /* timer_tick */ re = G_TIMERRESOLUTION(res); break; case 1: /* TCP timestamp tick */ re = G_TIMESTAMPRESOLUTION(res); break; case 2: /* DACK tick */ re = G_DELAYEDACKRESOLUTION(res); break; default: return (EDOOFUS); } unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000); return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); } static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc; u_int dack_tmr, dack_re, v; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = 0; dack_re = G_DELAYEDACKRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); dack_tmr = t4_read_reg(sc, A_TP_DACK_TIMER); } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); v = ((cclk_ps << dack_re) / 1000000) * dack_tmr; return (sysctl_handle_int(oidp, &v, 0, req)); } static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, reg = arg2; u_int tre; u_long tp_tick_us, v; u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX || reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX || reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL || reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = 0; tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); tp_tick_us = (cclk_ps << tre) / 1000000; if (reg == A_TP_INIT_SRTT) v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg)); else v = tp_tick_us * t4_read_reg(sc, reg); } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); else return (sysctl_handle_long(oidp, &v, 0, req)); } /* * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is * passed to this function. */ static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, idx = arg2; u_int v; MPASS(idx >= 0 && idx <= 24); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = 0; v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf; } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); else return (sysctl_handle_int(oidp, &v, 0, req)); } static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS) { struct adapter *sc = arg1; int rc, idx = arg2; u_int shift, v, r; MPASS(idx >= 0 && idx < 16); r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3); shift = (idx & 3) << 3; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else { rc = 0; v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0; } mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); else return (sysctl_handle_int(oidp, &v, 0, req)); } static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int idx, rc, i; struct sge_ofld_rxq *ofld_rxq; uint8_t v; idx = vi->ofld_tmr_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < 0 || idx >= SGE_NTIMERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4otmr"); if (rc) return (rc); v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1); for_each_ofld_rxq(vi, i, ofld_rxq) { #ifdef atomic_store_rel_8 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); #else ofld_rxq->iq.intr_params = v; #endif } vi->ofld_tmr_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (0); } static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS) { struct vi_info *vi = arg1; struct adapter *sc = vi->adapter; int idx, rc; idx = vi->ofld_pktc_idx; rc = sysctl_handle_int(oidp, &idx, 0, req); if (rc != 0 || req->newptr == NULL) return (rc); if (idx < -1 || idx >= SGE_NCOUNTERS) return (EINVAL); rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, "t4opktc"); if (rc) return (rc); if (vi->flags & VI_INIT_DONE) rc = EBUSY; /* cannot be changed once the queues are created */ else vi->ofld_pktc_idx = idx; end_synchronized_op(sc, LOCK_HELD); return (rc); } #endif static int get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) { int rc; if (cntxt->cid > M_CTXTQID) return (EINVAL); if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) return (EINVAL); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } if (sc->flags & FW_OK) { rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); if (rc == 0) goto done; } /* * Read via firmware failed or wasn't even attempted. Read directly via * the backdoor. */ rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); done: end_synchronized_op(sc, 0); return (rc); } static int load_fw(struct adapter *sc, struct t4_data *fw) { int rc; uint8_t *fw_data; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } /* * The firmware, with the sole exception of the memory parity error * handler, runs from memory and not flash. It is almost always safe to * install a new firmware on a running system. Just set bit 1 in * hw.cxgbe.dflags or dev...dflags first. */ if (sc->flags & FULL_INIT_DONE && (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) { rc = EBUSY; goto done; } fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); rc = copyin(fw->data, fw_data, fw->len); if (rc == 0) rc = -t4_load_fw(sc, fw_data, fw->len); free(fw_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_cfg(struct adapter *sc, struct t4_data *cfg) { int rc; uint8_t *cfg_data = NULL; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } if (cfg->len == 0) { /* clear */ rc = -t4_load_cfg(sc, NULL, 0); goto done; } cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK); rc = copyin(cfg->data, cfg_data, cfg->len); if (rc == 0) rc = -t4_load_cfg(sc, cfg_data, cfg->len); free(cfg_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_boot(struct adapter *sc, struct t4_bootrom *br) { int rc; uint8_t *br_data = NULL; u_int offset; if (br->len > 1024 * 1024) return (EFBIG); if (br->pf_offset == 0) { /* pfidx */ if (br->pfidx_addr > 7) return (EINVAL); offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr, A_PCIE_PF_EXPROM_OFST))); } else if (br->pf_offset == 1) { /* offset */ offset = G_OFFSET(br->pfidx_addr); } else { return (EINVAL); } rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } if (br->len == 0) { /* clear */ rc = -t4_load_boot(sc, NULL, offset, 0); goto done; } br_data = malloc(br->len, M_CXGBE, M_WAITOK); rc = copyin(br->data, br_data, br->len); if (rc == 0) rc = -t4_load_boot(sc, br_data, offset, br->len); free(br_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int load_bootcfg(struct adapter *sc, struct t4_data *bc) { int rc; uint8_t *bc_data = NULL; rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf"); if (rc) return (rc); if (hw_off_limits(sc)) { rc = ENXIO; goto done; } if (bc->len == 0) { /* clear */ rc = -t4_load_bootcfg(sc, NULL, 0); goto done; } bc_data = malloc(bc->len, M_CXGBE, M_WAITOK); rc = copyin(bc->data, bc_data, bc->len); if (rc == 0) rc = -t4_load_bootcfg(sc, bc_data, bc->len); free(bc_data, M_CXGBE); done: end_synchronized_op(sc, 0); return (rc); } static int cudbg_dump(struct adapter *sc, struct t4_cudbg_dump *dump) { int rc; struct cudbg_init *cudbg; void *handle, *buf; /* buf is large, don't block if no memory is available */ buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO); if (buf == NULL) return (ENOMEM); handle = cudbg_alloc_handle(); if (handle == NULL) { rc = ENOMEM; goto done; } cudbg = cudbg_get_init(handle); cudbg->adap = sc; cudbg->print = (cudbg_print_cb)printf; #ifndef notyet device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n", __func__, dump->wr_flash, dump->len, dump->data); #endif if (dump->wr_flash) cudbg->use_flash = 1; MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap)); memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap)); rc = cudbg_collect(handle, buf, &dump->len); if (rc != 0) goto done; rc = copyout(buf, dump->data, dump->len); done: cudbg_free_handle(handle); free(buf, M_CXGBE); return (rc); } static void free_offload_policy(struct t4_offload_policy *op) { struct offload_rule *r; int i; if (op == NULL) return; r = &op->rule[0]; for (i = 0; i < op->nrules; i++, r++) { free(r->bpf_prog.bf_insns, M_CXGBE); } free(op->rule, M_CXGBE); free(op, M_CXGBE); } static int set_offload_policy(struct adapter *sc, struct t4_offload_policy *uop) { int i, rc, len; struct t4_offload_policy *op, *old; struct bpf_program *bf; const struct offload_settings *s; struct offload_rule *r; void *u; if (!is_offload(sc)) return (ENODEV); if (uop->nrules == 0) { /* Delete installed policies. */ op = NULL; goto set_policy; } else if (uop->nrules > 256) { /* arbitrary */ return (E2BIG); } /* Copy userspace offload policy to kernel */ op = malloc(sizeof(*op), M_CXGBE, M_ZERO | M_WAITOK); op->nrules = uop->nrules; len = op->nrules * sizeof(struct offload_rule); op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); rc = copyin(uop->rule, op->rule, len); if (rc) { free(op->rule, M_CXGBE); free(op, M_CXGBE); return (rc); } r = &op->rule[0]; for (i = 0; i < op->nrules; i++, r++) { /* Validate open_type */ if (r->open_type != OPEN_TYPE_LISTEN && r->open_type != OPEN_TYPE_ACTIVE && r->open_type != OPEN_TYPE_PASSIVE && r->open_type != OPEN_TYPE_DONTCARE) { error: /* * Rules 0 to i have malloc'd filters that need to be * freed. Rules i+1 to nrules have userspace pointers * and should be left alone. */ op->nrules = i; free_offload_policy(op); return (rc); } /* Validate settings */ s = &r->settings; if ((s->offload != 0 && s->offload != 1) || s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED || s->sched_class < -1 || s->sched_class >= sc->params.nsched_cls) { rc = EINVAL; goto error; } bf = &r->bpf_prog; u = bf->bf_insns; /* userspace ptr */ bf->bf_insns = NULL; if (bf->bf_len == 0) { /* legal, matches everything */ continue; } len = bf->bf_len * sizeof(*bf->bf_insns); bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK); rc = copyin(u, bf->bf_insns, len); if (rc != 0) goto error; if (!bpf_validate(bf->bf_insns, bf->bf_len)) { rc = EINVAL; goto error; } } set_policy: rw_wlock(&sc->policy_lock); old = sc->policy; sc->policy = op; rw_wunlock(&sc->policy_lock); free_offload_policy(old); return (0); } #define MAX_READ_BUF_SIZE (128 * 1024) static int read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) { uint32_t addr, remaining, n; uint32_t *buf; int rc; uint8_t *dst; mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else rc = validate_mem_range(sc, mr->addr, mr->len); mtx_unlock(&sc->reg_lock); if (rc != 0) return (rc); buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK); addr = mr->addr; remaining = mr->len; dst = (void *)mr->data; while (remaining) { n = min(remaining, MAX_READ_BUF_SIZE); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else read_via_memwin(sc, 2, addr, buf, n); mtx_unlock(&sc->reg_lock); if (rc != 0) break; rc = copyout(buf, dst, n); if (rc != 0) break; dst += n; remaining -= n; addr += n; } free(buf, M_CXGBE); return (rc); } #undef MAX_READ_BUF_SIZE static int read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) { int rc; if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) return (EINVAL); if (i2cd->len > sizeof(i2cd->data)) return (EFBIG); rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); if (rc) return (rc); if (hw_off_limits(sc)) rc = ENXIO; else rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, i2cd->offset, i2cd->len, &i2cd->data[0]); end_synchronized_op(sc, 0); return (rc); } static int clear_stats(struct adapter *sc, u_int port_id) { int i, v, chan_map; struct port_info *pi; struct vi_info *vi; struct sge_rxq *rxq; struct sge_txq *txq; struct sge_wrq *wrq; #if defined(TCP_OFFLOAD) || defined(RATELIMIT) struct sge_ofld_txq *ofld_txq; #endif #ifdef TCP_OFFLOAD struct sge_ofld_rxq *ofld_rxq; #endif if (port_id >= sc->params.nports) return (EINVAL); pi = sc->port[port_id]; if (pi == NULL) return (EIO); mtx_lock(&sc->reg_lock); if (!hw_off_limits(sc)) { /* MAC stats */ t4_clr_port_stats(sc, pi->tx_chan); if (is_t6(sc)) { if (pi->fcs_reg != -1) pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg); else pi->stats.rx_fcs_err = 0; } for_each_vi(pi, v, vi) { if (vi->flags & VI_INIT_DONE) t4_clr_vi_stats(sc, vi->vin); } chan_map = pi->rx_e_chan_map; v = 0; /* reuse */ while (chan_map) { i = ffs(chan_map) - 1; t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1, A_TP_MIB_TNL_CNG_DROP_0 + i); chan_map &= ~(1 << i); } } mtx_unlock(&sc->reg_lock); pi->tx_parse_error = 0; pi->tnl_cong_drops = 0; /* * Since this command accepts a port, clear stats for * all VIs on this port. */ for_each_vi(pi, v, vi) { if (vi->flags & VI_INIT_DONE) { for_each_rxq(vi, i, rxq) { #if defined(INET) || defined(INET6) rxq->lro.lro_queued = 0; rxq->lro.lro_flushed = 0; #endif rxq->rxcsum = 0; rxq->vlan_extraction = 0; rxq->vxlan_rxcsum = 0; rxq->fl.cl_allocated = 0; rxq->fl.cl_recycled = 0; rxq->fl.cl_fast_recycled = 0; } for_each_txq(vi, i, txq) { txq->txcsum = 0; txq->tso_wrs = 0; txq->vlan_insertion = 0; txq->imm_wrs = 0; txq->sgl_wrs = 0; txq->txpkt_wrs = 0; txq->txpkts0_wrs = 0; txq->txpkts1_wrs = 0; txq->txpkts0_pkts = 0; txq->txpkts1_pkts = 0; txq->txpkts_flush = 0; txq->raw_wrs = 0; txq->vxlan_tso_wrs = 0; txq->vxlan_txcsum = 0; txq->kern_tls_records = 0; txq->kern_tls_short = 0; txq->kern_tls_partial = 0; txq->kern_tls_full = 0; txq->kern_tls_octets = 0; txq->kern_tls_waste = 0; txq->kern_tls_options = 0; txq->kern_tls_header = 0; txq->kern_tls_fin = 0; txq->kern_tls_fin_short = 0; txq->kern_tls_cbc = 0; txq->kern_tls_gcm = 0; mp_ring_reset_stats(txq->r); } #if defined(TCP_OFFLOAD) || defined(RATELIMIT) for_each_ofld_txq(vi, i, ofld_txq) { ofld_txq->wrq.tx_wrs_direct = 0; ofld_txq->wrq.tx_wrs_copied = 0; counter_u64_zero(ofld_txq->tx_iscsi_pdus); counter_u64_zero(ofld_txq->tx_iscsi_octets); counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs); counter_u64_zero(ofld_txq->tx_aio_jobs); counter_u64_zero(ofld_txq->tx_aio_octets); counter_u64_zero(ofld_txq->tx_toe_tls_records); counter_u64_zero(ofld_txq->tx_toe_tls_octets); } #endif #ifdef TCP_OFFLOAD for_each_ofld_rxq(vi, i, ofld_rxq) { ofld_rxq->fl.cl_allocated = 0; ofld_rxq->fl.cl_recycled = 0; ofld_rxq->fl.cl_fast_recycled = 0; counter_u64_zero( ofld_rxq->rx_iscsi_ddp_setup_ok); counter_u64_zero( ofld_rxq->rx_iscsi_ddp_setup_error); ofld_rxq->rx_iscsi_ddp_pdus = 0; ofld_rxq->rx_iscsi_ddp_octets = 0; ofld_rxq->rx_iscsi_fl_pdus = 0; ofld_rxq->rx_iscsi_fl_octets = 0; ofld_rxq->rx_aio_ddp_jobs = 0; ofld_rxq->rx_aio_ddp_octets = 0; ofld_rxq->rx_toe_tls_records = 0; ofld_rxq->rx_toe_tls_octets = 0; ofld_rxq->rx_toe_ddp_octets = 0; counter_u64_zero(ofld_rxq->ddp_buffer_alloc); counter_u64_zero(ofld_rxq->ddp_buffer_reuse); counter_u64_zero(ofld_rxq->ddp_buffer_free); } #endif if (IS_MAIN_VI(vi)) { wrq = &sc->sge.ctrlq[pi->port_id]; wrq->tx_wrs_direct = 0; wrq->tx_wrs_copied = 0; } } } return (0); } static int hold_clip_addr(struct adapter *sc, struct t4_clip_addr *ca) { #ifdef INET6 struct in6_addr in6; bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr)); if (t4_get_clip_entry(sc, &in6, true) != NULL) return (0); else return (EIO); #else return (ENOTSUP); #endif } static int release_clip_addr(struct adapter *sc, struct t4_clip_addr *ca) { #ifdef INET6 struct in6_addr in6; bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr)); return (t4_release_clip_addr(sc, &in6)); #else return (ENOTSUP); #endif } int t4_os_find_pci_capability(struct adapter *sc, int cap) { int i; return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); } int t4_os_pci_save_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_save(dev, dinfo, 0); return (0); } int t4_os_pci_restore_state(struct adapter *sc) { device_t dev; struct pci_devinfo *dinfo; dev = sc->dev; dinfo = device_get_ivars(dev); pci_cfg_restore(dev, dinfo); return (0); } void t4_os_portmod_changed(struct port_info *pi) { struct adapter *sc = pi->adapter; struct vi_info *vi; if_t ifp; static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" }; KASSERT((pi->flags & FIXED_IFMEDIA) == 0, ("%s: port_type %u", __func__, pi->port_type)); vi = &pi->vi[0]; if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) { PORT_LOCK(pi); build_medialist(pi); if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) { fixup_link_config(pi); apply_link_config(pi); } PORT_UNLOCK(pi); end_synchronized_op(sc, LOCK_HELD); } ifp = vi->ifp; if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) if_printf(ifp, "transceiver unplugged.\n"); else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) if_printf(ifp, "unknown transceiver inserted.\n"); else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) if_printf(ifp, "unsupported transceiver inserted.\n"); else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { if_printf(ifp, "%dGbps %s transceiver inserted.\n", port_top_speed(pi), mod_str[pi->mod_type]); } else { if_printf(ifp, "transceiver (type %d) inserted.\n", pi->mod_type); } } void t4_os_link_changed(struct port_info *pi) { struct vi_info *vi; if_t ifp; struct link_config *lc = &pi->link_cfg; struct adapter *sc = pi->adapter; int v; PORT_LOCK_ASSERT_OWNED(pi); if (is_t6(sc)) { if (lc->link_ok) { if (lc->speed > 25000 || (lc->speed == 25000 && lc->fec == FEC_RS)) { pi->fcs_reg = T5_PORT_REG(pi->tx_chan, A_MAC_PORT_AFRAMECHECKSEQUENCEERRORS); } else { pi->fcs_reg = T5_PORT_REG(pi->tx_chan, A_MAC_PORT_MTIP_1G10G_RX_CRCERRORS); } pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg); pi->stats.rx_fcs_err = 0; } else { pi->fcs_reg = -1; } } else { MPASS(pi->fcs_reg != -1); MPASS(pi->fcs_base == 0); } for_each_vi(pi, v, vi) { ifp = vi->ifp; if (ifp == NULL || IS_DETACHING(vi)) continue; if (lc->link_ok) { if_setbaudrate(ifp, IF_Mbps(lc->speed)); if_link_state_change(ifp, LINK_STATE_UP); } else { if_link_state_change(ifp, LINK_STATE_DOWN); } } } void t4_iterate(void (*func)(struct adapter *, void *), void *arg) { struct adapter *sc; sx_slock(&t4_list_lock); SLIST_FOREACH(sc, &t4_list, link) { /* * func should not make any assumptions about what state sc is * in - the only guarantee is that sc->sc_lock is a valid lock. */ func(sc, arg); } sx_sunlock(&t4_list_lock); } static int t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { int rc; struct adapter *sc = dev->si_drv1; rc = priv_check(td, PRIV_DRIVER); if (rc != 0) return (rc); switch (cmd) { case CHELSIO_T4_GETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else if (edata->size == 4) edata->val = t4_read_reg(sc, edata->addr); else if (edata->size == 8) edata->val = t4_read_reg64(sc, edata->addr); else rc = EINVAL; mtx_unlock(&sc->reg_lock); break; } case CHELSIO_T4_SETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else if (edata->size == 4) { if (edata->val & 0xffffffff00000000) rc = EINVAL; t4_write_reg(sc, edata->addr, (uint32_t) edata->val); } else if (edata->size == 8) t4_write_reg64(sc, edata->addr, edata->val); else rc = EINVAL; mtx_unlock(&sc->reg_lock); break; } case CHELSIO_T4_REGDUMP: { struct t4_regdump *regs = (struct t4_regdump *)data; int reglen = t4_get_regs_len(sc); uint8_t *buf; if (regs->len < reglen) { regs->len = reglen; /* hint to the caller */ return (ENOBUFS); } regs->len = reglen; buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); mtx_lock(&sc->reg_lock); if (hw_off_limits(sc)) rc = ENXIO; else get_regs(sc, regs, buf); mtx_unlock(&sc->reg_lock); if (rc == 0) rc = copyout(buf, regs->data, reglen); free(buf, M_CXGBE); break; } case CHELSIO_T4_GET_FILTER_MODE: rc = get_filter_mode(sc, (uint32_t *)data); break; case CHELSIO_T4_SET_FILTER_MODE: rc = set_filter_mode(sc, *(uint32_t *)data); break; case CHELSIO_T4_SET_FILTER_MASK: rc = set_filter_mask(sc, *(uint32_t *)data); break; case CHELSIO_T4_GET_FILTER: rc = get_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_SET_FILTER: rc = set_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_DEL_FILTER: rc = del_filter(sc, (struct t4_filter *)data); break; case CHELSIO_T4_GET_SGE_CONTEXT: rc = get_sge_context(sc, (struct t4_sge_context *)data); break; case CHELSIO_T4_LOAD_FW: rc = load_fw(sc, (struct t4_data *)data); break; case CHELSIO_T4_GET_MEM: rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); break; case CHELSIO_T4_GET_I2C: rc = read_i2c(sc, (struct t4_i2c_data *)data); break; case CHELSIO_T4_CLEAR_STATS: rc = clear_stats(sc, *(uint32_t *)data); break; case CHELSIO_T4_SCHED_CLASS: rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); break; case CHELSIO_T4_SCHED_QUEUE: rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); break; case CHELSIO_T4_GET_TRACER: rc = t4_get_tracer(sc, (struct t4_tracer *)data); break; case CHELSIO_T4_SET_TRACER: rc = t4_set_tracer(sc, (struct t4_tracer *)data); break; case CHELSIO_T4_LOAD_CFG: rc = load_cfg(sc, (struct t4_data *)data); break; case CHELSIO_T4_LOAD_BOOT: rc = load_boot(sc, (struct t4_bootrom *)data); break; case CHELSIO_T4_LOAD_BOOTCFG: rc = load_bootcfg(sc, (struct t4_data *)data); break; case CHELSIO_T4_CUDBG_DUMP: rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data); break; case CHELSIO_T4_SET_OFLD_POLICY: rc = set_offload_policy(sc, (struct t4_offload_policy *)data); break; case CHELSIO_T4_HOLD_CLIP_ADDR: rc = hold_clip_addr(sc, (struct t4_clip_addr *)data); break; case CHELSIO_T4_RELEASE_CLIP_ADDR: rc = release_clip_addr(sc, (struct t4_clip_addr *)data); break; default: rc = ENOTTY; } return (rc); } #ifdef TCP_OFFLOAD int toe_capability(struct vi_info *vi, bool enable) { int rc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; ASSERT_SYNCHRONIZED_OP(sc); if (!is_offload(sc)) return (ENODEV); if (hw_off_limits(sc)) return (ENXIO); if (enable) { #ifdef KERN_TLS if (sc->flags & KERN_TLS_ON && is_t6(sc)) { int i, j, n; struct port_info *p; struct vi_info *v; /* * Reconfigure hardware for TOE if TXTLS is not enabled * on any ifnet. */ n = 0; for_each_port(sc, i) { p = sc->port[i]; for_each_vi(p, j, v) { if (if_getcapenable(v->ifp) & IFCAP_TXTLS) { CH_WARN(sc, "%s has NIC TLS enabled.\n", device_get_nameunit(v->dev)); n++; } } } if (n > 0) { CH_WARN(sc, "Disable NIC TLS on all interfaces " "associated with this adapter before " "trying to enable TOE.\n"); return (EAGAIN); } rc = t6_config_kern_tls(sc, false); if (rc) return (rc); } #endif if ((if_getcapenable(vi->ifp) & IFCAP_TOE) != 0) { /* TOE is already enabled. */ return (0); } /* * We need the port's queues around so that we're able to send * and receive CPLs to/from the TOE even if the ifnet for this * port has never been UP'd administratively. */ if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0)) return (rc); if (!(pi->vi[0].flags & VI_INIT_DONE) && ((rc = vi_init(&pi->vi[0])) != 0)) return (rc); if (isset(&sc->offload_map, pi->port_id)) { /* TOE is enabled on another VI of this port. */ MPASS(pi->uld_vis > 0); pi->uld_vis++; return (0); } if (!uld_active(sc, ULD_TOM)) { rc = t4_activate_uld(sc, ULD_TOM); if (rc == EAGAIN) { log(LOG_WARNING, "You must kldload t4_tom.ko before trying " "to enable TOE on a cxgbe interface.\n"); } if (rc != 0) return (rc); KASSERT(sc->tom_softc != NULL, ("%s: TOM activated but softc NULL", __func__)); KASSERT(uld_active(sc, ULD_TOM), ("%s: TOM activated but flag not set", __func__)); } /* Activate iWARP and iSCSI too, if the modules are loaded. */ if (!uld_active(sc, ULD_IWARP)) (void) t4_activate_uld(sc, ULD_IWARP); if (!uld_active(sc, ULD_ISCSI)) (void) t4_activate_uld(sc, ULD_ISCSI); if (pi->uld_vis++ == 0) setbit(&sc->offload_map, pi->port_id); } else { if ((if_getcapenable(vi->ifp) & IFCAP_TOE) == 0) { /* TOE is already disabled. */ return (0); } MPASS(isset(&sc->offload_map, pi->port_id)); MPASS(pi->uld_vis > 0); if (--pi->uld_vis == 0) clrbit(&sc->offload_map, pi->port_id); } return (0); } /* * Add an upper layer driver to the global list. */ int t4_register_uld(struct uld_info *ui, int id) { int rc; if (id < 0 || id > ULD_MAX) return (EINVAL); sx_xlock(&t4_uld_list_lock); if (t4_uld_list[id] != NULL) rc = EEXIST; else { t4_uld_list[id] = ui; rc = 0; } sx_xunlock(&t4_uld_list_lock); return (rc); } int t4_unregister_uld(struct uld_info *ui, int id) { if (id < 0 || id > ULD_MAX) return (EINVAL); sx_xlock(&t4_uld_list_lock); MPASS(t4_uld_list[id] == ui); t4_uld_list[id] = NULL; sx_xunlock(&t4_uld_list_lock); return (0); } int t4_activate_uld(struct adapter *sc, int id) { int rc; ASSERT_SYNCHRONIZED_OP(sc); if (id < 0 || id > ULD_MAX) return (EINVAL); /* Adapter needs to be initialized before any ULD can be activated. */ if (!(sc->flags & FULL_INIT_DONE)) { rc = adapter_init(sc); if (rc != 0) return (rc); } sx_slock(&t4_uld_list_lock); if (t4_uld_list[id] == NULL) rc = EAGAIN; /* load the KLD with this ULD and try again. */ else { rc = t4_uld_list[id]->uld_activate(sc); if (rc == 0) setbit(&sc->active_ulds, id); } sx_sunlock(&t4_uld_list_lock); return (rc); } int t4_deactivate_uld(struct adapter *sc, int id) { int rc; ASSERT_SYNCHRONIZED_OP(sc); if (id < 0 || id > ULD_MAX) return (EINVAL); sx_slock(&t4_uld_list_lock); if (t4_uld_list[id] == NULL) rc = ENXIO; else { rc = t4_uld_list[id]->uld_deactivate(sc); if (rc == 0) clrbit(&sc->active_ulds, id); } sx_sunlock(&t4_uld_list_lock); return (rc); } static int deactivate_all_uld(struct adapter *sc) { int i, rc; rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4detuld"); if (rc != 0) return (ENXIO); sx_slock(&t4_uld_list_lock); for (i = 0; i <= ULD_MAX; i++) { if (t4_uld_list[i] == NULL || !uld_active(sc, i)) continue; rc = t4_uld_list[i]->uld_deactivate(sc); if (rc != 0) break; clrbit(&sc->active_ulds, i); } sx_sunlock(&t4_uld_list_lock); end_synchronized_op(sc, 0); return (rc); } static void stop_all_uld(struct adapter *sc) { int i; if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4uldst") != 0) return; sx_slock(&t4_uld_list_lock); for (i = 0; i <= ULD_MAX; i++) { if (t4_uld_list[i] == NULL || !uld_active(sc, i) || t4_uld_list[i]->uld_stop == NULL) continue; (void) t4_uld_list[i]->uld_stop(sc); } sx_sunlock(&t4_uld_list_lock); end_synchronized_op(sc, 0); } static void restart_all_uld(struct adapter *sc) { int i; if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4uldre") != 0) return; sx_slock(&t4_uld_list_lock); for (i = 0; i <= ULD_MAX; i++) { if (t4_uld_list[i] == NULL || !uld_active(sc, i) || t4_uld_list[i]->uld_restart == NULL) continue; (void) t4_uld_list[i]->uld_restart(sc); } sx_sunlock(&t4_uld_list_lock); end_synchronized_op(sc, 0); } int uld_active(struct adapter *sc, int id) { MPASS(id >= 0 && id <= ULD_MAX); return (isset(&sc->active_ulds, id)); } #endif #ifdef KERN_TLS static int ktls_capability(struct adapter *sc, bool enable) { ASSERT_SYNCHRONIZED_OP(sc); if (!is_ktls(sc)) return (ENODEV); if (!is_t6(sc)) return (0); if (hw_off_limits(sc)) return (ENXIO); if (enable) { if (sc->flags & KERN_TLS_ON) return (0); /* already on */ if (sc->offload_map != 0) { CH_WARN(sc, "Disable TOE on all interfaces associated with " "this adapter before trying to enable NIC TLS.\n"); return (EAGAIN); } return (t6_config_kern_tls(sc, true)); } else { /* * Nothing to do for disable. If TOE is enabled sometime later * then toe_capability will reconfigure the hardware. */ return (0); } } #endif /* * t = ptr to tunable. * nc = number of CPUs. * c = compiled in default for that tunable. */ static void calculate_nqueues(int *t, int nc, const int c) { int nq; if (*t > 0) return; nq = *t < 0 ? -*t : c; *t = min(nc, nq); } /* * Come up with reasonable defaults for some of the tunables, provided they're * not set by the user (in which case we'll use the values as is). */ static void tweak_tunables(void) { int nc = mp_ncpus; /* our snapshot of the number of CPUs */ if (t4_ntxq < 1) { #ifdef RSS t4_ntxq = rss_getnumbuckets(); #else calculate_nqueues(&t4_ntxq, nc, NTXQ); #endif } calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI); if (t4_nrxq < 1) { #ifdef RSS t4_nrxq = rss_getnumbuckets(); #else calculate_nqueues(&t4_nrxq, nc, NRXQ); #endif } calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI); #if defined(TCP_OFFLOAD) || defined(RATELIMIT) calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ); calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI); #endif #ifdef TCP_OFFLOAD calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ); calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI); #endif #if defined(TCP_OFFLOAD) || defined(KERN_TLS) if (t4_toecaps_allowed == -1) t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; #else if (t4_toecaps_allowed == -1) t4_toecaps_allowed = 0; #endif #ifdef TCP_OFFLOAD if (t4_rdmacaps_allowed == -1) { t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | FW_CAPS_CONFIG_RDMA_RDMAC; } if (t4_iscsicaps_allowed == -1) { t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | FW_CAPS_CONFIG_ISCSI_TARGET_PDU | FW_CAPS_CONFIG_ISCSI_T10DIF; } if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS) t4_tmr_idx_ofld = TMR_IDX_OFLD; if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS) t4_pktc_idx_ofld = PKTC_IDX_OFLD; #else if (t4_rdmacaps_allowed == -1) t4_rdmacaps_allowed = 0; if (t4_iscsicaps_allowed == -1) t4_iscsicaps_allowed = 0; #endif #ifdef DEV_NETMAP calculate_nqueues(&t4_nnmtxq, nc, NNMTXQ); calculate_nqueues(&t4_nnmrxq, nc, NNMRXQ); calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI); calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI); #endif if (t4_tmr_idx < 0 || t4_tmr_idx >= SGE_NTIMERS) t4_tmr_idx = TMR_IDX; if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS) t4_pktc_idx = PKTC_IDX; if (t4_qsize_txq < 128) t4_qsize_txq = 128; if (t4_qsize_rxq < 128) t4_qsize_rxq = 128; while (t4_qsize_rxq & 7) t4_qsize_rxq++; t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; /* * Number of VIs to create per-port. The first VI is the "main" regular * VI for the port. The rest are additional virtual interfaces on the * same physical port. Note that the main VI does not have native * netmap support but the extra VIs do. * * Limit the number of VIs per port to the number of available * MAC addresses per port. */ if (t4_num_vis < 1) t4_num_vis = 1; if (t4_num_vis > nitems(vi_mac_funcs)) { t4_num_vis = nitems(vi_mac_funcs); printf("cxgbe: number of VIs limited to %d\n", t4_num_vis); } if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) { pcie_relaxed_ordering = 1; #if defined(__i386__) || defined(__amd64__) if (cpu_vendor_id == CPU_VENDOR_INTEL) pcie_relaxed_ordering = 0; #endif } } #ifdef DDB static void t4_dump_mem(struct adapter *sc, u_int addr, u_int len) { uint32_t base, j, off, pf, reg, save, win_pos; reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2); save = t4_read_reg(sc, reg); base = sc->memwin[2].mw_base; if (is_t4(sc)) { pf = 0; win_pos = addr & ~0xf; /* start must be 16B aligned */ } else { pf = V_PFNUM(sc->pf); win_pos = addr & ~0x7f; /* start must be 128B aligned */ } off = addr - win_pos; t4_write_reg(sc, reg, win_pos | pf); t4_read_reg(sc, reg); while (len > 0 && !db_pager_quit) { uint32_t buf[8]; for (j = 0; j < 8; j++, off += 4) buf[j] = htonl(t4_read_reg(sc, base + off)); db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7]); if (len <= sizeof(buf)) len = 0; else len -= sizeof(buf); } t4_write_reg(sc, reg, save); t4_read_reg(sc, reg); } static void t4_dump_tcb(struct adapter *sc, int tid) { uint32_t tcb_addr; /* Dump TCB for the tid */ tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE); tcb_addr += tid * TCB_SIZE; t4_dump_mem(sc, tcb_addr, TCB_SIZE); } static void t4_dump_devlog(struct adapter *sc) { struct devlog_params *dparams = &sc->params.devlog; struct fw_devlog_e e; int i, first, j, m, nentries, rc; uint64_t ftstamp = UINT64_MAX; if (dparams->start == 0) { db_printf("devlog params not valid\n"); return; } nentries = dparams->size / sizeof(struct fw_devlog_e); m = fwmtype_to_hwmtype(dparams->memtype); /* Find the first entry. */ first = -1; for (i = 0; i < nentries && !db_pager_quit; i++) { rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), sizeof(e), (void *)&e); if (rc != 0) break; if (e.timestamp == 0) break; e.timestamp = be64toh(e.timestamp); if (e.timestamp < ftstamp) { ftstamp = e.timestamp; first = i; } } if (first == -1) return; i = first; do { rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), sizeof(e), (void *)&e); if (rc != 0) return; if (e.timestamp == 0) return; e.timestamp = be64toh(e.timestamp); e.seqno = be32toh(e.seqno); for (j = 0; j < 8; j++) e.params[j] = be32toh(e.params[j]); db_printf("%10d %15ju %8s %8s ", e.seqno, e.timestamp, (e.level < nitems(devlog_level_strings) ? devlog_level_strings[e.level] : "UNKNOWN"), (e.facility < nitems(devlog_facility_strings) ? devlog_facility_strings[e.facility] : "UNKNOWN")); db_printf(e.fmt, e.params[0], e.params[1], e.params[2], e.params[3], e.params[4], e.params[5], e.params[6], e.params[7]); if (++i == nentries) i = 0; } while (i != first && !db_pager_quit); } static DB_DEFINE_TABLE(show, t4, show_t4); DB_TABLE_COMMAND_FLAGS(show_t4, devlog, db_show_devlog, CS_OWN) { device_t dev; int t; bool valid; valid = false; t = db_read_token(); if (t == tIDENT) { dev = device_lookup_by_name(db_tok_string); valid = true; } db_skip_to_eol(); if (!valid) { db_printf("usage: show t4 devlog \n"); return; } if (dev == NULL) { db_printf("device not found\n"); return; } t4_dump_devlog(device_get_softc(dev)); } DB_TABLE_COMMAND_FLAGS(show_t4, tcb, db_show_t4tcb, CS_OWN) { device_t dev; int radix, tid, t; bool valid; valid = false; radix = db_radix; db_radix = 10; t = db_read_token(); if (t == tIDENT) { dev = device_lookup_by_name(db_tok_string); t = db_read_token(); if (t == tNUMBER) { tid = db_tok_number; valid = true; } } db_radix = radix; db_skip_to_eol(); if (!valid) { db_printf("usage: show t4 tcb \n"); return; } if (dev == NULL) { db_printf("device not found\n"); return; } if (tid < 0) { db_printf("invalid tid\n"); return; } t4_dump_tcb(device_get_softc(dev), tid); } DB_TABLE_COMMAND_FLAGS(show_t4, memdump, db_show_memdump, CS_OWN) { device_t dev; int radix, t; bool valid; valid = false; radix = db_radix; db_radix = 10; t = db_read_token(); if (t == tIDENT) { dev = device_lookup_by_name(db_tok_string); t = db_read_token(); if (t == tNUMBER) { addr = db_tok_number; t = db_read_token(); if (t == tNUMBER) { count = db_tok_number; valid = true; } } } db_radix = radix; db_skip_to_eol(); if (!valid) { db_printf("usage: show t4 memdump \n"); return; } if (dev == NULL) { db_printf("device not found\n"); return; } if (addr < 0) { db_printf("invalid address\n"); return; } if (count <= 0) { db_printf("invalid length\n"); return; } t4_dump_mem(device_get_softc(dev), addr, count); } #endif static eventhandler_tag vxlan_start_evtag; static eventhandler_tag vxlan_stop_evtag; struct vxlan_evargs { if_t ifp; uint16_t port; }; static void enable_vxlan_rx(struct adapter *sc) { int i, rc; struct port_info *pi; uint8_t match_all_mac[ETHER_ADDR_LEN] = {0}; ASSERT_SYNCHRONIZED_OP(sc); t4_write_reg(sc, A_MPS_RX_VXLAN_TYPE, V_VXLAN(sc->vxlan_port) | F_VXLAN_EN); for_each_port(sc, i) { pi = sc->port[i]; if (pi->vxlan_tcam_entry == true) continue; rc = t4_alloc_raw_mac_filt(sc, pi->vi[0].viid, match_all_mac, match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id, true); if (rc < 0) { rc = -rc; CH_ERR(&pi->vi[0], "failed to add VXLAN TCAM entry: %d.\n", rc); } else { MPASS(rc == sc->rawf_base + pi->port_id); pi->vxlan_tcam_entry = true; } } } static void t4_vxlan_start(struct adapter *sc, void *arg) { struct vxlan_evargs *v = arg; if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5) return; if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxst") != 0) return; if (sc->vxlan_refcount == 0) { sc->vxlan_port = v->port; sc->vxlan_refcount = 1; if (!hw_off_limits(sc)) enable_vxlan_rx(sc); } else if (sc->vxlan_port == v->port) { sc->vxlan_refcount++; } else { CH_ERR(sc, "VXLAN already configured on port %d; " "ignoring attempt to configure it on port %d\n", sc->vxlan_port, v->port); } end_synchronized_op(sc, 0); } static void t4_vxlan_stop(struct adapter *sc, void *arg) { struct vxlan_evargs *v = arg; if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5) return; if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxsp") != 0) return; /* * VXLANs may have been configured before the driver was loaded so we * may see more stops than starts. This is not handled cleanly but at * least we keep the refcount sane. */ if (sc->vxlan_port != v->port) goto done; if (sc->vxlan_refcount == 0) { CH_ERR(sc, "VXLAN operation on port %d was stopped earlier; " "ignoring attempt to stop it again.\n", sc->vxlan_port); } else if (--sc->vxlan_refcount == 0 && !hw_off_limits(sc)) t4_set_reg_field(sc, A_MPS_RX_VXLAN_TYPE, F_VXLAN_EN, 0); done: end_synchronized_op(sc, 0); } static void t4_vxlan_start_handler(void *arg __unused, if_t ifp, sa_family_t family, u_int port) { struct vxlan_evargs v; MPASS(family == AF_INET || family == AF_INET6); v.ifp = ifp; v.port = port; t4_iterate(t4_vxlan_start, &v); } static void t4_vxlan_stop_handler(void *arg __unused, if_t ifp, sa_family_t family, u_int port) { struct vxlan_evargs v; MPASS(family == AF_INET || family == AF_INET6); v.ifp = ifp; v.port = port; t4_iterate(t4_vxlan_stop, &v); } static struct sx mlu; /* mod load unload */ SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); static int mod_event(module_t mod, int cmd, void *arg) { int rc = 0; static int loaded = 0; switch (cmd) { case MOD_LOAD: sx_xlock(&mlu); if (loaded++ == 0) { t4_sge_modload(); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, t4_filter_rpl, CPL_COOKIE_FILTER); t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl, CPL_COOKIE_FILTER); t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER); t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER); t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt); t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt); t4_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); sx_init(&t4_list_lock, "T4/T5 adapters"); SLIST_INIT(&t4_list); callout_init(&fatal_callout, 1); #ifdef TCP_OFFLOAD sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); #endif #ifdef INET6 t4_clip_modload(); #endif #ifdef KERN_TLS t6_ktls_modload(); #endif t4_tracer_modload(); tweak_tunables(); vxlan_start_evtag = EVENTHANDLER_REGISTER(vxlan_start, t4_vxlan_start_handler, NULL, EVENTHANDLER_PRI_ANY); vxlan_stop_evtag = EVENTHANDLER_REGISTER(vxlan_stop, t4_vxlan_stop_handler, NULL, EVENTHANDLER_PRI_ANY); reset_tq = taskqueue_create("t4_rst_tq", M_WAITOK, taskqueue_thread_enqueue, &reset_tq); taskqueue_start_threads(&reset_tq, 1, PI_SOFT, "t4_rst_thr"); } sx_xunlock(&mlu); break; case MOD_UNLOAD: sx_xlock(&mlu); if (--loaded == 0) { #ifdef TCP_OFFLOAD int i; #endif int tries; taskqueue_free(reset_tq); tries = 0; while (tries++ < 5 && t4_sge_extfree_refs() != 0) { uprintf("%ju clusters with custom free routine " "still is use.\n", t4_sge_extfree_refs()); pause("t4unload", 2 * hz); } sx_slock(&t4_list_lock); if (!SLIST_EMPTY(&t4_list)) { rc = EBUSY; sx_sunlock(&t4_list_lock); goto done_unload; } #ifdef TCP_OFFLOAD sx_slock(&t4_uld_list_lock); for (i = 0; i <= ULD_MAX; i++) { if (t4_uld_list[i] != NULL) { rc = EBUSY; sx_sunlock(&t4_uld_list_lock); sx_sunlock(&t4_list_lock); goto done_unload; } } sx_sunlock(&t4_uld_list_lock); #endif sx_sunlock(&t4_list_lock); if (t4_sge_extfree_refs() == 0) { EVENTHANDLER_DEREGISTER(vxlan_start, vxlan_start_evtag); EVENTHANDLER_DEREGISTER(vxlan_stop, vxlan_stop_evtag); t4_tracer_modunload(); #ifdef KERN_TLS t6_ktls_modunload(); #endif #ifdef INET6 t4_clip_modunload(); #endif #ifdef TCP_OFFLOAD sx_destroy(&t4_uld_list_lock); #endif sx_destroy(&t4_list_lock); t4_sge_modunload(); loaded = 0; } else { rc = EBUSY; loaded++; /* undo earlier decrement */ } } done_unload: sx_xunlock(&mlu); break; } return (rc); } DRIVER_MODULE(t4nex, pci, t4_driver, mod_event, 0); MODULE_VERSION(t4nex, 1); MODULE_DEPEND(t4nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t4nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(t5nex, pci, t5_driver, mod_event, 0); MODULE_VERSION(t5nex, 1); MODULE_DEPEND(t5nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t5nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(t6nex, pci, t6_driver, mod_event, 0); MODULE_VERSION(t6nex, 1); MODULE_DEPEND(t6nex, crypto, 1, 1, 1); MODULE_DEPEND(t6nex, firmware, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(t6nex, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, 0, 0); MODULE_VERSION(cxgbe, 1); DRIVER_MODULE(cxl, t5nex, cxl_driver, 0, 0); MODULE_VERSION(cxl, 1); DRIVER_MODULE(cc, t6nex, cc_driver, 0, 0); MODULE_VERSION(cc, 1); DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, 0, 0); MODULE_VERSION(vcxgbe, 1); DRIVER_MODULE(vcxl, cxl, vcxl_driver, 0, 0); MODULE_VERSION(vcxl, 1); DRIVER_MODULE(vcc, cc, vcc_driver, 0, 0); MODULE_VERSION(vcc, 1); diff --git a/sys/dev/dpaa2/dpaa2_mc.c b/sys/dev/dpaa2/dpaa2_mc.c index c84b22d1d74d..8abfc3bfe1cc 100644 --- a/sys/dev/dpaa2/dpaa2_mc.c +++ b/sys/dev/dpaa2/dpaa2_mc.c @@ -1,880 +1,880 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright © 2021-2022 Dmitry Salychev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * The DPAA2 Management Complex (MC) bus driver. * * MC is a hardware resource manager which can be found in several NXP * SoCs (LX2160A, for example) and provides an access to the specialized * hardware objects used in network-oriented packet processing applications. */ #include "opt_acpi.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #include #include #endif #include "pcib_if.h" #include "pci_if.h" #include "dpaa2_mc.h" /* Macros to read/write MC registers */ #define mcreg_read_4(_sc, _r) bus_read_4(&(_sc)->map[1], (_r)) #define mcreg_write_4(_sc, _r, _v) bus_write_4(&(_sc)->map[1], (_r), (_v)) #define IORT_DEVICE_NAME "MCE" /* MC Registers */ #define MC_REG_GCR1 0x0000u #define MC_REG_GCR2 0x0004u /* TODO: Does it exist? */ #define MC_REG_GSR 0x0008u #define MC_REG_FAPR 0x0028u /* General Control Register 1 (GCR1) */ #define GCR1_P1_STOP 0x80000000u #define GCR1_P2_STOP 0x40000000u /* General Status Register (GSR) */ #define GSR_HW_ERR(v) (((v) & 0x80000000u) >> 31) #define GSR_CAT_ERR(v) (((v) & 0x40000000u) >> 30) #define GSR_DPL_OFFSET(v) (((v) & 0x3FFFFF00u) >> 8) #define GSR_MCS(v) (((v) & 0xFFu) >> 0) /* Timeouts to wait for the MC status. */ #define MC_STAT_TIMEOUT 1000u /* us */ #define MC_STAT_ATTEMPTS 100u /** * @brief Structure to describe a DPAA2 device as a managed resource. */ struct dpaa2_mc_devinfo { STAILQ_ENTRY(dpaa2_mc_devinfo) link; device_t dpaa2_dev; uint32_t flags; uint32_t owners; }; MALLOC_DEFINE(M_DPAA2_MC, "dpaa2_mc", "DPAA2 Management Complex"); static struct resource_spec dpaa2_mc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE | RF_UNMAPPED }, { SYS_RES_MEMORY, 1, RF_ACTIVE | RF_UNMAPPED | RF_OPTIONAL }, RESOURCE_SPEC_END }; static u_int dpaa2_mc_get_xref(device_t, device_t); static u_int dpaa2_mc_map_id(device_t, device_t, uintptr_t *); static int dpaa2_mc_alloc_msi_impl(device_t, device_t, int, int, int *); static int dpaa2_mc_release_msi_impl(device_t, device_t, int, int *); static int dpaa2_mc_map_msi_impl(device_t, device_t, int, uint64_t *, uint32_t *); /* * For device interface. */ int dpaa2_mc_attach(device_t dev) { struct dpaa2_mc_softc *sc; struct resource_map_request req; uint32_t val; int error; sc = device_get_softc(dev); sc->dev = dev; sc->msi_allocated = false; sc->msi_owner = NULL; error = bus_alloc_resources(sc->dev, dpaa2_mc_spec, sc->res); if (error) { device_printf(dev, "%s: failed to allocate resources\n", __func__); return (ENXIO); } if (sc->res[1]) { resource_init_map_request(&req); req.memattr = VM_MEMATTR_DEVICE; error = bus_map_resource(sc->dev, SYS_RES_MEMORY, sc->res[1], &req, &sc->map[1]); if (error) { device_printf(dev, "%s: failed to map control " "registers\n", __func__); dpaa2_mc_detach(dev); return (ENXIO); } if (bootverbose) device_printf(dev, "GCR1=0x%x, GCR2=0x%x, GSR=0x%x, FAPR=0x%x\n", mcreg_read_4(sc, MC_REG_GCR1), mcreg_read_4(sc, MC_REG_GCR2), mcreg_read_4(sc, MC_REG_GSR), mcreg_read_4(sc, MC_REG_FAPR)); /* Reset P1_STOP and P2_STOP bits to resume MC processor. */ val = mcreg_read_4(sc, MC_REG_GCR1) & ~(GCR1_P1_STOP | GCR1_P2_STOP); mcreg_write_4(sc, MC_REG_GCR1, val); /* Poll MC status. */ if (bootverbose) device_printf(dev, "polling MC status...\n"); for (int i = 0; i < MC_STAT_ATTEMPTS; i++) { val = mcreg_read_4(sc, MC_REG_GSR); if (GSR_MCS(val) != 0u) break; DELAY(MC_STAT_TIMEOUT); } if (bootverbose) device_printf(dev, "GCR1=0x%x, GCR2=0x%x, GSR=0x%x, FAPR=0x%x\n", mcreg_read_4(sc, MC_REG_GCR1), mcreg_read_4(sc, MC_REG_GCR2), mcreg_read_4(sc, MC_REG_GSR), mcreg_read_4(sc, MC_REG_FAPR)); } /* At least 64 bytes of the command portal should be available. */ if (rman_get_size(sc->res[0]) < DPAA2_MCP_MEM_WIDTH) { device_printf(dev, "%s: MC portal memory region too small: " "%jd\n", __func__, rman_get_size(sc->res[0])); dpaa2_mc_detach(dev); return (ENXIO); } /* Map MC portal memory resource. */ resource_init_map_request(&req); req.memattr = VM_MEMATTR_DEVICE; error = bus_map_resource(sc->dev, SYS_RES_MEMORY, sc->res[0], &req, &sc->map[0]); if (error) { device_printf(dev, "Failed to map MC portal memory\n"); dpaa2_mc_detach(dev); return (ENXIO); } /* Initialize a resource manager for the DPAA2 I/O objects. */ sc->dpio_rman.rm_type = RMAN_ARRAY; sc->dpio_rman.rm_descr = "DPAA2 DPIO objects"; error = rman_init(&sc->dpio_rman); if (error) { device_printf(dev, "Failed to initialize a resource manager for " "the DPAA2 I/O objects: error=%d\n", error); dpaa2_mc_detach(dev); return (ENXIO); } /* Initialize a resource manager for the DPAA2 buffer pools. */ sc->dpbp_rman.rm_type = RMAN_ARRAY; sc->dpbp_rman.rm_descr = "DPAA2 DPBP objects"; error = rman_init(&sc->dpbp_rman); if (error) { device_printf(dev, "Failed to initialize a resource manager for " "the DPAA2 buffer pools: error=%d\n", error); dpaa2_mc_detach(dev); return (ENXIO); } /* Initialize a resource manager for the DPAA2 concentrators. */ sc->dpcon_rman.rm_type = RMAN_ARRAY; sc->dpcon_rman.rm_descr = "DPAA2 DPCON objects"; error = rman_init(&sc->dpcon_rman); if (error) { device_printf(dev, "Failed to initialize a resource manager for " "the DPAA2 concentrators: error=%d\n", error); dpaa2_mc_detach(dev); return (ENXIO); } /* Initialize a resource manager for the DPAA2 MC portals. */ sc->dpmcp_rman.rm_type = RMAN_ARRAY; sc->dpmcp_rman.rm_descr = "DPAA2 DPMCP objects"; error = rman_init(&sc->dpmcp_rman); if (error) { device_printf(dev, "Failed to initialize a resource manager for " "the DPAA2 MC portals: error=%d\n", error); dpaa2_mc_detach(dev); return (ENXIO); } /* Initialize a list of non-allocatable DPAA2 devices. */ mtx_init(&sc->mdev_lock, "MC portal mdev lock", NULL, MTX_DEF); STAILQ_INIT(&sc->mdev_list); mtx_init(&sc->msi_lock, "MC MSI lock", NULL, MTX_DEF); /* * Add a root resource container as the only child of the bus. All of * the direct descendant containers will be attached to the root one * instead of the MC device. */ sc->rcdev = device_add_child(dev, "dpaa2_rc", 0); if (sc->rcdev == NULL) { dpaa2_mc_detach(dev); return (ENXIO); } bus_identify_children(dev); bus_attach_children(dev); return (0); } int dpaa2_mc_detach(device_t dev) { struct dpaa2_mc_softc *sc; struct dpaa2_devinfo *dinfo = NULL; int error; error = bus_generic_detach(dev); if (error != 0) return (error); sc = device_get_softc(dev); bus_release_resources(dev, dpaa2_mc_spec, sc->res); dinfo = device_get_ivars(dev); if (dinfo) free(dinfo, M_DPAA2_MC); - return (device_delete_children(dev)); + return (0); } /* * For bus interface. */ struct resource * dpaa2_mc_alloc_resource(device_t mcdev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; struct rman *rm; int error; rm = dpaa2_mc_rman(mcdev, type, flags); if (rm == NULL) return (bus_generic_alloc_resource(mcdev, child, type, rid, start, end, count, flags)); /* * Skip managing DPAA2-specific resource. It must be provided to MC by * calling DPAA2_MC_MANAGE_DEV() beforehand. */ if (type <= DPAA2_DEV_MC) { error = rman_manage_region(rm, start, end); if (error) { device_printf(mcdev, "rman_manage_region() failed: " "start=%#jx, end=%#jx, error=%d\n", start, end, error); goto fail; } } res = bus_generic_rman_alloc_resource(mcdev, child, type, rid, start, end, count, flags); if (res == NULL) goto fail; return (res); fail: device_printf(mcdev, "%s() failed: type=%d, rid=%d, start=%#jx, " "end=%#jx, count=%#jx, flags=%x\n", __func__, type, *rid, start, end, count, flags); return (NULL); } int dpaa2_mc_adjust_resource(device_t mcdev, device_t child, struct resource *r, rman_res_t start, rman_res_t end) { struct rman *rm; rm = dpaa2_mc_rman(mcdev, rman_get_type(r), rman_get_flags(r)); if (rm) return (bus_generic_rman_adjust_resource(mcdev, child, r, start, end)); return (bus_generic_adjust_resource(mcdev, child, r, start, end)); } int dpaa2_mc_release_resource(device_t mcdev, device_t child, struct resource *r) { struct rman *rm; rm = dpaa2_mc_rman(mcdev, rman_get_type(r), rman_get_flags(r)); if (rm) return (bus_generic_rman_release_resource(mcdev, child, r)); return (bus_generic_release_resource(mcdev, child, r)); } int dpaa2_mc_activate_resource(device_t mcdev, device_t child, struct resource *r) { struct rman *rm; rm = dpaa2_mc_rman(mcdev, rman_get_type(r), rman_get_flags(r)); if (rm) return (bus_generic_rman_activate_resource(mcdev, child, r)); return (bus_generic_activate_resource(mcdev, child, r)); } int dpaa2_mc_deactivate_resource(device_t mcdev, device_t child, struct resource *r) { struct rman *rm; rm = dpaa2_mc_rman(mcdev, rman_get_type(r), rman_get_flags(r)); if (rm) return (bus_generic_rman_deactivate_resource(mcdev, child, r)); return (bus_generic_deactivate_resource(mcdev, child, r)); } /* * For pseudo-pcib interface. */ int dpaa2_mc_alloc_msi(device_t mcdev, device_t child, int count, int maxcount, int *irqs) { #if defined(INTRNG) return (dpaa2_mc_alloc_msi_impl(mcdev, child, count, maxcount, irqs)); #else return (ENXIO); #endif } int dpaa2_mc_release_msi(device_t mcdev, device_t child, int count, int *irqs) { #if defined(INTRNG) return (dpaa2_mc_release_msi_impl(mcdev, child, count, irqs)); #else return (ENXIO); #endif } int dpaa2_mc_map_msi(device_t mcdev, device_t child, int irq, uint64_t *addr, uint32_t *data) { #if defined(INTRNG) return (dpaa2_mc_map_msi_impl(mcdev, child, irq, addr, data)); #else return (ENXIO); #endif } int dpaa2_mc_get_id(device_t mcdev, device_t child, enum pci_id_type type, uintptr_t *id) { struct dpaa2_devinfo *dinfo; dinfo = device_get_ivars(child); if (strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (ENXIO); if (type == PCI_ID_MSI) return (dpaa2_mc_map_id(mcdev, child, id)); *id = dinfo->icid; return (0); } /* * For DPAA2 Management Complex bus driver interface. */ int dpaa2_mc_manage_dev(device_t mcdev, device_t dpaa2_dev, uint32_t flags) { struct dpaa2_mc_softc *sc; struct dpaa2_devinfo *dinfo; struct dpaa2_mc_devinfo *di; struct rman *rm; int error; sc = device_get_softc(mcdev); dinfo = device_get_ivars(dpaa2_dev); if (!sc || !dinfo || strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); di = malloc(sizeof(*di), M_DPAA2_MC, M_WAITOK | M_ZERO); di->dpaa2_dev = dpaa2_dev; di->flags = flags; di->owners = 0; /* Append a new managed DPAA2 device to the queue. */ mtx_assert(&sc->mdev_lock, MA_NOTOWNED); mtx_lock(&sc->mdev_lock); STAILQ_INSERT_TAIL(&sc->mdev_list, di, link); mtx_unlock(&sc->mdev_lock); if (flags & DPAA2_MC_DEV_ALLOCATABLE) { /* Select rman based on a type of the DPAA2 device. */ rm = dpaa2_mc_rman(mcdev, dinfo->dtype, 0); if (!rm) return (ENOENT); /* Manage DPAA2 device as an allocatable resource. */ error = rman_manage_region(rm, (rman_res_t) dpaa2_dev, (rman_res_t) dpaa2_dev); if (error) return (error); } return (0); } int dpaa2_mc_get_free_dev(device_t mcdev, device_t *dpaa2_dev, enum dpaa2_dev_type devtype) { struct rman *rm; rman_res_t start, end; int error; if (strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); /* Select resource manager based on a type of the DPAA2 device. */ rm = dpaa2_mc_rman(mcdev, devtype, 0); if (!rm) return (ENOENT); /* Find first free DPAA2 device of the given type. */ error = rman_first_free_region(rm, &start, &end); if (error) return (error); KASSERT(start == end, ("start != end, but should be the same pointer " "to the DPAA2 device: start=%jx, end=%jx", start, end)); *dpaa2_dev = (device_t) start; return (0); } int dpaa2_mc_get_dev(device_t mcdev, device_t *dpaa2_dev, enum dpaa2_dev_type devtype, uint32_t obj_id) { struct dpaa2_mc_softc *sc; struct dpaa2_devinfo *dinfo; struct dpaa2_mc_devinfo *di; int error = ENOENT; sc = device_get_softc(mcdev); if (!sc || strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); mtx_assert(&sc->mdev_lock, MA_NOTOWNED); mtx_lock(&sc->mdev_lock); STAILQ_FOREACH(di, &sc->mdev_list, link) { dinfo = device_get_ivars(di->dpaa2_dev); if (dinfo->dtype == devtype && dinfo->id == obj_id) { *dpaa2_dev = di->dpaa2_dev; error = 0; break; } } mtx_unlock(&sc->mdev_lock); return (error); } int dpaa2_mc_get_shared_dev(device_t mcdev, device_t *dpaa2_dev, enum dpaa2_dev_type devtype) { struct dpaa2_mc_softc *sc; struct dpaa2_devinfo *dinfo; struct dpaa2_mc_devinfo *di; device_t dev = NULL; uint32_t owners = UINT32_MAX; int error = ENOENT; sc = device_get_softc(mcdev); if (!sc || strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); mtx_assert(&sc->mdev_lock, MA_NOTOWNED); mtx_lock(&sc->mdev_lock); STAILQ_FOREACH(di, &sc->mdev_list, link) { dinfo = device_get_ivars(di->dpaa2_dev); if ((dinfo->dtype == devtype) && (di->flags & DPAA2_MC_DEV_SHAREABLE) && (di->owners < owners)) { dev = di->dpaa2_dev; owners = di->owners; } } if (dev) { *dpaa2_dev = dev; error = 0; } mtx_unlock(&sc->mdev_lock); return (error); } int dpaa2_mc_reserve_dev(device_t mcdev, device_t dpaa2_dev, enum dpaa2_dev_type devtype) { struct dpaa2_mc_softc *sc; struct dpaa2_mc_devinfo *di; int error = ENOENT; sc = device_get_softc(mcdev); if (!sc || strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); mtx_assert(&sc->mdev_lock, MA_NOTOWNED); mtx_lock(&sc->mdev_lock); STAILQ_FOREACH(di, &sc->mdev_list, link) { if (di->dpaa2_dev == dpaa2_dev && (di->flags & DPAA2_MC_DEV_SHAREABLE)) { di->owners++; error = 0; break; } } mtx_unlock(&sc->mdev_lock); return (error); } int dpaa2_mc_release_dev(device_t mcdev, device_t dpaa2_dev, enum dpaa2_dev_type devtype) { struct dpaa2_mc_softc *sc; struct dpaa2_mc_devinfo *di; int error = ENOENT; sc = device_get_softc(mcdev); if (!sc || strcmp(device_get_name(mcdev), "dpaa2_mc") != 0) return (EINVAL); mtx_assert(&sc->mdev_lock, MA_NOTOWNED); mtx_lock(&sc->mdev_lock); STAILQ_FOREACH(di, &sc->mdev_list, link) { if (di->dpaa2_dev == dpaa2_dev && (di->flags & DPAA2_MC_DEV_SHAREABLE)) { di->owners -= di->owners > 0 ? 1 : 0; error = 0; break; } } mtx_unlock(&sc->mdev_lock); return (error); } /** * @internal */ static u_int dpaa2_mc_get_xref(device_t mcdev, device_t child) { struct dpaa2_mc_softc *sc = device_get_softc(mcdev); struct dpaa2_devinfo *dinfo = device_get_ivars(child); #ifdef DEV_ACPI u_int xref, devid; #endif #ifdef FDT phandle_t msi_parent; #endif int error; if (sc && dinfo) { #ifdef DEV_ACPI if (sc->acpi_based) { /* * NOTE: The first named component from the IORT table * with the given name (as a substring) will be used. */ error = acpi_iort_map_named_msi(IORT_DEVICE_NAME, dinfo->icid, &xref, &devid); if (error) return (0); return (xref); } #endif #ifdef FDT if (!sc->acpi_based) { /* FDT-based driver. */ error = ofw_bus_msimap(sc->ofw_node, dinfo->icid, &msi_parent, NULL); if (error) return (0); return ((u_int) msi_parent); } #endif } return (0); } /** * @internal */ static u_int dpaa2_mc_map_id(device_t mcdev, device_t child, uintptr_t *id) { struct dpaa2_devinfo *dinfo; #ifdef DEV_ACPI u_int xref, devid; int error; #endif dinfo = device_get_ivars(child); if (dinfo) { /* * The first named components from IORT table with the given * name (as a substring) will be used. */ #ifdef DEV_ACPI error = acpi_iort_map_named_msi(IORT_DEVICE_NAME, dinfo->icid, &xref, &devid); if (error == 0) *id = devid; else #endif *id = dinfo->icid; /* RID not in IORT, likely FW bug */ return (0); } return (ENXIO); } /** * @internal * @brief Obtain a resource manager based on the given type of the resource. */ struct rman * dpaa2_mc_rman(device_t mcdev, int type, u_int flags) { struct dpaa2_mc_softc *sc; sc = device_get_softc(mcdev); switch (type) { case DPAA2_DEV_IO: return (&sc->dpio_rman); case DPAA2_DEV_BP: return (&sc->dpbp_rman); case DPAA2_DEV_CON: return (&sc->dpcon_rman); case DPAA2_DEV_MCP: return (&sc->dpmcp_rman); default: break; } return (NULL); } #if defined(INTRNG) && !defined(IOMMU) /** * @internal * @brief Allocates requested number of MSIs. * * NOTE: This function is a part of fallback solution when IOMMU isn't available. * Total number of IRQs is limited to 32. */ static int dpaa2_mc_alloc_msi_impl(device_t mcdev, device_t child, int count, int maxcount, int *irqs) { struct dpaa2_mc_softc *sc = device_get_softc(mcdev); int msi_irqs[DPAA2_MC_MSI_COUNT]; int error; /* Pre-allocate a bunch of MSIs for MC to be used by its children. */ if (!sc->msi_allocated) { error = intr_alloc_msi(mcdev, child, dpaa2_mc_get_xref(mcdev, child), DPAA2_MC_MSI_COUNT, DPAA2_MC_MSI_COUNT, msi_irqs); if (error) { device_printf(mcdev, "failed to pre-allocate %d MSIs: " "error=%d\n", DPAA2_MC_MSI_COUNT, error); return (error); } mtx_assert(&sc->msi_lock, MA_NOTOWNED); mtx_lock(&sc->msi_lock); for (int i = 0; i < DPAA2_MC_MSI_COUNT; i++) { sc->msi[i].child = NULL; sc->msi[i].irq = msi_irqs[i]; } sc->msi_owner = child; sc->msi_allocated = true; mtx_unlock(&sc->msi_lock); } error = ENOENT; /* Find the first free MSIs from the pre-allocated pool. */ mtx_assert(&sc->msi_lock, MA_NOTOWNED); mtx_lock(&sc->msi_lock); for (int i = 0; i < DPAA2_MC_MSI_COUNT; i++) { if (sc->msi[i].child != NULL) continue; error = 0; for (int j = 0; j < count; j++) { if (i + j >= DPAA2_MC_MSI_COUNT) { device_printf(mcdev, "requested %d MSIs exceed " "limit of %d available\n", count, DPAA2_MC_MSI_COUNT); error = E2BIG; break; } sc->msi[i + j].child = child; irqs[j] = sc->msi[i + j].irq; } break; } mtx_unlock(&sc->msi_lock); return (error); } /** * @internal * @brief Marks IRQs as free in the pre-allocated pool of MSIs. * * NOTE: This function is a part of fallback solution when IOMMU isn't available. * Total number of IRQs is limited to 32. * NOTE: MSIs are kept allocated in the kernel as a part of the pool. */ static int dpaa2_mc_release_msi_impl(device_t mcdev, device_t child, int count, int *irqs) { struct dpaa2_mc_softc *sc = device_get_softc(mcdev); mtx_assert(&sc->msi_lock, MA_NOTOWNED); mtx_lock(&sc->msi_lock); for (int i = 0; i < DPAA2_MC_MSI_COUNT; i++) { if (sc->msi[i].child != child) continue; for (int j = 0; j < count; j++) { if (sc->msi[i].irq == irqs[j]) { sc->msi[i].child = NULL; break; } } } mtx_unlock(&sc->msi_lock); return (0); } /** * @internal * @brief Provides address to write to and data according to the given MSI from * the pre-allocated pool. * * NOTE: This function is a part of fallback solution when IOMMU isn't available. * Total number of IRQs is limited to 32. */ static int dpaa2_mc_map_msi_impl(device_t mcdev, device_t child, int irq, uint64_t *addr, uint32_t *data) { struct dpaa2_mc_softc *sc = device_get_softc(mcdev); int error = EINVAL; mtx_assert(&sc->msi_lock, MA_NOTOWNED); mtx_lock(&sc->msi_lock); for (int i = 0; i < DPAA2_MC_MSI_COUNT; i++) { if (sc->msi[i].child == child && sc->msi[i].irq == irq) { error = 0; break; } } mtx_unlock(&sc->msi_lock); if (error) return (error); return (intr_map_msi(mcdev, sc->msi_owner, dpaa2_mc_get_xref(mcdev, sc->msi_owner), irq, addr, data)); } #endif /* defined(INTRNG) && !defined(IOMMU) */ static device_method_t dpaa2_mc_methods[] = { DEVMETHOD_END }; DEFINE_CLASS_0(dpaa2_mc, dpaa2_mc_driver, dpaa2_mc_methods, sizeof(struct dpaa2_mc_softc)); diff --git a/sys/dev/dpaa2/dpaa2_rc.c b/sys/dev/dpaa2/dpaa2_rc.c index 7f8eddff1353..d8e15e388bf5 100644 --- a/sys/dev/dpaa2/dpaa2_rc.c +++ b/sys/dev/dpaa2/dpaa2_rc.c @@ -1,3552 +1,3552 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright © 2021-2022 Dmitry Salychev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * The DPAA2 Resource Container (DPRC) bus driver. * * DPRC holds all the resources and object information that a software context * (kernel, virtual machine, etc.) can access or use. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" #include "pci_if.h" #include "dpaa2_mcp.h" #include "dpaa2_mc.h" #include "dpaa2_ni.h" #include "dpaa2_mc_if.h" #include "dpaa2_cmd_if.h" /* Timeouts to wait for a command response from MC. */ #define CMD_SPIN_TIMEOUT 100u /* us */ #define CMD_SPIN_ATTEMPTS 2000u /* max. 200 ms */ #define TYPE_LEN_MAX 16u #define LABEL_LEN_MAX 16u MALLOC_DEFINE(M_DPAA2_RC, "dpaa2_rc", "DPAA2 Resource Container"); /* Discover and add devices to the resource container. */ static int dpaa2_rc_discover(struct dpaa2_rc_softc *); static int dpaa2_rc_add_child(struct dpaa2_rc_softc *, struct dpaa2_cmd *, struct dpaa2_obj *); static int dpaa2_rc_add_managed_child(struct dpaa2_rc_softc *, struct dpaa2_cmd *, struct dpaa2_obj *); /* Helper routines. */ static int dpaa2_rc_enable_irq(struct dpaa2_mcp *, struct dpaa2_cmd *, uint8_t, bool, uint16_t); static int dpaa2_rc_configure_irq(device_t, device_t, int, uint64_t, uint32_t); static int dpaa2_rc_add_res(device_t, device_t, enum dpaa2_dev_type, int *, int); static int dpaa2_rc_print_type(struct resource_list *, enum dpaa2_dev_type); static struct dpaa2_mcp *dpaa2_rc_select_portal(device_t, device_t); /* Routines to send commands to MC. */ static int dpaa2_rc_exec_cmd(struct dpaa2_mcp *, struct dpaa2_cmd *, uint16_t); static int dpaa2_rc_send_cmd(struct dpaa2_mcp *, struct dpaa2_cmd *); static int dpaa2_rc_wait_for_cmd(struct dpaa2_mcp *, struct dpaa2_cmd *); static int dpaa2_rc_reset_cmd_params(struct dpaa2_cmd *); static int dpaa2_rc_probe(device_t dev) { /* DPRC device will be added by the parent DPRC or MC bus itself. */ device_set_desc(dev, "DPAA2 Resource Container"); return (BUS_PROBE_DEFAULT); } static int dpaa2_rc_detach(device_t dev) { struct dpaa2_devinfo *dinfo; int error; error = bus_generic_detach(dev); if (error) return (error); dinfo = device_get_ivars(dev); if (dinfo->portal) dpaa2_mcp_free_portal(dinfo->portal); if (dinfo) free(dinfo, M_DPAA2_RC); - return (device_delete_children(dev)); + return (0); } static int dpaa2_rc_attach(device_t dev) { device_t pdev; struct dpaa2_mc_softc *mcsc; struct dpaa2_rc_softc *sc; struct dpaa2_devinfo *dinfo = NULL; int error; sc = device_get_softc(dev); sc->dev = dev; sc->unit = device_get_unit(dev); if (sc->unit == 0) { /* Root DPRC should be attached directly to the MC bus. */ pdev = device_get_parent(dev); mcsc = device_get_softc(pdev); KASSERT(strcmp(device_get_name(pdev), "dpaa2_mc") == 0, ("root DPRC should be attached to the MC bus")); /* * Allocate devinfo to let the parent MC bus access ICID of the * DPRC object. */ dinfo = malloc(sizeof(struct dpaa2_devinfo), M_DPAA2_RC, M_WAITOK | M_ZERO); if (!dinfo) { device_printf(dev, "%s: failed to allocate " "dpaa2_devinfo\n", __func__); dpaa2_rc_detach(dev); return (ENXIO); } device_set_ivars(dev, dinfo); dinfo->pdev = pdev; dinfo->dev = dev; dinfo->dtype = DPAA2_DEV_RC; dinfo->portal = NULL; /* Prepare helper portal object to send commands to MC. */ error = dpaa2_mcp_init_portal(&dinfo->portal, mcsc->res[0], &mcsc->map[0], DPAA2_PORTAL_DEF); if (error) { device_printf(dev, "%s: failed to initialize dpaa2_mcp: " "error=%d\n", __func__, error); dpaa2_rc_detach(dev); return (ENXIO); } } else { /* TODO: Child DPRCs aren't supported yet. */ return (ENXIO); } /* Create DPAA2 devices for objects in this container. */ error = dpaa2_rc_discover(sc); if (error) { device_printf(dev, "%s: failed to discover objects in " "container: error=%d\n", __func__, error); dpaa2_rc_detach(dev); return (error); } return (0); } /* * Bus interface. */ static struct resource_list * dpaa2_rc_get_resource_list(device_t rcdev, device_t child) { struct dpaa2_devinfo *dinfo = device_get_ivars(child); return (&dinfo->resources); } static void dpaa2_rc_delete_resource(device_t rcdev, device_t child, int type, int rid) { struct resource_list *rl; struct resource_list_entry *rle; struct dpaa2_devinfo *dinfo; if (device_get_parent(child) != rcdev) return; dinfo = device_get_ivars(child); rl = &dinfo->resources; rle = resource_list_find(rl, type, rid); if (rle == NULL) return; if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, type, rid)) { device_printf(rcdev, "%s: resource still owned by " "child: type=%d, rid=%d, start=%jx\n", __func__, type, rid, rman_get_start(rle->res)); return; } resource_list_unreserve(rl, rcdev, child, type, rid); } resource_list_delete(rl, type, rid); } static struct resource * dpaa2_rc_alloc_multi_resource(device_t rcdev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource_list *rl; struct dpaa2_devinfo *dinfo; dinfo = device_get_ivars(child); rl = &dinfo->resources; /* * By default, software portal interrupts are message-based, that is, * they are issued from QMan using a 4 byte write. * * TODO: However this default behavior can be changed by programming one * or more software portals to issue their interrupts via a * dedicated software portal interrupt wire. * See registers SWP_INTW0_CFG to SWP_INTW3_CFG for details. */ if (type == SYS_RES_IRQ && *rid == 0) return (NULL); return (resource_list_alloc(rl, rcdev, child, type, rid, start, end, count, flags)); } static struct resource * dpaa2_rc_alloc_resource(device_t rcdev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { if (device_get_parent(child) != rcdev) return (BUS_ALLOC_RESOURCE(device_get_parent(rcdev), child, type, rid, start, end, count, flags)); return (dpaa2_rc_alloc_multi_resource(rcdev, child, type, rid, start, end, count, flags)); } static int dpaa2_rc_release_resource(device_t rcdev, device_t child, struct resource *r) { struct resource_list *rl; struct dpaa2_devinfo *dinfo; if (device_get_parent(child) != rcdev) return (BUS_RELEASE_RESOURCE(device_get_parent(rcdev), child, r)); dinfo = device_get_ivars(child); rl = &dinfo->resources; return (resource_list_release(rl, rcdev, child, r)); } static void dpaa2_rc_child_deleted(device_t rcdev, device_t child) { struct dpaa2_devinfo *dinfo; struct resource_list *rl; struct resource_list_entry *rle; dinfo = device_get_ivars(child); rl = &dinfo->resources; /* Free all allocated resources */ STAILQ_FOREACH(rle, rl, link) { if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, rle->type, rle->rid)) { device_printf(child, "%s: resource still owned: " "type=%d, rid=%d, addr=%lx\n", __func__, rle->type, rle->rid, rman_get_start(rle->res)); bus_release_resource(child, rle->type, rle->rid, rle->res); } resource_list_unreserve(rl, rcdev, child, rle->type, rle->rid); } } resource_list_free(rl); if (dinfo) free(dinfo, M_DPAA2_RC); } static void dpaa2_rc_child_detached(device_t rcdev, device_t child) { struct dpaa2_devinfo *dinfo; struct resource_list *rl; dinfo = device_get_ivars(child); rl = &dinfo->resources; if (resource_list_release_active(rl, rcdev, child, SYS_RES_IRQ) != 0) device_printf(child, "%s: leaked IRQ resources!\n", __func__); if (dinfo->msi.msi_alloc != 0) { device_printf(child, "%s: leaked %d MSI vectors!\n", __func__, dinfo->msi.msi_alloc); PCI_RELEASE_MSI(rcdev, child); } if (resource_list_release_active(rl, rcdev, child, SYS_RES_MEMORY) != 0) device_printf(child, "%s: leaked memory resources!\n", __func__); } static int dpaa2_rc_setup_intr(device_t rcdev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { struct dpaa2_devinfo *dinfo; uint64_t addr; uint32_t data; void *cookie; int error, rid; error = bus_generic_setup_intr(rcdev, child, irq, flags, filter, intr, arg, &cookie); if (error) { device_printf(rcdev, "%s: bus_generic_setup_intr() failed: " "error=%d\n", __func__, error); return (error); } /* If this is not a direct child, just bail out. */ if (device_get_parent(child) != rcdev) { *cookiep = cookie; return (0); } rid = rman_get_rid(irq); if (rid == 0) { if (bootverbose) device_printf(rcdev, "%s: cannot setup interrupt with " "rid=0: INTx are not supported by DPAA2 objects " "yet\n", __func__); return (EINVAL); } else { dinfo = device_get_ivars(child); KASSERT(dinfo->msi.msi_alloc > 0, ("No MSI interrupts allocated")); /* * Ask our parent to map the MSI and give us the address and * data register values. If we fail for some reason, teardown * the interrupt handler. */ error = PCIB_MAP_MSI(device_get_parent(rcdev), child, rman_get_start(irq), &addr, &data); if (error) { device_printf(rcdev, "%s: PCIB_MAP_MSI failed: " "error=%d\n", __func__, error); (void)bus_generic_teardown_intr(rcdev, child, irq, cookie); return (error); } /* Configure MSI for this DPAA2 object. */ error = dpaa2_rc_configure_irq(rcdev, child, rid, addr, data); if (error) { device_printf(rcdev, "%s: failed to configure IRQ for " "DPAA2 object: rid=%d, type=%s, unit=%d\n", __func__, rid, dpaa2_ttos(dinfo->dtype), device_get_unit(child)); return (error); } dinfo->msi.msi_handlers++; } *cookiep = cookie; return (0); } static int dpaa2_rc_teardown_intr(device_t rcdev, device_t child, struct resource *irq, void *cookie) { struct resource_list_entry *rle; struct dpaa2_devinfo *dinfo; int error, rid; if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE)) return (EINVAL); /* If this isn't a direct child, just bail out */ if (device_get_parent(child) != rcdev) return(bus_generic_teardown_intr(rcdev, child, irq, cookie)); rid = rman_get_rid(irq); if (rid == 0) { if (bootverbose) device_printf(rcdev, "%s: cannot teardown interrupt " "with rid=0: INTx are not supported by DPAA2 " "objects yet\n", __func__); return (EINVAL); } else { dinfo = device_get_ivars(child); rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); if (rle->res != irq) return (EINVAL); dinfo->msi.msi_handlers--; } error = bus_generic_teardown_intr(rcdev, child, irq, cookie); if (rid > 0) KASSERT(error == 0, ("%s: generic teardown failed for MSI", __func__)); return (error); } static int dpaa2_rc_print_child(device_t rcdev, device_t child) { struct dpaa2_devinfo *dinfo = device_get_ivars(child); struct resource_list *rl = &dinfo->resources; int retval = 0; retval += bus_print_child_header(rcdev, child); retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx"); retval += resource_list_print_type(rl, "iomem", SYS_RES_MEMORY, "%#jx"); retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd"); /* Print DPAA2-specific resources. */ retval += dpaa2_rc_print_type(rl, DPAA2_DEV_IO); retval += dpaa2_rc_print_type(rl, DPAA2_DEV_BP); retval += dpaa2_rc_print_type(rl, DPAA2_DEV_CON); retval += dpaa2_rc_print_type(rl, DPAA2_DEV_MCP); retval += printf(" at %s (id=%u)", dpaa2_ttos(dinfo->dtype), dinfo->id); retval += bus_print_child_domain(rcdev, child); retval += bus_print_child_footer(rcdev, child); return (retval); } /* * Pseudo-PCI interface. */ /* * Attempt to allocate *count MSI messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at a rid 1. * * NOTE: Implementation is similar to sys/dev/pci/pci.c. */ static int dpaa2_rc_alloc_msi(device_t rcdev, device_t child, int *count) { struct dpaa2_devinfo *rcinfo = device_get_ivars(rcdev); struct dpaa2_devinfo *dinfo = device_get_ivars(child); int error, actual, i, run, irqs[32]; /* Don't let count == 0 get us into trouble. */ if (*count == 0) return (EINVAL); /* MSI should be allocated by the resource container. */ if (rcinfo->dtype != DPAA2_DEV_RC) return (ENODEV); /* Already have allocated messages? */ if (dinfo->msi.msi_alloc != 0) return (ENXIO); /* Don't ask for more than the device supports. */ actual = min(*count, dinfo->msi.msi_msgnum); /* Don't ask for more than 32 messages. */ actual = min(actual, 32); /* MSI requires power of 2 number of messages. */ if (!powerof2(actual)) return (EINVAL); for (;;) { /* Try to allocate N messages. */ error = PCIB_ALLOC_MSI(device_get_parent(rcdev), child, actual, actual, irqs); if (error == 0) break; if (actual == 1) return (error); /* Try N / 2. */ actual >>= 1; } /* * We now have N actual messages mapped onto SYS_RES_IRQ resources in * the irqs[] array, so add new resources starting at rid 1. */ for (i = 0; i < actual; i++) resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irqs[i], irqs[i], 1); if (bootverbose) { if (actual == 1) { device_printf(child, "using IRQ %d for MSI\n", irqs[0]); } else { /* * Be fancy and try to print contiguous runs * of IRQ values as ranges. 'run' is true if * we are in a range. */ device_printf(child, "using IRQs %d", irqs[0]); run = 0; for (i = 1; i < actual; i++) { /* Still in a run? */ if (irqs[i] == irqs[i - 1] + 1) { run = 1; continue; } /* Finish previous range. */ if (run) { printf("-%d", irqs[i - 1]); run = 0; } /* Start new range. */ printf(",%d", irqs[i]); } /* Unfinished range? */ if (run) printf("-%d", irqs[actual - 1]); printf(" for MSI\n"); } } /* Update counts of alloc'd messages. */ dinfo->msi.msi_alloc = actual; dinfo->msi.msi_handlers = 0; *count = actual; return (0); } /* * Release the MSI messages associated with this DPAA2 device. * * NOTE: Implementation is similar to sys/dev/pci/pci.c. */ static int dpaa2_rc_release_msi(device_t rcdev, device_t child) { struct dpaa2_devinfo *rcinfo = device_get_ivars(rcdev); struct dpaa2_devinfo *dinfo = device_get_ivars(child); struct resource_list_entry *rle; int i, irqs[32]; /* MSI should be released by the resource container. */ if (rcinfo->dtype != DPAA2_DEV_RC) return (ENODEV); /* Do we have any messages to release? */ if (dinfo->msi.msi_alloc == 0) return (ENODEV); KASSERT(dinfo->msi.msi_alloc <= 32, ("more than 32 alloc'd MSI messages")); /* Make sure none of the resources are allocated. */ if (dinfo->msi.msi_handlers > 0) return (EBUSY); for (i = 0; i < dinfo->msi.msi_alloc; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing MSI resource")); if (rle->res != NULL) return (EBUSY); irqs[i] = rle->start; } /* Release the messages. */ PCIB_RELEASE_MSI(device_get_parent(rcdev), child, dinfo->msi.msi_alloc, irqs); for (i = 0; i < dinfo->msi.msi_alloc; i++) resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); /* Update alloc count. */ dinfo->msi.msi_alloc = 0; return (0); } /** * @brief Return the maximum number of the MSI supported by this DPAA2 device. */ static int dpaa2_rc_msi_count(device_t rcdev, device_t child) { struct dpaa2_devinfo *dinfo = device_get_ivars(child); return (dinfo->msi.msi_msgnum); } static int dpaa2_rc_get_id(device_t rcdev, device_t child, enum pci_id_type type, uintptr_t *id) { struct dpaa2_devinfo *rcinfo = device_get_ivars(rcdev); if (rcinfo->dtype != DPAA2_DEV_RC) return (ENODEV); return (PCIB_GET_ID(device_get_parent(rcdev), child, type, id)); } /* * DPAA2 MC command interface. */ static int dpaa2_rc_mng_get_version(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t *major, uint32_t *minor, uint32_t *rev) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || major == NULL || minor == NULL || rev == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MNG_GET_VER); if (!error) { *major = cmd->params[0] >> 32; *minor = cmd->params[1] & 0xFFFFFFFF; *rev = cmd->params[0] & 0xFFFFFFFF; } return (error); } static int dpaa2_rc_mng_get_soc_version(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t *pvr, uint32_t *svr) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || pvr == NULL || svr == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MNG_GET_SOC_VER); if (!error) { *pvr = cmd->params[0] >> 32; *svr = cmd->params[0] & 0xFFFFFFFF; } return (error); } static int dpaa2_rc_mng_get_container_id(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t *cont_id) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || cont_id == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MNG_GET_CONT_ID); if (!error) *cont_id = cmd->params[0] & 0xFFFFFFFF; return (error); } static int dpaa2_rc_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t cont_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = cont_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_CLOSE)); } static int dpaa2_rc_get_obj_count(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t *obj_count) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || obj_count == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_OBJ_COUNT); if (!error) *obj_count = (uint32_t)(cmd->params[0] >> 32); return (error); } static int dpaa2_rc_get_obj(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t obj_idx, struct dpaa2_obj *obj) { struct __packed dpaa2_obj_resp { uint32_t _reserved1; uint32_t id; uint16_t vendor; uint8_t irq_count; uint8_t reg_count; uint32_t state; uint16_t ver_major; uint16_t ver_minor; uint16_t flags; uint16_t _reserved2; uint8_t type[16]; uint8_t label[16]; } *pobj; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || obj == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = obj_idx; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_OBJ); if (!error) { pobj = (struct dpaa2_obj_resp *) &cmd->params[0]; obj->id = pobj->id; obj->vendor = pobj->vendor; obj->irq_count = pobj->irq_count; obj->reg_count = pobj->reg_count; obj->state = pobj->state; obj->ver_major = pobj->ver_major; obj->ver_minor = pobj->ver_minor; obj->flags = pobj->flags; obj->type = dpaa2_stot((const char *) pobj->type); memcpy(obj->label, pobj->label, sizeof(pobj->label)); } /* Some DPAA2 objects might not be supported by the driver yet. */ if (obj->type == DPAA2_DEV_NOTYPE) error = DPAA2_CMD_STAT_UNKNOWN_OBJ; return (error); } static int dpaa2_rc_get_obj_descriptor(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t obj_id, enum dpaa2_dev_type dtype, struct dpaa2_obj *obj) { struct __packed get_obj_desc_args { uint32_t obj_id; uint32_t _reserved1; uint8_t type[16]; } *args; struct __packed dpaa2_obj_resp { uint32_t _reserved1; uint32_t id; uint16_t vendor; uint8_t irq_count; uint8_t reg_count; uint32_t state; uint16_t ver_major; uint16_t ver_minor; uint16_t flags; uint16_t _reserved2; uint8_t type[16]; uint8_t label[16]; } *pobj; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); const char *type = dpaa2_ttos(dtype); int error; if (portal == NULL || cmd == NULL || obj == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct get_obj_desc_args *) &cmd->params[0]; args->obj_id = obj_id; memcpy(args->type, type, min(strlen(type) + 1, TYPE_LEN_MAX)); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_OBJ_DESC); if (!error) { pobj = (struct dpaa2_obj_resp *) &cmd->params[0]; obj->id = pobj->id; obj->vendor = pobj->vendor; obj->irq_count = pobj->irq_count; obj->reg_count = pobj->reg_count; obj->state = pobj->state; obj->ver_major = pobj->ver_major; obj->ver_minor = pobj->ver_minor; obj->flags = pobj->flags; obj->type = dpaa2_stot((const char *) pobj->type); memcpy(obj->label, pobj->label, sizeof(pobj->label)); } /* Some DPAA2 objects might not be supported by the driver yet. */ if (obj->type == DPAA2_DEV_NOTYPE) error = DPAA2_CMD_STAT_UNKNOWN_OBJ; return (error); } static int dpaa2_rc_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_rc_attr *attr) { struct __packed dpaa2_rc_attr { uint32_t cont_id; uint32_t icid; uint32_t options; uint32_t portal_id; } *pattr; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_ATTR); if (!error) { pattr = (struct dpaa2_rc_attr *) &cmd->params[0]; attr->cont_id = pattr->cont_id; attr->portal_id = pattr->portal_id; attr->options = pattr->options; attr->icid = pattr->icid; } return (error); } static int dpaa2_rc_get_obj_region(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t obj_id, uint8_t reg_idx, enum dpaa2_dev_type dtype, struct dpaa2_rc_obj_region *reg) { struct __packed obj_region_args { uint32_t obj_id; uint16_t _reserved1; uint8_t reg_idx; uint8_t _reserved2; uint64_t _reserved3; uint64_t _reserved4; uint8_t type[16]; } *args; struct __packed obj_region { uint64_t _reserved1; uint64_t base_offset; uint32_t size; uint32_t type; uint32_t flags; uint32_t _reserved2; uint64_t base_paddr; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); uint16_t cmdid, api_major, api_minor; const char *type = dpaa2_ttos(dtype); int error; if (portal == NULL || cmd == NULL || reg == NULL) return (DPAA2_CMD_STAT_ERR); /* * If the DPRC object version was not yet cached, cache it now. * Otherwise use the already cached value. */ if (!portal->rc_api_major && !portal->rc_api_minor) { error = DPAA2_CMD_RC_GET_API_VERSION(dev, child, cmd, &api_major, &api_minor); if (error) return (error); portal->rc_api_major = api_major; portal->rc_api_minor = api_minor; } else { api_major = portal->rc_api_major; api_minor = portal->rc_api_minor; } /* TODO: Remove magic numbers. */ if (api_major > 6u || (api_major == 6u && api_minor >= 6u)) /* * MC API version 6.6 changed the size of the MC portals and * software portals to 64K (as implemented by hardware). */ cmdid = CMDID_RC_GET_OBJ_REG_V3; else if (api_major == 6u && api_minor >= 3u) /* * MC API version 6.3 introduced a new field to the region * descriptor: base_address. */ cmdid = CMDID_RC_GET_OBJ_REG_V2; else cmdid = CMDID_RC_GET_OBJ_REG; args = (struct obj_region_args *) &cmd->params[0]; args->obj_id = obj_id; args->reg_idx = reg_idx; memcpy(args->type, type, min(strlen(type) + 1, TYPE_LEN_MAX)); error = dpaa2_rc_exec_cmd(portal, cmd, cmdid); if (!error) { resp = (struct obj_region *) &cmd->params[0]; reg->base_paddr = resp->base_paddr; reg->base_offset = resp->base_offset; reg->size = resp->size; reg->flags = resp->flags; reg->type = resp->type & 0xFu; } return (error); } static int dpaa2_rc_get_api_version(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint16_t *major, uint16_t *minor) { struct __packed rc_api_version { uint16_t major; uint16_t minor; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || major == NULL || minor == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_API_VERSION); if (!error) { resp = (struct rc_api_version *) &cmd->params[0]; *major = resp->major; *minor = resp->minor; } return (error); } static int dpaa2_rc_set_irq_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint8_t enable) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_enable_irq(portal, cmd, irq_idx, enable, CMDID_RC_SET_IRQ_ENABLE)); } static int dpaa2_rc_set_obj_irq(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint64_t addr, uint32_t data, uint32_t irq_usr, uint32_t obj_id, enum dpaa2_dev_type dtype) { struct __packed set_obj_irq_args { uint32_t data; uint8_t irq_idx; uint8_t _reserved1[3]; uint64_t addr; uint32_t irq_usr; uint32_t obj_id; uint8_t type[16]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); const char *type = dpaa2_ttos(dtype); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct set_obj_irq_args *) &cmd->params[0]; args->irq_idx = irq_idx; args->addr = addr; args->data = data; args->irq_usr = irq_usr; args->obj_id = obj_id; memcpy(args->type, type, min(strlen(type) + 1, TYPE_LEN_MAX)); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_SET_OBJ_IRQ)); } static int dpaa2_rc_get_conn(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ep_desc *ep1_desc, struct dpaa2_ep_desc *ep2_desc, uint32_t *link_stat) { struct __packed get_conn_args { uint32_t ep1_id; uint32_t ep1_ifid; uint8_t ep1_type[16]; uint64_t _reserved[4]; } *args; struct __packed get_conn_resp { uint64_t _reserved1[3]; uint32_t ep2_id; uint32_t ep2_ifid; uint8_t ep2_type[16]; uint32_t link_stat; uint32_t _reserved2; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || ep1_desc == NULL || ep2_desc == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct get_conn_args *) &cmd->params[0]; args->ep1_id = ep1_desc->obj_id; args->ep1_ifid = ep1_desc->if_id; /* TODO: Remove magic number. */ strncpy(args->ep1_type, dpaa2_ttos(ep1_desc->type), 16); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_RC_GET_CONN); if (!error) { resp = (struct get_conn_resp *) &cmd->params[0]; ep2_desc->obj_id = resp->ep2_id; ep2_desc->if_id = resp->ep2_ifid; ep2_desc->type = dpaa2_stot((const char *) resp->ep2_type); if (link_stat != NULL) *link_stat = resp->link_stat; } return (error); } static int dpaa2_rc_ni_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpni_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpni_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_ni_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_CLOSE)); } static int dpaa2_rc_ni_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_ENABLE)); } static int dpaa2_rc_ni_disable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_DISABLE)); } static int dpaa2_rc_ni_get_api_version(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint16_t *major, uint16_t *minor) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || major == NULL || minor == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_API_VER); if (!error) { *major = cmd->params[0] & 0xFFFFU; *minor = (cmd->params[0] >> 16) & 0xFFFFU; } return (error); } static int dpaa2_rc_ni_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_RESET)); } static int dpaa2_rc_ni_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_attr *attr) { struct __packed ni_attr { uint32_t options; uint8_t num_queues; uint8_t num_rx_tcs; uint8_t mac_entries; uint8_t num_tx_tcs; uint8_t vlan_entries; uint8_t num_channels; uint8_t qos_entries; uint8_t _reserved1; uint16_t fs_entries; uint16_t _reserved2; uint8_t qos_key_size; uint8_t fs_key_size; uint16_t wriop_ver; uint8_t num_cgs; uint8_t _reserved3; uint16_t _reserved4; uint64_t _reserved5[4]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_ATTR); if (!error) { resp = (struct ni_attr *) &cmd->params[0]; attr->options = resp->options; attr->wriop_ver = resp->wriop_ver; attr->entries.fs = resp->fs_entries; attr->entries.mac = resp->mac_entries; attr->entries.vlan = resp->vlan_entries; attr->entries.qos = resp->qos_entries; attr->num.queues = resp->num_queues; attr->num.rx_tcs = resp->num_rx_tcs; attr->num.tx_tcs = resp->num_tx_tcs; attr->num.channels = resp->num_channels; attr->num.cgs = resp->num_cgs; attr->key_size.fs = resp->fs_key_size; attr->key_size.qos = resp->qos_key_size; } return (error); } static int dpaa2_rc_ni_set_buf_layout(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_buf_layout *bl) { struct __packed set_buf_layout_args { uint8_t queue_type; uint8_t _reserved1; uint16_t _reserved2; uint16_t options; uint8_t params; uint8_t _reserved3; uint16_t priv_data_size; uint16_t data_align; uint16_t head_room; uint16_t tail_room; uint64_t _reserved4[5]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || bl == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct set_buf_layout_args *) &cmd->params[0]; args->queue_type = (uint8_t) bl->queue_type; args->options = bl->options; args->params = 0; args->priv_data_size = bl->pd_size; args->data_align = bl->fd_align; args->head_room = bl->head_size; args->tail_room = bl->tail_size; args->params |= bl->pass_timestamp ? 1U : 0U; args->params |= bl->pass_parser_result ? 2U : 0U; args->params |= bl->pass_frame_status ? 4U : 0U; args->params |= bl->pass_sw_opaque ? 8U : 0U; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_BUF_LAYOUT)); } static int dpaa2_rc_ni_get_tx_data_offset(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint16_t *offset) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || offset == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_TX_DATA_OFF); if (!error) *offset = cmd->params[0] & 0xFFFFU; return (error); } static int dpaa2_rc_ni_get_port_mac_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_PORT_MAC_ADDR); if (!error) { mac[0] = (cmd->params[0] >> 56) & 0xFFU; mac[1] = (cmd->params[0] >> 48) & 0xFFU; mac[2] = (cmd->params[0] >> 40) & 0xFFU; mac[3] = (cmd->params[0] >> 32) & 0xFFU; mac[4] = (cmd->params[0] >> 24) & 0xFFU; mac[5] = (cmd->params[0] >> 16) & 0xFFU; } return (error); } static int dpaa2_rc_ni_set_prim_mac_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct __packed set_prim_mac_args { uint8_t _reserved[2]; uint8_t mac[ETHER_ADDR_LEN]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_EINVAL); args = (struct set_prim_mac_args *) &cmd->params[0]; for (int i = 1; i <= ETHER_ADDR_LEN; i++) args->mac[i - 1] = mac[ETHER_ADDR_LEN - i]; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_PRIM_MAC_ADDR)); } static int dpaa2_rc_ni_get_prim_mac_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct __packed get_prim_mac_resp { uint8_t _reserved[2]; uint8_t mac[ETHER_ADDR_LEN]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_EINVAL); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_PRIM_MAC_ADDR); if (!error) { resp = (struct get_prim_mac_resp *) &cmd->params[0]; for (int i = 1; i <= ETHER_ADDR_LEN; i++) mac[ETHER_ADDR_LEN - i] = resp->mac[i - 1]; } return (error); } static int dpaa2_rc_ni_set_link_cfg(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_link_cfg *cfg) { struct __packed link_cfg_args { uint64_t _reserved1; uint32_t rate; uint32_t _reserved2; uint64_t options; uint64_t adv_speeds; uint64_t _reserved3[3]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); args = (struct link_cfg_args *) &cmd->params[0]; args->rate = cfg->rate; args->options = cfg->options; args->adv_speeds = cfg->adv_speeds; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_LINK_CFG)); } static int dpaa2_rc_ni_get_link_cfg(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_link_cfg *cfg) { struct __packed link_cfg_resp { uint64_t _reserved1; uint32_t rate; uint32_t _reserved2; uint64_t options; uint64_t adv_speeds; uint64_t _reserved3[3]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_LINK_CFG); if (!error) { resp = (struct link_cfg_resp *) &cmd->params[0]; cfg->rate = resp->rate; cfg->options = resp->options; cfg->adv_speeds = resp->adv_speeds; } return (error); } static int dpaa2_rc_ni_get_link_state(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_link_state *state) { struct __packed link_state_resp { uint32_t _reserved1; uint32_t flags; uint32_t rate; uint32_t _reserved2; uint64_t options; uint64_t supported; uint64_t advert; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || state == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_LINK_STATE); if (!error) { resp = (struct link_state_resp *) &cmd->params[0]; state->options = resp->options; state->adv_speeds = resp->advert; state->sup_speeds = resp->supported; state->rate = resp->rate; state->link_up = resp->flags & 0x1u ? true : false; state->state_valid = resp->flags & 0x2u ? true : false; } return (error); } static int dpaa2_rc_ni_set_qos_table(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_qos_table *tbl) { struct __packed qos_table_args { uint32_t _reserved1; uint8_t default_tc; uint8_t options; uint16_t _reserved2; uint64_t _reserved[5]; uint64_t kcfg_busaddr; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || tbl == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct qos_table_args *) &cmd->params[0]; args->default_tc = tbl->default_tc; args->kcfg_busaddr = tbl->kcfg_busaddr; args->options |= tbl->discard_on_miss ? 1U : 0U; args->options |= tbl->keep_entries ? 2U : 0U; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_QOS_TABLE)); } static int dpaa2_rc_ni_clear_qos_table(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_CLEAR_QOS_TABLE)); } static int dpaa2_rc_ni_set_pools(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_pools_cfg *cfg) { struct __packed set_pools_args { uint8_t pools_num; uint8_t backup_pool_mask; uint8_t _reserved1; uint8_t pool_as; /* assigning: 0 - QPRI, 1 - QDBIN */ uint32_t bp_obj_id[DPAA2_NI_MAX_POOLS]; uint16_t buf_sz[DPAA2_NI_MAX_POOLS]; uint32_t _reserved2; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_pools_args *) &cmd->params[0]; args->pools_num = cfg->pools_num < DPAA2_NI_MAX_POOLS ? cfg->pools_num : DPAA2_NI_MAX_POOLS; for (uint32_t i = 0; i < args->pools_num; i++) { args->bp_obj_id[i] = cfg->pools[i].bp_obj_id; args->buf_sz[i] = cfg->pools[i].buf_sz; args->backup_pool_mask |= (cfg->pools[i].backup_flag & 1) << i; } return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_POOLS)); } static int dpaa2_rc_ni_set_err_behavior(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_err_cfg *cfg) { struct __packed err_behavior_args { uint32_t err_mask; uint8_t flags; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct err_behavior_args *) &cmd->params[0]; args->err_mask = cfg->err_mask; args->flags |= cfg->set_err_fas ? 0x10u : 0u; args->flags |= ((uint8_t) cfg->action) & 0x0Fu; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_ERR_BEHAVIOR)); } static int dpaa2_rc_ni_get_queue(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_queue_cfg *cfg) { struct __packed get_queue_args { uint8_t queue_type; uint8_t tc; uint8_t idx; uint8_t chan_id; } *args; struct __packed get_queue_resp { uint64_t _reserved1; uint32_t dest_id; uint16_t _reserved2; uint8_t priority; uint8_t flags; uint64_t flc; uint64_t user_ctx; uint32_t fqid; uint16_t qdbin; uint16_t _reserved3; uint8_t cgid; uint8_t _reserved[15]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_queue_args *) &cmd->params[0]; args->queue_type = (uint8_t) cfg->type; args->tc = cfg->tc; args->idx = cfg->idx; args->chan_id = cfg->chan_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_QUEUE); if (!error) { resp = (struct get_queue_resp *) &cmd->params[0]; cfg->dest_id = resp->dest_id; cfg->priority = resp->priority; cfg->flow_ctx = resp->flc; cfg->user_ctx = resp->user_ctx; cfg->fqid = resp->fqid; cfg->qdbin = resp->qdbin; cfg->cgid = resp->cgid; cfg->dest_type = (enum dpaa2_ni_dest_type) resp->flags & 0x0Fu; cfg->cgid_valid = (resp->flags & 0x20u) > 0u ? true : false; cfg->stash_control = (resp->flags & 0x40u) > 0u ? true : false; cfg->hold_active = (resp->flags & 0x80u) > 0u ? true : false; } return (error); } static int dpaa2_rc_ni_set_queue(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_ni_queue_cfg *cfg) { struct __packed set_queue_args { uint8_t queue_type; uint8_t tc; uint8_t idx; uint8_t options; uint32_t _reserved1; uint32_t dest_id; uint16_t _reserved2; uint8_t priority; uint8_t flags; uint64_t flc; uint64_t user_ctx; uint8_t cgid; uint8_t chan_id; uint8_t _reserved[23]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_queue_args *) &cmd->params[0]; args->queue_type = (uint8_t) cfg->type; args->tc = cfg->tc; args->idx = cfg->idx; args->options = cfg->options; args->dest_id = cfg->dest_id; args->priority = cfg->priority; args->flc = cfg->flow_ctx; args->user_ctx = cfg->user_ctx; args->cgid = cfg->cgid; args->chan_id = cfg->chan_id; args->flags |= (uint8_t)(cfg->dest_type & 0x0Fu); args->flags |= cfg->stash_control ? 0x40u : 0u; args->flags |= cfg->hold_active ? 0x80u : 0u; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_QUEUE)); } static int dpaa2_rc_ni_get_qdid(device_t dev, device_t child, struct dpaa2_cmd *cmd, enum dpaa2_ni_queue_type type, uint16_t *qdid) { struct __packed get_qdid_args { uint8_t queue_type; } *args; struct __packed get_qdid_resp { uint16_t qdid; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || qdid == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_qdid_args *) &cmd->params[0]; args->queue_type = (uint8_t) type; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_QDID); if (!error) { resp = (struct get_qdid_resp *) &cmd->params[0]; *qdid = resp->qdid; } return (error); } static int dpaa2_rc_ni_add_mac_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct __packed add_mac_args { uint8_t flags; uint8_t _reserved; uint8_t mac[ETHER_ADDR_LEN]; uint8_t tc_id; uint8_t fq_id; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct add_mac_args *) &cmd->params[0]; for (int i = 1; i <= ETHER_ADDR_LEN; i++) args->mac[i - 1] = mac[ETHER_ADDR_LEN - i]; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_ADD_MAC_ADDR)); } static int dpaa2_rc_ni_remove_mac_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct __packed rem_mac_args { uint16_t _reserved; uint8_t mac[ETHER_ADDR_LEN]; uint64_t _reserved1[6]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct rem_mac_args *) &cmd->params[0]; for (int i = 1; i <= ETHER_ADDR_LEN; i++) args->mac[i - 1] = mac[ETHER_ADDR_LEN - i]; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_REMOVE_MAC_ADDR)); } static int dpaa2_rc_ni_clear_mac_filters(device_t dev, device_t child, struct dpaa2_cmd *cmd, bool rm_uni, bool rm_multi) { struct __packed clear_mac_filters_args { uint8_t flags; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct clear_mac_filters_args *) &cmd->params[0]; args->flags |= rm_uni ? 0x1 : 0x0; args->flags |= rm_multi ? 0x2 : 0x0; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_CLEAR_MAC_FILTERS)); } static int dpaa2_rc_ni_set_mfl(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint16_t length) { struct __packed set_mfl_args { uint16_t length; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_mfl_args *) &cmd->params[0]; args->length = length; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_MFL)); } static int dpaa2_rc_ni_set_offload(device_t dev, device_t child, struct dpaa2_cmd *cmd, enum dpaa2_ni_ofl_type ofl_type, bool en) { struct __packed set_ofl_args { uint8_t _reserved[3]; uint8_t ofl_type; uint32_t config; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_ofl_args *) &cmd->params[0]; args->ofl_type = (uint8_t) ofl_type; args->config = en ? 1u : 0u; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_OFFLOAD)); } static int dpaa2_rc_ni_set_irq_mask(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t mask) { struct __packed set_irq_mask_args { uint32_t mask; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_mask_args *) &cmd->params[0]; args->mask = mask; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_IRQ_MASK)); } static int dpaa2_rc_ni_set_irq_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, bool en) { struct __packed set_irq_enable_args { uint32_t en; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_enable_args *) &cmd->params[0]; args->en = en ? 1u : 0u; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_IRQ_ENABLE)); } static int dpaa2_rc_ni_get_irq_status(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t *status) { struct __packed get_irq_stat_args { uint32_t status; uint8_t irq_idx; } *args; struct __packed get_irq_stat_resp { uint32_t status; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || status == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_irq_stat_args *) &cmd->params[0]; args->status = *status; args->irq_idx = irq_idx; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_IRQ_STATUS); if (!error) { resp = (struct get_irq_stat_resp *) &cmd->params[0]; *status = resp->status; } return (error); } static int dpaa2_rc_ni_set_uni_promisc(device_t dev, device_t child, struct dpaa2_cmd *cmd, bool en) { struct __packed set_uni_promisc_args { uint8_t en; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_uni_promisc_args *) &cmd->params[0]; args->en = en ? 1u : 0u; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_UNI_PROMISC)); } static int dpaa2_rc_ni_set_multi_promisc(device_t dev, device_t child, struct dpaa2_cmd *cmd, bool en) { /* TODO: Implementation is the same as for ni_set_uni_promisc(). */ struct __packed set_multi_promisc_args { uint8_t en; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_multi_promisc_args *) &cmd->params[0]; args->en = en ? 1u : 0u; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_MULTI_PROMISC)); } static int dpaa2_rc_ni_get_statistics(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t page, uint16_t param, uint64_t *cnt) { struct __packed get_statistics_args { uint8_t page; uint16_t param; } *args; struct __packed get_statistics_resp { uint64_t cnt[7]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || cnt == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_statistics_args *) &cmd->params[0]; args->page = page; args->param = param; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_GET_STATISTICS); if (!error) { resp = (struct get_statistics_resp *) &cmd->params[0]; for (int i = 0; i < DPAA2_NI_STAT_COUNTERS; i++) cnt[i] = resp->cnt[i]; } return (error); } static int dpaa2_rc_ni_set_rx_tc_dist(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint16_t dist_size, uint8_t tc, enum dpaa2_ni_dist_mode dist_mode, bus_addr_t key_cfg_buf) { struct __packed set_rx_tc_dist_args { uint16_t dist_size; uint8_t tc; uint8_t ma_dm; /* miss action + dist. mode */ uint32_t _reserved1; uint64_t _reserved2[5]; uint64_t key_cfg_iova; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_rx_tc_dist_args *) &cmd->params[0]; args->dist_size = dist_size; args->tc = tc; args->ma_dm = ((uint8_t) dist_mode) & 0x0Fu; args->key_cfg_iova = key_cfg_buf; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_NI_SET_RX_TC_DIST)); } static int dpaa2_rc_io_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpio_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpio_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_io_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_CLOSE)); } static int dpaa2_rc_io_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_ENABLE)); } static int dpaa2_rc_io_disable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_DISABLE)); } static int dpaa2_rc_io_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_RESET)); } static int dpaa2_rc_io_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_io_attr *attr) { struct __packed dpaa2_io_attr { uint32_t id; uint16_t swp_id; uint8_t priors_num; uint8_t chan_mode; uint64_t swp_ce_paddr; uint64_t swp_ci_paddr; uint32_t swp_version; uint32_t _reserved1; uint32_t swp_clk; uint32_t _reserved2[5]; } *pattr; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_GET_ATTR); if (!error) { pattr = (struct dpaa2_io_attr *) &cmd->params[0]; attr->swp_ce_paddr = pattr->swp_ce_paddr; attr->swp_ci_paddr = pattr->swp_ci_paddr; attr->swp_version = pattr->swp_version; attr->swp_clk = pattr->swp_clk; attr->id = pattr->id; attr->swp_id = pattr->swp_id; attr->priors_num = pattr->priors_num; attr->chan_mode = (enum dpaa2_io_chan_mode) pattr->chan_mode; } return (error); } static int dpaa2_rc_io_set_irq_mask(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t mask) { /* TODO: Extract similar *_set_irq_mask() into one function. */ struct __packed set_irq_mask_args { uint32_t mask; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_mask_args *) &cmd->params[0]; args->mask = mask; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_SET_IRQ_MASK)); } static int dpaa2_rc_io_get_irq_status(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t *status) { /* TODO: Extract similar *_get_irq_status() into one function. */ struct __packed get_irq_stat_args { uint32_t status; uint8_t irq_idx; } *args; struct __packed get_irq_stat_resp { uint32_t status; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || status == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_irq_stat_args *) &cmd->params[0]; args->status = *status; args->irq_idx = irq_idx; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_GET_IRQ_STATUS); if (!error) { resp = (struct get_irq_stat_resp *) &cmd->params[0]; *status = resp->status; } return (error); } static int dpaa2_rc_io_set_irq_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, bool en) { /* TODO: Extract similar *_set_irq_enable() into one function. */ struct __packed set_irq_enable_args { uint32_t en; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_enable_args *) &cmd->params[0]; args->en = en ? 1u : 0u; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_SET_IRQ_ENABLE)); } static int dpaa2_rc_io_add_static_dq_chan(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpcon_id, uint8_t *chan_idx) { struct __packed add_static_dq_chan_args { uint32_t dpcon_id; } *args; struct __packed add_static_dq_chan_resp { uint8_t chan_idx; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || chan_idx == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct add_static_dq_chan_args *) &cmd->params[0]; args->dpcon_id = dpcon_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_IO_ADD_STATIC_DQ_CHAN); if (!error) { resp = (struct add_static_dq_chan_resp *) &cmd->params[0]; *chan_idx = resp->chan_idx; } return (error); } static int dpaa2_rc_bp_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpbp_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpbp_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_bp_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_CLOSE)); } static int dpaa2_rc_bp_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_ENABLE)); } static int dpaa2_rc_bp_disable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_DISABLE)); } static int dpaa2_rc_bp_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_RESET)); } static int dpaa2_rc_bp_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_bp_attr *attr) { struct __packed dpaa2_bp_attr { uint16_t _reserved1; uint16_t bpid; uint32_t id; } *pattr; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_BP_GET_ATTR); if (!error) { pattr = (struct dpaa2_bp_attr *) &cmd->params[0]; attr->id = pattr->id; attr->bpid = pattr->bpid; } return (error); } static int dpaa2_rc_mac_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpmac_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpmac_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_mac_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_CLOSE)); } static int dpaa2_rc_mac_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_RESET)); } static int dpaa2_rc_mac_mdio_read(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t phy, uint16_t reg, uint16_t *val) { struct __packed mdio_read_args { uint8_t clause; /* set to 0 by default */ uint8_t phy; uint16_t reg; uint32_t _reserved1; uint64_t _reserved2[6]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || val == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct mdio_read_args *) &cmd->params[0]; args->phy = phy; args->reg = reg; args->clause = 0; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_MDIO_READ); if (!error) *val = cmd->params[0] & 0xFFFF; return (error); } static int dpaa2_rc_mac_mdio_write(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t phy, uint16_t reg, uint16_t val) { struct __packed mdio_write_args { uint8_t clause; /* set to 0 by default */ uint8_t phy; uint16_t reg; uint16_t val; uint16_t _reserved1; uint64_t _reserved2[6]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct mdio_write_args *) &cmd->params[0]; args->phy = phy; args->reg = reg; args->val = val; args->clause = 0; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_MDIO_WRITE)); } static int dpaa2_rc_mac_get_addr(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t *mac) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || mac == NULL) return (DPAA2_CMD_STAT_ERR); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_GET_ADDR); if (!error) { mac[0] = (cmd->params[0] >> 56) & 0xFFU; mac[1] = (cmd->params[0] >> 48) & 0xFFU; mac[2] = (cmd->params[0] >> 40) & 0xFFU; mac[3] = (cmd->params[0] >> 32) & 0xFFU; mac[4] = (cmd->params[0] >> 24) & 0xFFU; mac[5] = (cmd->params[0] >> 16) & 0xFFU; } return (error); } static int dpaa2_rc_mac_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_mac_attr *attr) { struct __packed mac_attr_resp { uint8_t eth_if; uint8_t link_type; uint16_t id; uint32_t max_rate; uint8_t fec_mode; uint8_t ifg_mode; uint8_t ifg_len; uint8_t _reserved1; uint32_t _reserved2; uint8_t sgn_post_pre; uint8_t serdes_cfg_mode; uint8_t eq_amp_red; uint8_t eq_post1q; uint8_t eq_preq; uint8_t eq_type; uint16_t _reserved3; uint64_t _reserved[4]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_EINVAL); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_GET_ATTR); if (!error) { resp = (struct mac_attr_resp *) &cmd->params[0]; attr->id = resp->id; attr->max_rate = resp->max_rate; attr->eth_if = resp->eth_if; attr->link_type = resp->link_type; } return (error); } static int dpaa2_rc_mac_set_link_state(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_mac_link_state *state) { struct __packed mac_set_link_args { uint64_t options; uint32_t rate; uint32_t _reserved1; uint32_t flags; uint32_t _reserved2; uint64_t supported; uint64_t advert; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || state == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct mac_set_link_args *) &cmd->params[0]; args->options = state->options; args->rate = state->rate; args->supported = state->supported; args->advert = state->advert; args->flags |= state->up ? 0x1u : 0u; args->flags |= state->state_valid ? 0x2u : 0u; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_SET_LINK_STATE)); } static int dpaa2_rc_mac_set_irq_mask(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t mask) { /* TODO: Implementation is the same as for ni_set_irq_mask(). */ struct __packed set_irq_mask_args { uint32_t mask; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_mask_args *) &cmd->params[0]; args->mask = mask; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_SET_IRQ_MASK)); } static int dpaa2_rc_mac_set_irq_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, bool en) { /* TODO: Implementation is the same as for ni_set_irq_enable(). */ struct __packed set_irq_enable_args { uint32_t en; uint8_t irq_idx; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct set_irq_enable_args *) &cmd->params[0]; args->en = en ? 1u : 0u; args->irq_idx = irq_idx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_SET_IRQ_ENABLE)); } static int dpaa2_rc_mac_get_irq_status(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint8_t irq_idx, uint32_t *status) { /* TODO: Implementation is the same as ni_get_irq_status(). */ struct __packed get_irq_stat_args { uint32_t status; uint8_t irq_idx; } *args; struct __packed get_irq_stat_resp { uint32_t status; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || status == NULL) return (DPAA2_CMD_STAT_EINVAL); dpaa2_rc_reset_cmd_params(cmd); args = (struct get_irq_stat_args *) &cmd->params[0]; args->status = *status; args->irq_idx = irq_idx; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MAC_GET_IRQ_STATUS); if (!error) { resp = (struct get_irq_stat_resp *) &cmd->params[0]; *status = resp->status; } return (error); } static int dpaa2_rc_con_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpcon_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpcon_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_con_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_CLOSE)); } static int dpaa2_rc_con_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_RESET)); } static int dpaa2_rc_con_enable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_ENABLE)); } static int dpaa2_rc_con_disable(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_DISABLE)); } static int dpaa2_rc_con_get_attributes(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_con_attr *attr) { struct __packed con_attr_resp { uint32_t id; uint16_t chan_id; uint8_t prior_num; uint8_t _reserved1; uint64_t _reserved2[6]; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || attr == NULL) return (DPAA2_CMD_STAT_EINVAL); error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_GET_ATTR); if (!error) { resp = (struct con_attr_resp *) &cmd->params[0]; attr->id = resp->id; attr->chan_id = resp->chan_id; attr->prior_num = resp->prior_num; } return (error); } static int dpaa2_rc_con_set_notif(device_t dev, device_t child, struct dpaa2_cmd *cmd, struct dpaa2_con_notif_cfg *cfg) { struct __packed set_notif_args { uint32_t dpio_id; uint8_t prior; uint8_t _reserved1; uint16_t _reserved2; uint64_t ctx; uint64_t _reserved3[5]; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL || cfg == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct set_notif_args *) &cmd->params[0]; args->dpio_id = cfg->dpio_id; args->prior = cfg->prior; args->ctx = cfg->qman_ctx; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_CON_SET_NOTIF)); } static int dpaa2_rc_mcp_create(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t portal_id, uint32_t options, uint32_t *dpmcp_id) { struct __packed mcp_create_args { uint32_t portal_id; uint32_t options; uint64_t _reserved[6]; } *args; struct __packed mcp_create_resp { uint32_t dpmcp_id; } *resp; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); int error; if (portal == NULL || cmd == NULL || dpmcp_id == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct mcp_create_args *) &cmd->params[0]; args->portal_id = portal_id; args->options = options; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MCP_CREATE); if (!error) { resp = (struct mcp_create_resp *) &cmd->params[0]; *dpmcp_id = resp->dpmcp_id; } return (error); } static int dpaa2_rc_mcp_destroy(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpmcp_id) { struct __packed mcp_destroy_args { uint32_t dpmcp_id; } *args; struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); args = (struct mcp_destroy_args *) &cmd->params[0]; args->dpmcp_id = dpmcp_id; return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MCP_DESTROY)); } static int dpaa2_rc_mcp_open(device_t dev, device_t child, struct dpaa2_cmd *cmd, uint32_t dpmcp_id, uint16_t *token) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); struct dpaa2_cmd_header *hdr; int error; if (portal == NULL || cmd == NULL || token == NULL) return (DPAA2_CMD_STAT_ERR); cmd->params[0] = dpmcp_id; error = dpaa2_rc_exec_cmd(portal, cmd, CMDID_MCP_OPEN); if (!error) { hdr = (struct dpaa2_cmd_header *) &cmd->header; *token = hdr->token; } return (error); } static int dpaa2_rc_mcp_close(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MCP_CLOSE)); } static int dpaa2_rc_mcp_reset(device_t dev, device_t child, struct dpaa2_cmd *cmd) { struct dpaa2_mcp *portal = dpaa2_rc_select_portal(dev, child); if (portal == NULL || cmd == NULL) return (DPAA2_CMD_STAT_ERR); return (dpaa2_rc_exec_cmd(portal, cmd, CMDID_MCP_RESET)); } /** * @brief Create and add devices for DPAA2 objects in this resource container. */ static int dpaa2_rc_discover(struct dpaa2_rc_softc *sc) { device_t rcdev = sc->dev; device_t child = sc->dev; struct dpaa2_devinfo *rcinfo = device_get_ivars(rcdev); struct dpaa2_cmd cmd; struct dpaa2_rc_attr dprc_attr; struct dpaa2_obj obj; uint32_t major, minor, rev, obj_count; uint16_t rc_token; int rc; DPAA2_CMD_INIT(&cmd); /* Print MC firmware version. */ rc = DPAA2_CMD_MNG_GET_VERSION(rcdev, child, &cmd, &major, &minor, &rev); if (rc) { device_printf(rcdev, "%s: failed to get MC firmware version: " "error=%d\n", __func__, rc); return (ENXIO); } device_printf(rcdev, "MC firmware version: %u.%u.%u\n", major, minor, rev); /* Obtain container ID associated with a given MC portal. */ rc = DPAA2_CMD_MNG_GET_CONTAINER_ID(rcdev, child, &cmd, &sc->cont_id); if (rc) { device_printf(rcdev, "%s: failed to get container id: " "error=%d\n", __func__, rc); return (ENXIO); } if (bootverbose) { device_printf(rcdev, "Resource container ID: %u\n", sc->cont_id); } /* Open the resource container. */ rc = DPAA2_CMD_RC_OPEN(rcdev, child, &cmd, sc->cont_id, &rc_token); if (rc) { device_printf(rcdev, "%s: failed to open container: cont_id=%u, " "error=%d\n", __func__, sc->cont_id, rc); return (ENXIO); } /* Obtain a number of objects in this container. */ rc = DPAA2_CMD_RC_GET_OBJ_COUNT(rcdev, child, &cmd, &obj_count); if (rc) { device_printf(rcdev, "%s: failed to count objects in container: " "cont_id=%u, error=%d\n", __func__, sc->cont_id, rc); (void)DPAA2_CMD_RC_CLOSE(rcdev, child, &cmd); return (ENXIO); } if (bootverbose) { device_printf(rcdev, "Objects in container: %u\n", obj_count); } rc = DPAA2_CMD_RC_GET_ATTRIBUTES(rcdev, child, &cmd, &dprc_attr); if (rc) { device_printf(rcdev, "%s: failed to get attributes of the " "container: cont_id=%u, error=%d\n", __func__, sc->cont_id, rc); DPAA2_CMD_RC_CLOSE(rcdev, child, &cmd); return (ENXIO); } if (bootverbose) { device_printf(rcdev, "Isolation context ID: %u\n", dprc_attr.icid); } if (rcinfo) { rcinfo->id = dprc_attr.cont_id; rcinfo->portal_id = dprc_attr.portal_id; rcinfo->icid = dprc_attr.icid; } /* * Add MC portals before everything else. * TODO: Discover DPAA2 objects on-demand. */ for (uint32_t i = 0; i < obj_count; i++) { rc = DPAA2_CMD_RC_GET_OBJ(rcdev, child, &cmd, i, &obj); if (rc) { continue; /* Skip silently for now. */ } if (obj.type != DPAA2_DEV_MCP) { continue; } dpaa2_rc_add_managed_child(sc, &cmd, &obj); } /* Probe and attach MC portals. */ bus_identify_children(rcdev); bus_attach_children(rcdev); /* Add managed devices (except DPMCPs) to the resource container. */ for (uint32_t i = 0; i < obj_count; i++) { rc = DPAA2_CMD_RC_GET_OBJ(rcdev, child, &cmd, i, &obj); if (rc && bootverbose) { if (rc == DPAA2_CMD_STAT_UNKNOWN_OBJ) { device_printf(rcdev, "%s: skip unsupported " "DPAA2 object: idx=%u\n", __func__, i); continue; } else { device_printf(rcdev, "%s: failed to get " "information about DPAA2 object: idx=%u, " "error=%d\n", __func__, i, rc); continue; } } if (obj.type == DPAA2_DEV_MCP) { continue; /* Already added. */ } dpaa2_rc_add_managed_child(sc, &cmd, &obj); } /* Probe and attach managed devices properly. */ bus_identify_children(rcdev); bus_attach_children(rcdev); /* Add other devices to the resource container. */ for (uint32_t i = 0; i < obj_count; i++) { rc = DPAA2_CMD_RC_GET_OBJ(rcdev, child, &cmd, i, &obj); if (rc == DPAA2_CMD_STAT_UNKNOWN_OBJ && bootverbose) { device_printf(rcdev, "%s: skip unsupported DPAA2 " "object: idx=%u\n", __func__, i); continue; } else if (rc) { device_printf(rcdev, "%s: failed to get object: " "idx=%u, error=%d\n", __func__, i, rc); continue; } dpaa2_rc_add_child(sc, &cmd, &obj); } DPAA2_CMD_RC_CLOSE(rcdev, child, &cmd); /* Probe and attach the rest of devices. */ bus_identify_children(rcdev); bus_attach_children(rcdev); return (0); } /** * @brief Add a new DPAA2 device to the resource container bus. */ static int dpaa2_rc_add_child(struct dpaa2_rc_softc *sc, struct dpaa2_cmd *cmd, struct dpaa2_obj *obj) { device_t rcdev, dev; struct dpaa2_devinfo *rcinfo; struct dpaa2_devinfo *dinfo; struct resource_spec *res_spec; const char *devclass; int dpio_n = 0; /* to limit DPIOs by # of CPUs */ int dpcon_n = 0; /* to limit DPCONs by # of CPUs */ int rid, error; rcdev = sc->dev; rcinfo = device_get_ivars(rcdev); switch (obj->type) { case DPAA2_DEV_NI: devclass = "dpaa2_ni"; res_spec = dpaa2_ni_spec; break; default: return (ENXIO); } /* Add a device for the DPAA2 object. */ dev = device_add_child(rcdev, devclass, -1); if (dev == NULL) { device_printf(rcdev, "%s: failed to add a device for DPAA2 " "object: type=%s, id=%u\n", __func__, dpaa2_ttos(obj->type), obj->id); return (ENXIO); } /* Allocate devinfo for a child. */ dinfo = malloc(sizeof(struct dpaa2_devinfo), M_DPAA2_RC, M_WAITOK | M_ZERO); if (!dinfo) { device_printf(rcdev, "%s: failed to allocate dpaa2_devinfo " "for: type=%s, id=%u\n", __func__, dpaa2_ttos(obj->type), obj->id); return (ENXIO); } device_set_ivars(dev, dinfo); dinfo->pdev = rcdev; dinfo->dev = dev; dinfo->id = obj->id; dinfo->dtype = obj->type; dinfo->portal = NULL; /* Children share their parent container's ICID and portal ID. */ dinfo->icid = rcinfo->icid; dinfo->portal_id = rcinfo->portal_id; /* MSI configuration */ dinfo->msi.msi_msgnum = obj->irq_count; dinfo->msi.msi_alloc = 0; dinfo->msi.msi_handlers = 0; /* Initialize a resource list for the child. */ resource_list_init(&dinfo->resources); /* Add DPAA2-specific resources to the resource list. */ for (; res_spec && res_spec->type != -1; res_spec++) { if (res_spec->type < DPAA2_DEV_MC) continue; /* Skip non-DPAA2 resource. */ rid = res_spec->rid; /* Limit DPIOs and DPCONs by number of CPUs. */ if (res_spec->type == DPAA2_DEV_IO && dpio_n >= mp_ncpus) { dpio_n++; continue; } if (res_spec->type == DPAA2_DEV_CON && dpcon_n >= mp_ncpus) { dpcon_n++; continue; } error = dpaa2_rc_add_res(rcdev, dev, res_spec->type, &rid, res_spec->flags); if (error) device_printf(rcdev, "%s: dpaa2_rc_add_res() failed: " "error=%d\n", __func__, error); if (res_spec->type == DPAA2_DEV_IO) dpio_n++; if (res_spec->type == DPAA2_DEV_CON) dpcon_n++; } return (0); } /** * @brief Add a new managed DPAA2 device to the resource container bus. * * There are DPAA2 objects (DPIO, DPBP) which have their own drivers and can be * allocated as resources or associated with the other DPAA2 objects. This * function is supposed to discover such managed objects in the resource * container and add them as children to perform a proper initialization. * * NOTE: It must be called together with bus_identify_children() and * bus_attach_children() before dpaa2_rc_add_child(). */ static int dpaa2_rc_add_managed_child(struct dpaa2_rc_softc *sc, struct dpaa2_cmd *cmd, struct dpaa2_obj *obj) { device_t rcdev, dev, child; struct dpaa2_devinfo *rcinfo, *dinfo; struct dpaa2_rc_obj_region reg; struct resource_spec *res_spec; const char *devclass; uint64_t start, end, count; uint32_t flags = 0; int rid, error; rcdev = sc->dev; child = sc->dev; rcinfo = device_get_ivars(rcdev); switch (obj->type) { case DPAA2_DEV_IO: devclass = "dpaa2_io"; res_spec = dpaa2_io_spec; flags = DPAA2_MC_DEV_ALLOCATABLE | DPAA2_MC_DEV_SHAREABLE; break; case DPAA2_DEV_BP: devclass = "dpaa2_bp"; res_spec = dpaa2_bp_spec; flags = DPAA2_MC_DEV_ALLOCATABLE; break; case DPAA2_DEV_CON: devclass = "dpaa2_con"; res_spec = dpaa2_con_spec; flags = DPAA2_MC_DEV_ALLOCATABLE; break; case DPAA2_DEV_MAC: devclass = "dpaa2_mac"; res_spec = dpaa2_mac_spec; flags = DPAA2_MC_DEV_ASSOCIATED; break; case DPAA2_DEV_MCP: devclass = "dpaa2_mcp"; res_spec = NULL; flags = DPAA2_MC_DEV_ALLOCATABLE | DPAA2_MC_DEV_SHAREABLE; break; default: /* Only managed devices above are supported. */ return (EINVAL); } /* Add a device for the DPAA2 object. */ dev = device_add_child(rcdev, devclass, -1); if (dev == NULL) { device_printf(rcdev, "%s: failed to add a device for DPAA2 " "object: type=%s, id=%u\n", __func__, dpaa2_ttos(obj->type), obj->id); return (ENXIO); } /* Allocate devinfo for the child. */ dinfo = malloc(sizeof(struct dpaa2_devinfo), M_DPAA2_RC, M_WAITOK | M_ZERO); if (!dinfo) { device_printf(rcdev, "%s: failed to allocate dpaa2_devinfo " "for: type=%s, id=%u\n", __func__, dpaa2_ttos(obj->type), obj->id); return (ENXIO); } device_set_ivars(dev, dinfo); dinfo->pdev = rcdev; dinfo->dev = dev; dinfo->id = obj->id; dinfo->dtype = obj->type; dinfo->portal = NULL; /* Children share their parent container's ICID and portal ID. */ dinfo->icid = rcinfo->icid; dinfo->portal_id = rcinfo->portal_id; /* MSI configuration */ dinfo->msi.msi_msgnum = obj->irq_count; dinfo->msi.msi_alloc = 0; dinfo->msi.msi_handlers = 0; /* Initialize a resource list for the child. */ resource_list_init(&dinfo->resources); /* Add memory regions to the resource list. */ for (uint8_t i = 0; i < obj->reg_count; i++) { error = DPAA2_CMD_RC_GET_OBJ_REGION(rcdev, child, cmd, obj->id, i, obj->type, ®); if (error) { device_printf(rcdev, "%s: failed to obtain memory " "region for type=%s, id=%u, reg_idx=%u: error=%d\n", __func__, dpaa2_ttos(obj->type), obj->id, i, error); continue; } count = reg.size; start = reg.base_paddr + reg.base_offset; end = reg.base_paddr + reg.base_offset + reg.size - 1; resource_list_add(&dinfo->resources, SYS_RES_MEMORY, i, start, end, count); } /* Add DPAA2-specific resources to the resource list. */ for (; res_spec && res_spec->type != -1; res_spec++) { if (res_spec->type < DPAA2_DEV_MC) continue; /* Skip non-DPAA2 resource. */ rid = res_spec->rid; error = dpaa2_rc_add_res(rcdev, dev, res_spec->type, &rid, res_spec->flags); if (error) device_printf(rcdev, "%s: dpaa2_rc_add_res() failed: " "error=%d\n", __func__, error); } /* Inform MC about a new managed device. */ error = DPAA2_MC_MANAGE_DEV(rcdev, dev, flags); if (error) { device_printf(rcdev, "%s: failed to add a managed DPAA2 device: " "type=%s, id=%u, error=%d\n", __func__, dpaa2_ttos(obj->type), obj->id, error); return (ENXIO); } return (0); } /** * @brief Configure given IRQ using MC command interface. */ static int dpaa2_rc_configure_irq(device_t rcdev, device_t child, int rid, uint64_t addr, uint32_t data) { struct dpaa2_devinfo *rcinfo; struct dpaa2_devinfo *dinfo; struct dpaa2_cmd cmd; uint16_t rc_token; int rc = EINVAL; DPAA2_CMD_INIT(&cmd); if (device_get_parent(child) == rcdev && rid >= 1) { rcinfo = device_get_ivars(rcdev); dinfo = device_get_ivars(child); rc = DPAA2_CMD_RC_OPEN(rcdev, child, &cmd, rcinfo->id, &rc_token); if (rc) { device_printf(rcdev, "%s: failed to open DPRC: " "error=%d\n", __func__, rc); return (ENODEV); } /* Set MSI address and value. */ rc = DPAA2_CMD_RC_SET_OBJ_IRQ(rcdev, child, &cmd, rid - 1, addr, data, rid, dinfo->id, dinfo->dtype); if (rc) { device_printf(rcdev, "%s: failed to setup IRQ: " "rid=%d, addr=%jx, data=%x, error=%d\n", __func__, rid, addr, data, rc); return (ENODEV); } rc = DPAA2_CMD_RC_CLOSE(rcdev, child, &cmd); if (rc) { device_printf(rcdev, "%s: failed to close DPRC: " "error=%d\n", __func__, rc); return (ENODEV); } rc = 0; } return (rc); } /** * @brief General implementation of the MC command to enable IRQ. */ static int dpaa2_rc_enable_irq(struct dpaa2_mcp *mcp, struct dpaa2_cmd *cmd, uint8_t irq_idx, bool enable, uint16_t cmdid) { struct __packed enable_irq_args { uint8_t enable; uint8_t _reserved1; uint16_t _reserved2; uint8_t irq_idx; uint8_t _reserved3; uint16_t _reserved4; uint64_t _reserved5[6]; } *args; if (!mcp || !cmd) return (DPAA2_CMD_STAT_ERR); args = (struct enable_irq_args *) &cmd->params[0]; args->irq_idx = irq_idx; args->enable = enable == 0u ? 0u : 1u; return (dpaa2_rc_exec_cmd(mcp, cmd, cmdid)); } /** * @brief Sends a command to MC and waits for response. */ static int dpaa2_rc_exec_cmd(struct dpaa2_mcp *mcp, struct dpaa2_cmd *cmd, uint16_t cmdid) { struct dpaa2_cmd_header *hdr; uint16_t flags; int error; if (!mcp || !cmd) return (DPAA2_CMD_STAT_ERR); /* Prepare a command for the MC hardware. */ hdr = (struct dpaa2_cmd_header *) &cmd->header; hdr->cmdid = cmdid; hdr->status = DPAA2_CMD_STAT_READY; DPAA2_MCP_LOCK(mcp, &flags); if (flags & DPAA2_PORTAL_DESTROYED) { /* Terminate operation if portal is destroyed. */ DPAA2_MCP_UNLOCK(mcp); return (DPAA2_CMD_STAT_INVALID_STATE); } /* Send a command to MC and wait for the result. */ dpaa2_rc_send_cmd(mcp, cmd); error = dpaa2_rc_wait_for_cmd(mcp, cmd); if (error) { DPAA2_MCP_UNLOCK(mcp); return (DPAA2_CMD_STAT_ERR); } if (hdr->status != DPAA2_CMD_STAT_OK) { DPAA2_MCP_UNLOCK(mcp); return (int)(hdr->status); } DPAA2_MCP_UNLOCK(mcp); return (DPAA2_CMD_STAT_OK); } /** * @brief Writes a command to the MC command portal. */ static int dpaa2_rc_send_cmd(struct dpaa2_mcp *mcp, struct dpaa2_cmd *cmd) { /* Write command parameters. */ for (uint32_t i = 1; i <= DPAA2_CMD_PARAMS_N; i++) bus_write_8(mcp->map, sizeof(uint64_t) * i, cmd->params[i-1]); bus_barrier(mcp->map, 0, sizeof(struct dpaa2_cmd), BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); /* Write command header to trigger execution. */ bus_write_8(mcp->map, 0, cmd->header); return (0); } /** * @brief Polls the MC command portal in order to receive a result of the * command execution. */ static int dpaa2_rc_wait_for_cmd(struct dpaa2_mcp *mcp, struct dpaa2_cmd *cmd) { struct dpaa2_cmd_header *hdr; uint64_t val; uint32_t i; /* Wait for a command execution result from the MC hardware. */ for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) { val = bus_read_8(mcp->map, 0); hdr = (struct dpaa2_cmd_header *) &val; if (hdr->status != DPAA2_CMD_STAT_READY) { break; } DELAY(CMD_SPIN_TIMEOUT); } if (i > CMD_SPIN_ATTEMPTS) { /* Return an error on expired timeout. */ return (DPAA2_CMD_STAT_TIMEOUT); } else { /* Read command response. */ cmd->header = val; for (i = 1; i <= DPAA2_CMD_PARAMS_N; i++) { cmd->params[i-1] = bus_read_8(mcp->map, i * sizeof(uint64_t)); } } return (DPAA2_CMD_STAT_OK); } /** * @brief Reserve a DPAA2-specific device of the given devtype for the child. */ static int dpaa2_rc_add_res(device_t rcdev, device_t child, enum dpaa2_dev_type devtype, int *rid, int flags) { device_t dpaa2_dev; struct dpaa2_devinfo *dinfo = device_get_ivars(child); struct resource *res; bool shared = false; int error; /* Request a free DPAA2 device of the given type from MC. */ error = DPAA2_MC_GET_FREE_DEV(rcdev, &dpaa2_dev, devtype); if (error && !(flags & RF_SHAREABLE)) { device_printf(rcdev, "%s: failed to obtain a free %s (rid=%d) " "for: %s (id=%u)\n", __func__, dpaa2_ttos(devtype), *rid, dpaa2_ttos(dinfo->dtype), dinfo->id); return (error); } /* Request a shared DPAA2 device of the given type from MC. */ if (error) { error = DPAA2_MC_GET_SHARED_DEV(rcdev, &dpaa2_dev, devtype); if (error) { device_printf(rcdev, "%s: failed to obtain a shared " "%s (rid=%d) for: %s (id=%u)\n", __func__, dpaa2_ttos(devtype), *rid, dpaa2_ttos(dinfo->dtype), dinfo->id); return (error); } shared = true; } /* Add DPAA2 device to the resource list of the child device. */ resource_list_add(&dinfo->resources, devtype, *rid, (rman_res_t) dpaa2_dev, (rman_res_t) dpaa2_dev, 1); /* Reserve a newly added DPAA2 resource. */ res = resource_list_reserve(&dinfo->resources, rcdev, child, devtype, rid, (rman_res_t) dpaa2_dev, (rman_res_t) dpaa2_dev, 1, flags & ~RF_ACTIVE); if (!res) { device_printf(rcdev, "%s: failed to reserve %s (rid=%d) for: %s " "(id=%u)\n", __func__, dpaa2_ttos(devtype), *rid, dpaa2_ttos(dinfo->dtype), dinfo->id); return (EBUSY); } /* Reserve a shared DPAA2 device of the given type. */ if (shared) { error = DPAA2_MC_RESERVE_DEV(rcdev, dpaa2_dev, devtype); if (error) { device_printf(rcdev, "%s: failed to reserve a shared " "%s (rid=%d) for: %s (id=%u)\n", __func__, dpaa2_ttos(devtype), *rid, dpaa2_ttos(dinfo->dtype), dinfo->id); return (error); } } return (0); } static int dpaa2_rc_print_type(struct resource_list *rl, enum dpaa2_dev_type type) { struct dpaa2_devinfo *dinfo; struct resource_list_entry *rle; uint32_t prev_id; int printed = 0, series = 0; int retval = 0; STAILQ_FOREACH(rle, rl, link) { if (rle->type == type) { dinfo = device_get_ivars((device_t) rle->start); if (printed == 0) { retval += printf(" %s (id=", dpaa2_ttos(dinfo->dtype)); } else { if (dinfo->id == prev_id + 1) { if (series == 0) { series = 1; retval += printf("-"); } } else { if (series == 1) { retval += printf("%u", prev_id); series = 0; } retval += printf(","); } } printed++; if (series == 0) retval += printf("%u", dinfo->id); prev_id = dinfo->id; } } if (printed) { if (series == 1) retval += printf("%u", prev_id); retval += printf(")"); } return (retval); } static int dpaa2_rc_reset_cmd_params(struct dpaa2_cmd *cmd) { if (cmd != NULL) { memset(cmd->params, 0, sizeof(cmd->params[0]) * DPAA2_CMD_PARAMS_N); } return (0); } static struct dpaa2_mcp * dpaa2_rc_select_portal(device_t dev, device_t child) { struct dpaa2_devinfo *dinfo = device_get_ivars(dev); struct dpaa2_devinfo *cinfo = device_get_ivars(child); if (cinfo == NULL || dinfo == NULL || dinfo->dtype != DPAA2_DEV_RC) return (NULL); return (cinfo->portal != NULL ? cinfo->portal : dinfo->portal); } static device_method_t dpaa2_rc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, dpaa2_rc_probe), DEVMETHOD(device_attach, dpaa2_rc_attach), DEVMETHOD(device_detach, dpaa2_rc_detach), /* Bus interface */ DEVMETHOD(bus_get_resource_list, dpaa2_rc_get_resource_list), DEVMETHOD(bus_delete_resource, dpaa2_rc_delete_resource), DEVMETHOD(bus_alloc_resource, dpaa2_rc_alloc_resource), DEVMETHOD(bus_release_resource, dpaa2_rc_release_resource), DEVMETHOD(bus_child_deleted, dpaa2_rc_child_deleted), DEVMETHOD(bus_child_detached, dpaa2_rc_child_detached), DEVMETHOD(bus_setup_intr, dpaa2_rc_setup_intr), DEVMETHOD(bus_teardown_intr, dpaa2_rc_teardown_intr), DEVMETHOD(bus_print_child, dpaa2_rc_print_child), DEVMETHOD(bus_add_child, device_add_child_ordered), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), /* Pseudo-PCI interface */ DEVMETHOD(pci_alloc_msi, dpaa2_rc_alloc_msi), DEVMETHOD(pci_release_msi, dpaa2_rc_release_msi), DEVMETHOD(pci_msi_count, dpaa2_rc_msi_count), DEVMETHOD(pci_get_id, dpaa2_rc_get_id), /* DPAA2 MC command interface */ DEVMETHOD(dpaa2_cmd_mng_get_version, dpaa2_rc_mng_get_version), DEVMETHOD(dpaa2_cmd_mng_get_soc_version, dpaa2_rc_mng_get_soc_version), DEVMETHOD(dpaa2_cmd_mng_get_container_id, dpaa2_rc_mng_get_container_id), /* DPRC commands */ DEVMETHOD(dpaa2_cmd_rc_open, dpaa2_rc_open), DEVMETHOD(dpaa2_cmd_rc_close, dpaa2_rc_close), DEVMETHOD(dpaa2_cmd_rc_get_obj_count, dpaa2_rc_get_obj_count), DEVMETHOD(dpaa2_cmd_rc_get_obj, dpaa2_rc_get_obj), DEVMETHOD(dpaa2_cmd_rc_get_obj_descriptor, dpaa2_rc_get_obj_descriptor), DEVMETHOD(dpaa2_cmd_rc_get_attributes, dpaa2_rc_get_attributes), DEVMETHOD(dpaa2_cmd_rc_get_obj_region, dpaa2_rc_get_obj_region), DEVMETHOD(dpaa2_cmd_rc_get_api_version, dpaa2_rc_get_api_version), DEVMETHOD(dpaa2_cmd_rc_set_irq_enable, dpaa2_rc_set_irq_enable), DEVMETHOD(dpaa2_cmd_rc_set_obj_irq, dpaa2_rc_set_obj_irq), DEVMETHOD(dpaa2_cmd_rc_get_conn, dpaa2_rc_get_conn), /* DPNI commands */ DEVMETHOD(dpaa2_cmd_ni_open, dpaa2_rc_ni_open), DEVMETHOD(dpaa2_cmd_ni_close, dpaa2_rc_ni_close), DEVMETHOD(dpaa2_cmd_ni_enable, dpaa2_rc_ni_enable), DEVMETHOD(dpaa2_cmd_ni_disable, dpaa2_rc_ni_disable), DEVMETHOD(dpaa2_cmd_ni_get_api_version, dpaa2_rc_ni_get_api_version), DEVMETHOD(dpaa2_cmd_ni_reset, dpaa2_rc_ni_reset), DEVMETHOD(dpaa2_cmd_ni_get_attributes, dpaa2_rc_ni_get_attributes), DEVMETHOD(dpaa2_cmd_ni_set_buf_layout, dpaa2_rc_ni_set_buf_layout), DEVMETHOD(dpaa2_cmd_ni_get_tx_data_off, dpaa2_rc_ni_get_tx_data_offset), DEVMETHOD(dpaa2_cmd_ni_get_port_mac_addr, dpaa2_rc_ni_get_port_mac_addr), DEVMETHOD(dpaa2_cmd_ni_set_prim_mac_addr, dpaa2_rc_ni_set_prim_mac_addr), DEVMETHOD(dpaa2_cmd_ni_get_prim_mac_addr, dpaa2_rc_ni_get_prim_mac_addr), DEVMETHOD(dpaa2_cmd_ni_set_link_cfg, dpaa2_rc_ni_set_link_cfg), DEVMETHOD(dpaa2_cmd_ni_get_link_cfg, dpaa2_rc_ni_get_link_cfg), DEVMETHOD(dpaa2_cmd_ni_get_link_state, dpaa2_rc_ni_get_link_state), DEVMETHOD(dpaa2_cmd_ni_set_qos_table, dpaa2_rc_ni_set_qos_table), DEVMETHOD(dpaa2_cmd_ni_clear_qos_table, dpaa2_rc_ni_clear_qos_table), DEVMETHOD(dpaa2_cmd_ni_set_pools, dpaa2_rc_ni_set_pools), DEVMETHOD(dpaa2_cmd_ni_set_err_behavior,dpaa2_rc_ni_set_err_behavior), DEVMETHOD(dpaa2_cmd_ni_get_queue, dpaa2_rc_ni_get_queue), DEVMETHOD(dpaa2_cmd_ni_set_queue, dpaa2_rc_ni_set_queue), DEVMETHOD(dpaa2_cmd_ni_get_qdid, dpaa2_rc_ni_get_qdid), DEVMETHOD(dpaa2_cmd_ni_add_mac_addr, dpaa2_rc_ni_add_mac_addr), DEVMETHOD(dpaa2_cmd_ni_remove_mac_addr, dpaa2_rc_ni_remove_mac_addr), DEVMETHOD(dpaa2_cmd_ni_clear_mac_filters, dpaa2_rc_ni_clear_mac_filters), DEVMETHOD(dpaa2_cmd_ni_set_mfl, dpaa2_rc_ni_set_mfl), DEVMETHOD(dpaa2_cmd_ni_set_offload, dpaa2_rc_ni_set_offload), DEVMETHOD(dpaa2_cmd_ni_set_irq_mask, dpaa2_rc_ni_set_irq_mask), DEVMETHOD(dpaa2_cmd_ni_set_irq_enable, dpaa2_rc_ni_set_irq_enable), DEVMETHOD(dpaa2_cmd_ni_get_irq_status, dpaa2_rc_ni_get_irq_status), DEVMETHOD(dpaa2_cmd_ni_set_uni_promisc, dpaa2_rc_ni_set_uni_promisc), DEVMETHOD(dpaa2_cmd_ni_set_multi_promisc, dpaa2_rc_ni_set_multi_promisc), DEVMETHOD(dpaa2_cmd_ni_get_statistics, dpaa2_rc_ni_get_statistics), DEVMETHOD(dpaa2_cmd_ni_set_rx_tc_dist, dpaa2_rc_ni_set_rx_tc_dist), /* DPIO commands */ DEVMETHOD(dpaa2_cmd_io_open, dpaa2_rc_io_open), DEVMETHOD(dpaa2_cmd_io_close, dpaa2_rc_io_close), DEVMETHOD(dpaa2_cmd_io_enable, dpaa2_rc_io_enable), DEVMETHOD(dpaa2_cmd_io_disable, dpaa2_rc_io_disable), DEVMETHOD(dpaa2_cmd_io_reset, dpaa2_rc_io_reset), DEVMETHOD(dpaa2_cmd_io_get_attributes, dpaa2_rc_io_get_attributes), DEVMETHOD(dpaa2_cmd_io_set_irq_mask, dpaa2_rc_io_set_irq_mask), DEVMETHOD(dpaa2_cmd_io_get_irq_status, dpaa2_rc_io_get_irq_status), DEVMETHOD(dpaa2_cmd_io_set_irq_enable, dpaa2_rc_io_set_irq_enable), DEVMETHOD(dpaa2_cmd_io_add_static_dq_chan, dpaa2_rc_io_add_static_dq_chan), /* DPBP commands */ DEVMETHOD(dpaa2_cmd_bp_open, dpaa2_rc_bp_open), DEVMETHOD(dpaa2_cmd_bp_close, dpaa2_rc_bp_close), DEVMETHOD(dpaa2_cmd_bp_enable, dpaa2_rc_bp_enable), DEVMETHOD(dpaa2_cmd_bp_disable, dpaa2_rc_bp_disable), DEVMETHOD(dpaa2_cmd_bp_reset, dpaa2_rc_bp_reset), DEVMETHOD(dpaa2_cmd_bp_get_attributes, dpaa2_rc_bp_get_attributes), /* DPMAC commands */ DEVMETHOD(dpaa2_cmd_mac_open, dpaa2_rc_mac_open), DEVMETHOD(dpaa2_cmd_mac_close, dpaa2_rc_mac_close), DEVMETHOD(dpaa2_cmd_mac_reset, dpaa2_rc_mac_reset), DEVMETHOD(dpaa2_cmd_mac_mdio_read, dpaa2_rc_mac_mdio_read), DEVMETHOD(dpaa2_cmd_mac_mdio_write, dpaa2_rc_mac_mdio_write), DEVMETHOD(dpaa2_cmd_mac_get_addr, dpaa2_rc_mac_get_addr), DEVMETHOD(dpaa2_cmd_mac_get_attributes, dpaa2_rc_mac_get_attributes), DEVMETHOD(dpaa2_cmd_mac_set_link_state, dpaa2_rc_mac_set_link_state), DEVMETHOD(dpaa2_cmd_mac_set_irq_mask, dpaa2_rc_mac_set_irq_mask), DEVMETHOD(dpaa2_cmd_mac_set_irq_enable, dpaa2_rc_mac_set_irq_enable), DEVMETHOD(dpaa2_cmd_mac_get_irq_status, dpaa2_rc_mac_get_irq_status), /* DPCON commands */ DEVMETHOD(dpaa2_cmd_con_open, dpaa2_rc_con_open), DEVMETHOD(dpaa2_cmd_con_close, dpaa2_rc_con_close), DEVMETHOD(dpaa2_cmd_con_reset, dpaa2_rc_con_reset), DEVMETHOD(dpaa2_cmd_con_enable, dpaa2_rc_con_enable), DEVMETHOD(dpaa2_cmd_con_disable, dpaa2_rc_con_disable), DEVMETHOD(dpaa2_cmd_con_get_attributes, dpaa2_rc_con_get_attributes), DEVMETHOD(dpaa2_cmd_con_set_notif, dpaa2_rc_con_set_notif), /* DPMCP commands */ DEVMETHOD(dpaa2_cmd_mcp_create, dpaa2_rc_mcp_create), DEVMETHOD(dpaa2_cmd_mcp_destroy, dpaa2_rc_mcp_destroy), DEVMETHOD(dpaa2_cmd_mcp_open, dpaa2_rc_mcp_open), DEVMETHOD(dpaa2_cmd_mcp_close, dpaa2_rc_mcp_close), DEVMETHOD(dpaa2_cmd_mcp_reset, dpaa2_rc_mcp_reset), DEVMETHOD_END }; static driver_t dpaa2_rc_driver = { "dpaa2_rc", dpaa2_rc_methods, sizeof(struct dpaa2_rc_softc), }; /* For root container */ DRIVER_MODULE(dpaa2_rc, dpaa2_mc, dpaa2_rc_driver, 0, 0); /* For child containers */ DRIVER_MODULE(dpaa2_rc, dpaa2_rc, dpaa2_rc_driver, 0, 0); diff --git a/sys/dev/gpio/gpiobus.c b/sys/dev/gpio/gpiobus.c index 37371fa4e1cc..e053adacf457 100644 --- a/sys/dev/gpio/gpiobus.c +++ b/sys/dev/gpio/gpiobus.c @@ -1,1078 +1,1070 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2009 Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #ifdef INTRNG #include #endif #include #include #include #include #include #include "gpiobus_if.h" #undef GPIOBUS_DEBUG #ifdef GPIOBUS_DEBUG #define dprintf printf #else #define dprintf(x, arg...) #endif static void gpiobus_print_pins(struct gpiobus_ivar *, struct sbuf *); static int gpiobus_parse_pins(struct gpiobus_softc *, device_t, int); static int gpiobus_probe(device_t); static int gpiobus_suspend(device_t); static int gpiobus_resume(device_t); static void gpiobus_probe_nomatch(device_t, device_t); static int gpiobus_print_child(device_t, device_t); static int gpiobus_child_location(device_t, device_t, struct sbuf *); static device_t gpiobus_add_child(device_t, u_int, const char *, int); static void gpiobus_hinted_child(device_t, const char *, int); /* * GPIOBUS interface */ static int gpiobus_acquire_bus(device_t, device_t, int); static void gpiobus_release_bus(device_t, device_t); static int gpiobus_pin_setflags(device_t, device_t, uint32_t, uint32_t); static int gpiobus_pin_getflags(device_t, device_t, uint32_t, uint32_t*); static int gpiobus_pin_getcaps(device_t, device_t, uint32_t, uint32_t*); static int gpiobus_pin_set(device_t, device_t, uint32_t, unsigned int); static int gpiobus_pin_get(device_t, device_t, uint32_t, unsigned int*); static int gpiobus_pin_toggle(device_t, device_t, uint32_t); /* * gpiobus_pin flags * The flags in struct gpiobus_pin are not related to the flags used by the * low-level controller driver in struct gpio_pin. Currently, only pins * acquired via FDT data have gpiobus_pin.flags set, sourced from the flags in * the FDT properties. In theory, these flags are defined per-platform. In * practice they are always the flags from the dt-bindings/gpio/gpio.h file. * The only one of those flags we currently support is for handling active-low * pins, so we just define that flag here instead of including a GPL'd header. */ #define GPIO_ACTIVE_LOW 1 /* * XXX -> Move me to better place - gpio_subr.c? * Also, this function must be changed when interrupt configuration * data will be moved into struct resource. */ #ifdef INTRNG struct resource * gpio_alloc_intr_resource(device_t consumer_dev, int *rid, u_int alloc_flags, gpio_pin_t pin, uint32_t intr_mode) { u_int irq; struct intr_map_data_gpio *gpio_data; struct resource *res; gpio_data = (struct intr_map_data_gpio *)intr_alloc_map_data( INTR_MAP_DATA_GPIO, sizeof(*gpio_data), M_WAITOK | M_ZERO); gpio_data->gpio_pin_num = pin->pin; gpio_data->gpio_pin_flags = pin->flags; gpio_data->gpio_intr_mode = intr_mode; irq = intr_map_irq(pin->dev, 0, (struct intr_map_data *)gpio_data); res = bus_alloc_resource(consumer_dev, SYS_RES_IRQ, rid, irq, irq, 1, alloc_flags); if (res == NULL) { intr_free_intr_map_data((struct intr_map_data *)gpio_data); return (NULL); } rman_set_virtual(res, gpio_data); return (res); } #else struct resource * gpio_alloc_intr_resource(device_t consumer_dev, int *rid, u_int alloc_flags, gpio_pin_t pin, uint32_t intr_mode) { return (NULL); } #endif int gpio_check_flags(uint32_t caps, uint32_t flags) { /* Filter unwanted flags. */ flags &= caps; /* Cannot mix input/output together. */ if (flags & GPIO_PIN_INPUT && flags & GPIO_PIN_OUTPUT) return (EINVAL); /* Cannot mix pull-up/pull-down together. */ if (flags & GPIO_PIN_PULLUP && flags & GPIO_PIN_PULLDOWN) return (EINVAL); /* Cannot mix output and interrupt flags together */ if (flags & GPIO_PIN_OUTPUT && flags & GPIO_INTR_MASK) return (EINVAL); /* Only one interrupt flag can be defined at once */ if ((flags & GPIO_INTR_MASK) & ((flags & GPIO_INTR_MASK) - 1)) return (EINVAL); /* The interrupt attached flag cannot be set */ if (flags & GPIO_INTR_ATTACHED) return (EINVAL); return (0); } int gpio_pin_get_by_bus_pinnum(device_t busdev, uint32_t pinnum, gpio_pin_t *ppin) { gpio_pin_t pin; int err; err = gpiobus_acquire_pin(busdev, pinnum); if (err != 0) return (EBUSY); pin = malloc(sizeof(*pin), M_DEVBUF, M_WAITOK | M_ZERO); pin->dev = device_get_parent(busdev); pin->pin = pinnum; pin->flags = 0; *ppin = pin; return (0); } int gpio_pin_get_by_child_index(device_t childdev, uint32_t idx, gpio_pin_t *ppin) { struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(childdev); if (idx >= devi->npins) return (EINVAL); return (gpio_pin_get_by_bus_pinnum(device_get_parent(childdev), devi->pins[idx], ppin)); } int gpio_pin_getcaps(gpio_pin_t pin, uint32_t *caps) { KASSERT(pin != NULL, ("GPIO pin is NULL.")); KASSERT(pin->dev != NULL, ("GPIO pin device is NULL.")); return (GPIO_PIN_GETCAPS(pin->dev, pin->pin, caps)); } int gpio_pin_is_active(gpio_pin_t pin, bool *active) { int rv; uint32_t tmp; KASSERT(pin != NULL, ("GPIO pin is NULL.")); KASSERT(pin->dev != NULL, ("GPIO pin device is NULL.")); rv = GPIO_PIN_GET(pin->dev, pin->pin, &tmp); if (rv != 0) { return (rv); } if (pin->flags & GPIO_ACTIVE_LOW) *active = tmp == 0; else *active = tmp != 0; return (0); } void gpio_pin_release(gpio_pin_t gpio) { device_t busdev; if (gpio == NULL) return; KASSERT(gpio->dev != NULL, ("GPIO pin device is NULL.")); busdev = GPIO_GET_BUS(gpio->dev); if (busdev != NULL) gpiobus_release_pin(busdev, gpio->pin); free(gpio, M_DEVBUF); } int gpio_pin_set_active(gpio_pin_t pin, bool active) { int rv; uint32_t tmp; if (pin->flags & GPIO_ACTIVE_LOW) tmp = active ? 0 : 1; else tmp = active ? 1 : 0; KASSERT(pin != NULL, ("GPIO pin is NULL.")); KASSERT(pin->dev != NULL, ("GPIO pin device is NULL.")); rv = GPIO_PIN_SET(pin->dev, pin->pin, tmp); return (rv); } int gpio_pin_setflags(gpio_pin_t pin, uint32_t flags) { int rv; KASSERT(pin != NULL, ("GPIO pin is NULL.")); KASSERT(pin->dev != NULL, ("GPIO pin device is NULL.")); rv = GPIO_PIN_SETFLAGS(pin->dev, pin->pin, flags); return (rv); } static void gpiobus_print_pins(struct gpiobus_ivar *devi, struct sbuf *sb) { int i, range_start, range_stop, need_coma; if (devi->npins == 0) return; need_coma = 0; range_start = range_stop = devi->pins[0]; for (i = 1; i < devi->npins; i++) { if (devi->pins[i] != (range_stop + 1)) { if (need_coma) sbuf_cat(sb, ","); if (range_start != range_stop) sbuf_printf(sb, "%d-%d", range_start, range_stop); else sbuf_printf(sb, "%d", range_start); range_start = range_stop = devi->pins[i]; need_coma = 1; } else range_stop++; } if (need_coma) sbuf_cat(sb, ","); if (range_start != range_stop) sbuf_printf(sb, "%d-%d", range_start, range_stop); else sbuf_printf(sb, "%d", range_start); } device_t gpiobus_attach_bus(device_t dev) { device_t busdev; busdev = device_add_child(dev, "gpiobus", DEVICE_UNIT_ANY); if (busdev == NULL) return (NULL); if (device_add_child(dev, "gpioc", -1) == NULL) { device_delete_child(dev, busdev); return (NULL); } #ifdef FDT ofw_gpiobus_register_provider(dev); #endif bus_attach_children(dev); return (busdev); } int gpiobus_detach_bus(device_t dev) { - int err; - #ifdef FDT ofw_gpiobus_unregister_provider(dev); #endif - err = bus_generic_detach(dev); - if (err != 0) - return (err); - - return (device_delete_children(dev)); + return (bus_generic_detach(dev)); } int gpiobus_init_softc(device_t dev) { struct gpiobus_softc *sc; sc = GPIOBUS_SOFTC(dev); sc->sc_busdev = dev; sc->sc_dev = device_get_parent(dev); sc->sc_intr_rman.rm_type = RMAN_ARRAY; sc->sc_intr_rman.rm_descr = "GPIO Interrupts"; if (rman_init(&sc->sc_intr_rman) != 0 || rman_manage_region(&sc->sc_intr_rman, 0, ~0) != 0) panic("%s: failed to set up rman.", __func__); if (GPIO_PIN_MAX(sc->sc_dev, &sc->sc_npins) != 0) return (ENXIO); KASSERT(sc->sc_npins >= 0, ("GPIO device with no pins")); /* Pins = GPIO_PIN_MAX() + 1 */ sc->sc_npins++; sc->sc_pins = malloc(sizeof(*sc->sc_pins) * sc->sc_npins, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_pins == NULL) return (ENOMEM); /* Initialize the bus lock. */ GPIOBUS_LOCK_INIT(sc); return (0); } int gpiobus_alloc_ivars(struct gpiobus_ivar *devi) { /* Allocate pins and flags memory. */ devi->pins = malloc(sizeof(uint32_t) * devi->npins, M_DEVBUF, M_NOWAIT | M_ZERO); if (devi->pins == NULL) return (ENOMEM); return (0); } void gpiobus_free_ivars(struct gpiobus_ivar *devi) { if (devi->pins) { free(devi->pins, M_DEVBUF); devi->pins = NULL; } devi->npins = 0; } int gpiobus_acquire_pin(device_t bus, uint32_t pin) { struct gpiobus_softc *sc; sc = device_get_softc(bus); /* Consistency check. */ if (pin >= sc->sc_npins) { device_printf(bus, "invalid pin %d, max: %d\n", pin, sc->sc_npins - 1); return (-1); } /* Mark pin as mapped and give warning if it's already mapped. */ if (sc->sc_pins[pin].mapped) { device_printf(bus, "warning: pin %d is already mapped\n", pin); return (-1); } sc->sc_pins[pin].mapped = 1; return (0); } /* Release mapped pin */ int gpiobus_release_pin(device_t bus, uint32_t pin) { struct gpiobus_softc *sc; sc = device_get_softc(bus); /* Consistency check. */ if (pin >= sc->sc_npins) { device_printf(bus, "invalid pin %d, max=%d\n", pin, sc->sc_npins - 1); return (-1); } if (!sc->sc_pins[pin].mapped) { device_printf(bus, "pin %d is not mapped\n", pin); return (-1); } sc->sc_pins[pin].mapped = 0; return (0); } static int gpiobus_acquire_child_pins(device_t dev, device_t child) { struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); int i; for (i = 0; i < devi->npins; i++) { /* Reserve the GPIO pin. */ if (gpiobus_acquire_pin(dev, devi->pins[i]) != 0) { device_printf(child, "cannot acquire pin %d\n", devi->pins[i]); while (--i >= 0) { (void)gpiobus_release_pin(dev, devi->pins[i]); } gpiobus_free_ivars(devi); return (EBUSY); } } for (i = 0; i < devi->npins; i++) { /* Use the child name as pin name. */ GPIOBUS_PIN_SETNAME(dev, devi->pins[i], device_get_nameunit(child)); } return (0); } static int gpiobus_parse_pins(struct gpiobus_softc *sc, device_t child, int mask) { struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); int i, npins; npins = 0; for (i = 0; i < 32; i++) { if (mask & (1 << i)) npins++; } if (npins == 0) { device_printf(child, "empty pin mask\n"); return (EINVAL); } devi->npins = npins; if (gpiobus_alloc_ivars(devi) != 0) { device_printf(child, "cannot allocate device ivars\n"); return (EINVAL); } npins = 0; for (i = 0; i < 32; i++) { if ((mask & (1 << i)) == 0) continue; devi->pins[npins++] = i; } return (0); } static int gpiobus_parse_pin_list(struct gpiobus_softc *sc, device_t child, const char *pins) { struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); const char *p; char *endp; unsigned long pin; int i, npins; npins = 0; p = pins; for (;;) { pin = strtoul(p, &endp, 0); if (endp == p) break; npins++; if (*endp == '\0') break; p = endp + 1; } if (*endp != '\0') { device_printf(child, "garbage in the pin list: %s\n", endp); return (EINVAL); } if (npins == 0) { device_printf(child, "empty pin list\n"); return (EINVAL); } devi->npins = npins; if (gpiobus_alloc_ivars(devi) != 0) { device_printf(child, "cannot allocate device ivars\n"); return (EINVAL); } i = 0; p = pins; for (;;) { pin = strtoul(p, &endp, 0); devi->pins[i] = pin; if (*endp == '\0') break; i++; p = endp + 1; } return (0); } static int gpiobus_probe(device_t dev) { device_set_desc(dev, "GPIO bus"); return (BUS_PROBE_GENERIC); } int gpiobus_attach(device_t dev) { int err; err = gpiobus_init_softc(dev); if (err != 0) return (err); /* * Get parent's pins and mark them as unmapped */ bus_identify_children(dev); bus_enumerate_hinted_children(dev); bus_attach_children(dev); return (0); } /* * Since this is not a self-enumerating bus, and since we always add * children in attach, we have to always delete children here. */ int gpiobus_detach(device_t dev) { struct gpiobus_softc *sc; int i, err; sc = GPIOBUS_SOFTC(dev); KASSERT(mtx_initialized(&sc->sc_mtx), ("gpiobus mutex not initialized")); GPIOBUS_LOCK_DESTROY(sc); if ((err = bus_detach_children(dev)) != 0) return (err); - if ((err = device_delete_children(dev)) != 0) - return (err); rman_fini(&sc->sc_intr_rman); if (sc->sc_pins) { for (i = 0; i < sc->sc_npins; i++) { if (sc->sc_pins[i].name != NULL) free(sc->sc_pins[i].name, M_DEVBUF); sc->sc_pins[i].name = NULL; } free(sc->sc_pins, M_DEVBUF); sc->sc_pins = NULL; } return (0); } static int gpiobus_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int gpiobus_resume(device_t dev) { return (bus_generic_resume(dev)); } static void gpiobus_probe_nomatch(device_t dev, device_t child) { char pins[128]; struct sbuf sb; struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(child); sbuf_new(&sb, pins, sizeof(pins), SBUF_FIXEDLEN); gpiobus_print_pins(devi, &sb); sbuf_finish(&sb); device_printf(dev, " at pin%s %s", devi->npins > 1 ? "s" : "", sbuf_data(&sb)); resource_list_print_type(&devi->rl, "irq", SYS_RES_IRQ, "%jd"); printf("\n"); } static int gpiobus_print_child(device_t dev, device_t child) { char pins[128]; struct sbuf sb; int retval = 0; struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(child); retval += bus_print_child_header(dev, child); if (devi->npins > 0) { if (devi->npins > 1) retval += printf(" at pins "); else retval += printf(" at pin "); sbuf_new(&sb, pins, sizeof(pins), SBUF_FIXEDLEN); gpiobus_print_pins(devi, &sb); sbuf_finish(&sb); retval += printf("%s", sbuf_data(&sb)); } resource_list_print_type(&devi->rl, "irq", SYS_RES_IRQ, "%jd"); retval += bus_print_child_footer(dev, child); return (retval); } static int gpiobus_child_location(device_t bus, device_t child, struct sbuf *sb) { struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(child); sbuf_printf(sb, "pins="); gpiobus_print_pins(devi, sb); return (0); } static device_t gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) { device_t child; struct gpiobus_ivar *devi; child = device_add_child_ordered(dev, order, name, unit); if (child == NULL) return (child); devi = malloc(sizeof(struct gpiobus_ivar), M_DEVBUF, M_NOWAIT | M_ZERO); if (devi == NULL) { device_delete_child(dev, child); return (NULL); } resource_list_init(&devi->rl); device_set_ivars(child, devi); return (child); } static void gpiobus_child_deleted(device_t dev, device_t child) { struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(child); if (devi == NULL) return; gpiobus_free_ivars(devi); resource_list_free(&devi->rl); free(devi, M_DEVBUF); } static int gpiobus_rescan(device_t dev) { /* * Re-scan is supposed to remove and add children, but if someone has * deleted the hints for a child we attached earlier, we have no easy * way to handle that. So this just attaches new children for whom new * hints or drivers have arrived since we last tried. */ bus_enumerate_hinted_children(dev); bus_attach_children(dev); return (0); } static void gpiobus_hinted_child(device_t bus, const char *dname, int dunit) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(bus); device_t child; const char *pins; int irq, pinmask; if (device_find_child(bus, dname, dunit) != NULL) { return; } child = BUS_ADD_CHILD(bus, 0, dname, dunit); if (resource_int_value(dname, dunit, "pins", &pinmask) == 0) { if (gpiobus_parse_pins(sc, child, pinmask)) { device_delete_child(bus, child); return; } } else if (resource_string_value(dname, dunit, "pin_list", &pins) == 0) { if (gpiobus_parse_pin_list(sc, child, pins)) { device_delete_child(bus, child); return; } } if (resource_int_value(dname, dunit, "irq", &irq) == 0) { if (bus_set_resource(child, SYS_RES_IRQ, 0, irq, 1) != 0) device_printf(bus, "warning: bus_set_resource() failed\n"); } } int gpiobus_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct gpiobus_ivar *devi; devi = GPIOBUS_IVAR(child); switch (which) { case GPIOBUS_IVAR_NPINS: *result = devi->npins; break; case GPIOBUS_IVAR_PINS: /* Children do not ever need to directly examine this. */ return (ENOTSUP); default: return (ENOENT); } return (0); } static int gpiobus_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { struct gpiobus_ivar *devi; const uint32_t *ptr; int i; devi = GPIOBUS_IVAR(child); switch (which) { case GPIOBUS_IVAR_NPINS: /* GPIO ivars are set once. */ if (devi->npins != 0) { return (EBUSY); } devi->npins = value; if (gpiobus_alloc_ivars(devi) != 0) { device_printf(child, "cannot allocate device ivars\n"); devi->npins = 0; return (ENOMEM); } break; case GPIOBUS_IVAR_PINS: ptr = (const uint32_t *)value; for (i = 0; i < devi->npins; i++) devi->pins[i] = ptr[i]; if (gpiobus_acquire_child_pins(dev, child) != 0) return (EBUSY); break; default: return (ENOENT); } return (0); } static struct rman * gpiobus_get_rman(device_t bus, int type, u_int flags) { struct gpiobus_softc *sc; sc = device_get_softc(bus); switch (type) { case SYS_RES_IRQ: return (&sc->sc_intr_rman); default: return (NULL); } } static struct resource * gpiobus_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource_list *rl; struct resource_list_entry *rle; int isdefault; isdefault = (RMAN_IS_DEFAULT_RANGE(start, end) && count == 1); if (isdefault) { rl = BUS_GET_RESOURCE_LIST(bus, child); if (rl == NULL) return (NULL); rle = resource_list_find(rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; count = rle->count; end = rle->end; } return (bus_generic_rman_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static struct resource_list * gpiobus_get_resource_list(device_t bus __unused, device_t child) { struct gpiobus_ivar *ivar; ivar = GPIOBUS_IVAR(child); return (&ivar->rl); } static int gpiobus_acquire_bus(device_t busdev, device_t child, int how) { struct gpiobus_softc *sc; sc = device_get_softc(busdev); GPIOBUS_ASSERT_UNLOCKED(sc); GPIOBUS_LOCK(sc); if (sc->sc_owner != NULL) { if (sc->sc_owner == child) panic("%s: %s still owns the bus.", device_get_nameunit(busdev), device_get_nameunit(child)); if (how == GPIOBUS_DONTWAIT) { GPIOBUS_UNLOCK(sc); return (EWOULDBLOCK); } while (sc->sc_owner != NULL) mtx_sleep(sc, &sc->sc_mtx, 0, "gpiobuswait", 0); } sc->sc_owner = child; GPIOBUS_UNLOCK(sc); return (0); } static void gpiobus_release_bus(device_t busdev, device_t child) { struct gpiobus_softc *sc; sc = device_get_softc(busdev); GPIOBUS_ASSERT_UNLOCKED(sc); GPIOBUS_LOCK(sc); if (sc->sc_owner == NULL) panic("%s: %s releasing unowned bus.", device_get_nameunit(busdev), device_get_nameunit(child)); if (sc->sc_owner != child) panic("%s: %s trying to release bus owned by %s", device_get_nameunit(busdev), device_get_nameunit(child), device_get_nameunit(sc->sc_owner)); sc->sc_owner = NULL; wakeup(sc); GPIOBUS_UNLOCK(sc); } static int gpiobus_pin_setflags(device_t dev, device_t child, uint32_t pin, uint32_t flags) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); uint32_t caps; if (pin >= devi->npins) return (EINVAL); if (GPIO_PIN_GETCAPS(sc->sc_dev, devi->pins[pin], &caps) != 0) return (EINVAL); if (gpio_check_flags(caps, flags) != 0) return (EINVAL); return (GPIO_PIN_SETFLAGS(sc->sc_dev, devi->pins[pin], flags)); } static int gpiobus_pin_getflags(device_t dev, device_t child, uint32_t pin, uint32_t *flags) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); if (pin >= devi->npins) return (EINVAL); return GPIO_PIN_GETFLAGS(sc->sc_dev, devi->pins[pin], flags); } static int gpiobus_pin_getcaps(device_t dev, device_t child, uint32_t pin, uint32_t *caps) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); if (pin >= devi->npins) return (EINVAL); return GPIO_PIN_GETCAPS(sc->sc_dev, devi->pins[pin], caps); } static int gpiobus_pin_set(device_t dev, device_t child, uint32_t pin, unsigned int value) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); if (pin >= devi->npins) return (EINVAL); return GPIO_PIN_SET(sc->sc_dev, devi->pins[pin], value); } static int gpiobus_pin_get(device_t dev, device_t child, uint32_t pin, unsigned int *value) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); if (pin >= devi->npins) return (EINVAL); return GPIO_PIN_GET(sc->sc_dev, devi->pins[pin], value); } static int gpiobus_pin_toggle(device_t dev, device_t child, uint32_t pin) { struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); if (pin >= devi->npins) return (EINVAL); return GPIO_PIN_TOGGLE(sc->sc_dev, devi->pins[pin]); } static int gpiobus_pin_getname(device_t dev, uint32_t pin, char *name) { struct gpiobus_softc *sc; sc = GPIOBUS_SOFTC(dev); if (pin > sc->sc_npins) return (EINVAL); /* Did we have a name for this pin ? */ if (sc->sc_pins[pin].name != NULL) { memcpy(name, sc->sc_pins[pin].name, GPIOMAXNAME); return (0); } /* Return the default pin name. */ return (GPIO_PIN_GETNAME(device_get_parent(dev), pin, name)); } static int gpiobus_pin_setname(device_t dev, uint32_t pin, const char *name) { struct gpiobus_softc *sc; sc = GPIOBUS_SOFTC(dev); if (pin > sc->sc_npins) return (EINVAL); if (name == NULL) return (EINVAL); /* Save the pin name. */ if (sc->sc_pins[pin].name == NULL) sc->sc_pins[pin].name = malloc(GPIOMAXNAME, M_DEVBUF, M_WAITOK | M_ZERO); strlcpy(sc->sc_pins[pin].name, name, GPIOMAXNAME); return (0); } static device_method_t gpiobus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gpiobus_probe), DEVMETHOD(device_attach, gpiobus_attach), DEVMETHOD(device_detach, gpiobus_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, gpiobus_suspend), DEVMETHOD(device_resume, gpiobus_resume), /* Bus interface */ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_config_intr, bus_generic_config_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_delete_resource, bus_generic_rl_delete_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_alloc_resource, gpiobus_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_rman_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_rman_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_rman_deactivate_resource), DEVMETHOD(bus_get_resource_list, gpiobus_get_resource_list), DEVMETHOD(bus_get_rman, gpiobus_get_rman), DEVMETHOD(bus_add_child, gpiobus_add_child), DEVMETHOD(bus_child_deleted, gpiobus_child_deleted), DEVMETHOD(bus_rescan, gpiobus_rescan), DEVMETHOD(bus_probe_nomatch, gpiobus_probe_nomatch), DEVMETHOD(bus_print_child, gpiobus_print_child), DEVMETHOD(bus_child_location, gpiobus_child_location), DEVMETHOD(bus_hinted_child, gpiobus_hinted_child), DEVMETHOD(bus_read_ivar, gpiobus_read_ivar), DEVMETHOD(bus_write_ivar, gpiobus_write_ivar), /* GPIO protocol */ DEVMETHOD(gpiobus_acquire_bus, gpiobus_acquire_bus), DEVMETHOD(gpiobus_release_bus, gpiobus_release_bus), DEVMETHOD(gpiobus_pin_getflags, gpiobus_pin_getflags), DEVMETHOD(gpiobus_pin_getcaps, gpiobus_pin_getcaps), DEVMETHOD(gpiobus_pin_setflags, gpiobus_pin_setflags), DEVMETHOD(gpiobus_pin_get, gpiobus_pin_get), DEVMETHOD(gpiobus_pin_set, gpiobus_pin_set), DEVMETHOD(gpiobus_pin_toggle, gpiobus_pin_toggle), DEVMETHOD(gpiobus_pin_getname, gpiobus_pin_getname), DEVMETHOD(gpiobus_pin_setname, gpiobus_pin_setname), DEVMETHOD_END }; driver_t gpiobus_driver = { "gpiobus", gpiobus_methods, sizeof(struct gpiobus_softc) }; EARLY_DRIVER_MODULE(gpiobus, gpio, gpiobus_driver, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); MODULE_VERSION(gpiobus, 1); diff --git a/sys/dev/hid/hidbus.c b/sys/dev/hid/hidbus.c index 9b8b7fed8887..96d36c8d191d 100644 --- a/sys/dev/hid/hidbus.c +++ b/sys/dev/hid/hidbus.c @@ -1,974 +1,973 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019-2020 Vladimir Kondratyev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define HID_DEBUG_VAR hid_debug #include #include #include #include "hid_if.h" #define INPUT_EPOCH global_epoch_preempt #define HID_RSIZE_MAX 1024 static hid_intr_t hidbus_intr; static device_probe_t hidbus_probe; static device_attach_t hidbus_attach; static device_detach_t hidbus_detach; struct hidbus_ivars { int32_t usage; uint8_t index; uint32_t flags; uintptr_t driver_info; /* for internal use */ struct mtx *mtx; /* child intr mtx */ hid_intr_t *intr_handler; /* executed under mtx*/ void *intr_ctx; unsigned int refcnt; /* protected by mtx */ struct epoch_context epoch_ctx; CK_STAILQ_ENTRY(hidbus_ivars) link; }; struct hidbus_softc { device_t dev; struct sx sx; struct mtx mtx; bool nowrite; struct hid_rdesc_info rdesc; bool overloaded; int nest; /* Child attach nesting lvl */ int nauto; /* Number of autochildren */ CK_STAILQ_HEAD(, hidbus_ivars) tlcs; }; static int hidbus_fill_rdesc_info(struct hid_rdesc_info *hri, const void *data, hid_size_t len) { int error = 0; hri->data = __DECONST(void *, data); hri->len = len; /* * If report descriptor is not available yet, set maximal * report sizes high enough to allow hidraw to work. */ hri->isize = len == 0 ? HID_RSIZE_MAX : hid_report_size_max(data, len, hid_input, &hri->iid); hri->osize = len == 0 ? HID_RSIZE_MAX : hid_report_size_max(data, len, hid_output, &hri->oid); hri->fsize = len == 0 ? HID_RSIZE_MAX : hid_report_size_max(data, len, hid_feature, &hri->fid); if (hri->isize > HID_RSIZE_MAX) { DPRINTF("input size is too large, %u bytes (truncating)\n", hri->isize); hri->isize = HID_RSIZE_MAX; error = EOVERFLOW; } if (hri->osize > HID_RSIZE_MAX) { DPRINTF("output size is too large, %u bytes (truncating)\n", hri->osize); hri->osize = HID_RSIZE_MAX; error = EOVERFLOW; } if (hri->fsize > HID_RSIZE_MAX) { DPRINTF("feature size is too large, %u bytes (truncating)\n", hri->fsize); hri->fsize = HID_RSIZE_MAX; error = EOVERFLOW; } return (error); } int hidbus_locate(const void *desc, hid_size_t size, int32_t u, enum hid_kind k, uint8_t tlc_index, uint8_t index, struct hid_location *loc, uint32_t *flags, uint8_t *id, struct hid_absinfo *ai) { struct hid_data *d; struct hid_item h; int i; d = hid_start_parse(desc, size, 1 << k); HIDBUS_FOREACH_ITEM(d, &h, tlc_index) { for (i = 0; i < h.nusages; i++) { if (h.kind == k && h.usages[i] == u) { if (index--) break; if (loc != NULL) *loc = h.loc; if (flags != NULL) *flags = h.flags; if (id != NULL) *id = h.report_ID; if (ai != NULL && (h.flags&HIO_RELATIVE) == 0) *ai = (struct hid_absinfo) { .max = h.logical_maximum, .min = h.logical_minimum, .res = hid_item_resolution(&h), }; hid_end_parse(d); return (1); } } } if (loc != NULL) loc->size = 0; if (flags != NULL) *flags = 0; if (id != NULL) *id = 0; hid_end_parse(d); return (0); } bool hidbus_is_collection(const void *desc, hid_size_t size, int32_t usage, uint8_t tlc_index) { struct hid_data *d; struct hid_item h; bool ret = false; d = hid_start_parse(desc, size, 0); HIDBUS_FOREACH_ITEM(d, &h, tlc_index) { if (h.kind == hid_collection && h.usage == usage) { ret = true; break; } } hid_end_parse(d); return (ret); } static device_t hidbus_add_child(device_t dev, u_int order, const char *name, int unit) { struct hidbus_softc *sc = device_get_softc(dev); struct hidbus_ivars *tlc; device_t child; child = device_add_child_ordered(dev, order, name, unit); if (child == NULL) return (child); tlc = malloc(sizeof(struct hidbus_ivars), M_DEVBUF, M_WAITOK | M_ZERO); tlc->mtx = &sc->mtx; device_set_ivars(child, tlc); sx_xlock(&sc->sx); CK_STAILQ_INSERT_TAIL(&sc->tlcs, tlc, link); sx_unlock(&sc->sx); return (child); } static int hidbus_enumerate_children(device_t dev, const void* data, hid_size_t len) { struct hidbus_softc *sc = device_get_softc(dev); struct hid_data *hd; struct hid_item hi; device_t child; uint8_t index = 0; if (data == NULL || len == 0) return (ENXIO); /* Add a child for each top level collection */ hd = hid_start_parse(data, len, 1 << hid_input); while (hid_get_item(hd, &hi)) { if (hi.kind != hid_collection || hi.collevel != 1) continue; child = BUS_ADD_CHILD(dev, 0, NULL, DEVICE_UNIT_ANY); if (child == NULL) { device_printf(dev, "Could not add HID device\n"); continue; } hidbus_set_index(child, index); hidbus_set_usage(child, hi.usage); hidbus_set_flags(child, HIDBUS_FLAG_AUTOCHILD); index++; DPRINTF("Add child TLC: 0x%04x:0x%04x\n", HID_GET_USAGE_PAGE(hi.usage), HID_GET_USAGE(hi.usage)); } hid_end_parse(hd); if (index == 0) return (ENXIO); sc->nauto = index; return (0); } static int hidbus_attach_children(device_t dev) { struct hidbus_softc *sc = device_get_softc(dev); int error; HID_INTR_SETUP(device_get_parent(dev), dev, hidbus_intr, sc, &sc->rdesc); error = hidbus_enumerate_children(dev, sc->rdesc.data, sc->rdesc.len); if (error != 0) DPRINTF("failed to enumerate children: error %d\n", error); /* * hidbus_attach_children() can recurse through device_identify-> * hid_set_report_descr() call sequence. Do not perform children * attach twice in that case. */ sc->nest++; bus_identify_children(dev); sc->nest--; if (sc->nest != 0) return (0); if (hid_is_keyboard(sc->rdesc.data, sc->rdesc.len) != 0) bus_attach_children(dev); else bus_delayed_attach_children(dev); return (0); } static int hidbus_detach_children(device_t dev) { device_t *children, bus; bool is_bus; int i, error; error = 0; is_bus = device_get_devclass(dev) == devclass_find("hidbus"); bus = is_bus ? dev : device_get_parent(dev); KASSERT(device_get_devclass(bus) == devclass_find("hidbus"), ("Device is not hidbus or it's child")); if (is_bus) { /* If hidbus is passed, delete all children. */ - bus_generic_detach(bus); - device_delete_children(bus); + error = bus_generic_detach(bus); } else { /* * If hidbus child is passed, delete all hidbus children * except caller. Deleting the caller may result in deadlock. */ error = device_get_children(bus, &children, &i); if (error != 0) return (error); while (i-- > 0) { if (children[i] == dev) continue; DPRINTF("Delete child. index=%d (%s)\n", hidbus_get_index(children[i]), device_get_nameunit(children[i])); error = device_delete_child(bus, children[i]); if (error) { DPRINTF("Failed deleting %s\n", device_get_nameunit(children[i])); break; } } free(children, M_TEMP); } HID_INTR_UNSETUP(device_get_parent(bus), bus); return (error); } static int hidbus_probe(device_t dev) { device_set_desc(dev, "HID bus"); /* Allow other subclasses to override this driver. */ return (BUS_PROBE_GENERIC); } static int hidbus_attach(device_t dev) { struct hidbus_softc *sc = device_get_softc(dev); struct hid_device_info *devinfo = device_get_ivars(dev); void *d_ptr = NULL; hid_size_t d_len; int error; sc->dev = dev; CK_STAILQ_INIT(&sc->tlcs); mtx_init(&sc->mtx, "hidbus ivar lock", NULL, MTX_DEF); sx_init(&sc->sx, "hidbus ivar list lock"); /* * Ignore error. It is possible for non-HID device e.g. XBox360 gamepad * to emulate HID through overloading of report descriptor. */ d_len = devinfo->rdescsize; if (d_len != 0) { d_ptr = malloc(d_len, M_DEVBUF, M_ZERO | M_WAITOK); error = hid_get_rdesc(dev, d_ptr, d_len); if (error != 0) { free(d_ptr, M_DEVBUF); d_len = 0; d_ptr = NULL; } } hidbus_fill_rdesc_info(&sc->rdesc, d_ptr, d_len); sc->nowrite = hid_test_quirk(devinfo, HQ_NOWRITE); error = hidbus_attach_children(dev); if (error != 0) { hidbus_detach(dev); return (ENXIO); } return (0); } static int hidbus_detach(device_t dev) { struct hidbus_softc *sc = device_get_softc(dev); hidbus_detach_children(dev); sx_destroy(&sc->sx); mtx_destroy(&sc->mtx); free(sc->rdesc.data, M_DEVBUF); return (0); } static void hidbus_child_detached(device_t bus, device_t child) { struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *tlc = device_get_ivars(child); KASSERT(tlc->refcnt == 0, ("Child device is running")); tlc->mtx = &sc->mtx; tlc->intr_handler = NULL; tlc->flags &= ~HIDBUS_FLAG_CAN_POLL; } /* * Epoch callback indicating tlc is safe to destroy */ static void hidbus_ivar_dtor(epoch_context_t ctx) { struct hidbus_ivars *tlc; tlc = __containerof(ctx, struct hidbus_ivars, epoch_ctx); free(tlc, M_DEVBUF); } static void hidbus_child_deleted(device_t bus, device_t child) { struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *tlc = device_get_ivars(child); sx_xlock(&sc->sx); KASSERT(tlc->refcnt == 0, ("Child device is running")); CK_STAILQ_REMOVE(&sc->tlcs, tlc, hidbus_ivars, link); sx_unlock(&sc->sx); epoch_call(INPUT_EPOCH, hidbus_ivar_dtor, &tlc->epoch_ctx); } static int hidbus_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *tlc = device_get_ivars(child); switch (which) { case HIDBUS_IVAR_INDEX: *result = tlc->index; break; case HIDBUS_IVAR_USAGE: *result = tlc->usage; break; case HIDBUS_IVAR_FLAGS: *result = tlc->flags; break; case HIDBUS_IVAR_DRIVER_INFO: *result = tlc->driver_info; break; case HIDBUS_IVAR_LOCK: *result = (uintptr_t)(tlc->mtx == &sc->mtx ? NULL : tlc->mtx); break; default: return (EINVAL); } return (0); } static int hidbus_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *tlc = device_get_ivars(child); switch (which) { case HIDBUS_IVAR_INDEX: tlc->index = value; break; case HIDBUS_IVAR_USAGE: tlc->usage = value; break; case HIDBUS_IVAR_FLAGS: tlc->flags = value; if ((value & HIDBUS_FLAG_CAN_POLL) != 0) HID_INTR_SETUP( device_get_parent(bus), bus, NULL, NULL, NULL); break; case HIDBUS_IVAR_DRIVER_INFO: tlc->driver_info = value; break; case HIDBUS_IVAR_LOCK: tlc->mtx = (struct mtx *)value == NULL ? &sc->mtx : (struct mtx *)value; break; default: return (EINVAL); } return (0); } /* Location hint for devctl(8) */ static int hidbus_child_location(device_t bus, device_t child, struct sbuf *sb) { struct hidbus_ivars *tlc = device_get_ivars(child); sbuf_printf(sb, "index=%hhu", tlc->index); return (0); } /* PnP information for devctl(8) */ static int hidbus_child_pnpinfo(device_t bus, device_t child, struct sbuf *sb) { struct hidbus_ivars *tlc = device_get_ivars(child); struct hid_device_info *devinfo = device_get_ivars(bus); sbuf_printf(sb, "page=0x%04x usage=0x%04x bus=0x%02hx " "vendor=0x%04hx product=0x%04hx version=0x%04hx%s%s", HID_GET_USAGE_PAGE(tlc->usage), HID_GET_USAGE(tlc->usage), devinfo->idBus, devinfo->idVendor, devinfo->idProduct, devinfo->idVersion, devinfo->idPnP[0] == '\0' ? "" : " _HID=", devinfo->idPnP[0] == '\0' ? "" : devinfo->idPnP); return (0); } void hidbus_set_desc(device_t child, const char *suffix) { device_t bus = device_get_parent(child); struct hidbus_softc *sc = device_get_softc(bus); struct hid_device_info *devinfo = device_get_ivars(bus); struct hidbus_ivars *tlc = device_get_ivars(child); /* Do not add NULL suffix or if device name already contains it. */ if (suffix != NULL && strcasestr(devinfo->name, suffix) == NULL && (sc->nauto > 1 || (tlc->flags & HIDBUS_FLAG_AUTOCHILD) == 0)) device_set_descf(child, "%s %s", devinfo->name, suffix); else device_set_desc(child, devinfo->name); } device_t hidbus_find_child(device_t bus, int32_t usage) { device_t *children, child; int ccount, i; bus_topo_assert(); /* Get a list of all hidbus children */ if (device_get_children(bus, &children, &ccount) != 0) return (NULL); /* Scan through to find required TLC */ for (i = 0, child = NULL; i < ccount; i++) { if (hidbus_get_usage(children[i]) == usage) { child = children[i]; break; } } free(children, M_TEMP); return (child); } void hidbus_intr(void *context, void *buf, hid_size_t len) { struct hidbus_softc *sc = context; struct hidbus_ivars *tlc; struct epoch_tracker et; /* * Broadcast input report to all subscribers. * TODO: Add check for input report ID. * * Relock mutex on every TLC item as we can't hold any locks over whole * TLC list here due to LOR with open()/close() handlers. */ if (!HID_IN_POLLING_MODE()) epoch_enter_preempt(INPUT_EPOCH, &et); CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { if (tlc->refcnt == 0 || tlc->intr_handler == NULL) continue; if (HID_IN_POLLING_MODE()) { if ((tlc->flags & HIDBUS_FLAG_CAN_POLL) != 0) tlc->intr_handler(tlc->intr_ctx, buf, len); } else { mtx_lock(tlc->mtx); tlc->intr_handler(tlc->intr_ctx, buf, len); mtx_unlock(tlc->mtx); } } if (!HID_IN_POLLING_MODE()) epoch_exit_preempt(INPUT_EPOCH, &et); } void hidbus_set_intr(device_t child, hid_intr_t *handler, void *context) { struct hidbus_ivars *tlc = device_get_ivars(child); tlc->intr_handler = handler; tlc->intr_ctx = context; } static int hidbus_intr_start(device_t bus, device_t child) { MPASS(bus == device_get_parent(child)); struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *ivar = device_get_ivars(child); struct hidbus_ivars *tlc; bool refcnted = false; int error; if (sx_xlock_sig(&sc->sx) != 0) return (EINTR); CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { refcnted |= (tlc->refcnt != 0); if (tlc == ivar) { mtx_lock(tlc->mtx); ++tlc->refcnt; mtx_unlock(tlc->mtx); } } error = refcnted ? 0 : hid_intr_start(bus); sx_unlock(&sc->sx); return (error); } static int hidbus_intr_stop(device_t bus, device_t child) { MPASS(bus == device_get_parent(child)); struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *ivar = device_get_ivars(child); struct hidbus_ivars *tlc; bool refcnted = false; int error; if (sx_xlock_sig(&sc->sx) != 0) return (EINTR); CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { if (tlc == ivar) { mtx_lock(tlc->mtx); MPASS(tlc->refcnt != 0); --tlc->refcnt; mtx_unlock(tlc->mtx); } refcnted |= (tlc->refcnt != 0); } error = refcnted ? 0 : hid_intr_stop(bus); sx_unlock(&sc->sx); return (error); } static void hidbus_intr_poll(device_t bus, device_t child __unused) { hid_intr_poll(bus); } struct hid_rdesc_info * hidbus_get_rdesc_info(device_t child) { device_t bus = device_get_parent(child); struct hidbus_softc *sc = device_get_softc(bus); return (&sc->rdesc); } /* * HID interface. * * Hidbus as well as any hidbus child can be passed as first arg. */ /* Read cached report descriptor */ int hid_get_report_descr(device_t dev, void **data, hid_size_t *len) { device_t bus; struct hidbus_softc *sc; bus = device_get_devclass(dev) == devclass_find("hidbus") ? dev : device_get_parent(dev); sc = device_get_softc(bus); /* * Do not send request to a transport backend. * Use cached report descriptor instead of it. */ if (sc->rdesc.data == NULL || sc->rdesc.len == 0) return (ENXIO); if (data != NULL) *data = sc->rdesc.data; if (len != NULL) *len = sc->rdesc.len; return (0); } /* * Replace cached report descriptor with top level driver provided one. * * It deletes all hidbus children except caller and enumerates them again after * new descriptor has been registered. Currently it can not be called from * autoenumerated (by report's TLC) child device context as it results in child * duplication. To overcome this limitation hid_set_report_descr() should be * called from device_identify driver's handler with hidbus itself passed as * 'device_t dev' parameter. */ int hid_set_report_descr(device_t dev, const void *data, hid_size_t len) { struct hid_rdesc_info rdesc; device_t bus; struct hidbus_softc *sc; bool is_bus; int error; bus_topo_assert(); is_bus = device_get_devclass(dev) == devclass_find("hidbus"); bus = is_bus ? dev : device_get_parent(dev); sc = device_get_softc(bus); /* * Do not overload already overloaded report descriptor in * device_identify handler. It causes infinite recursion loop. */ if (is_bus && sc->overloaded) return(0); DPRINTFN(5, "len=%d\n", len); DPRINTFN(5, "data = %*D\n", len, data, " "); error = hidbus_fill_rdesc_info(&rdesc, data, len); if (error != 0) return (error); error = hidbus_detach_children(dev); if (error != 0) return(error); /* Make private copy to handle a case of dynamicaly allocated data. */ rdesc.data = malloc(len, M_DEVBUF, M_ZERO | M_WAITOK); bcopy(data, rdesc.data, len); sc->overloaded = true; free(sc->rdesc.data, M_DEVBUF); bcopy(&rdesc, &sc->rdesc, sizeof(struct hid_rdesc_info)); error = hidbus_attach_children(bus); return (error); } static int hidbus_get_rdesc(device_t dev, device_t child __unused, void *data, hid_size_t len) { return (hid_get_rdesc(dev, data, len)); } static int hidbus_read(device_t dev, device_t child __unused, void *data, hid_size_t maxlen, hid_size_t *actlen) { return (hid_read(dev, data, maxlen, actlen)); } static int hidbus_write(device_t dev, device_t child __unused, const void *data, hid_size_t len) { struct hidbus_softc *sc; uint8_t id; sc = device_get_softc(dev); /* * Output interrupt endpoint is often optional. If HID device * does not provide it, send reports via control pipe. */ if (sc->nowrite) { /* try to extract the ID byte */ id = (sc->rdesc.oid & (len > 0)) ? *(const uint8_t*)data : 0; return (hid_set_report(dev, data, len, HID_OUTPUT_REPORT, id)); } return (hid_write(dev, data, len)); } static int hidbus_get_report(device_t dev, device_t child __unused, void *data, hid_size_t maxlen, hid_size_t *actlen, uint8_t type, uint8_t id) { return (hid_get_report(dev, data, maxlen, actlen, type, id)); } static int hidbus_set_report(device_t dev, device_t child __unused, const void *data, hid_size_t len, uint8_t type, uint8_t id) { return (hid_set_report(dev, data, len, type, id)); } static int hidbus_set_idle(device_t dev, device_t child __unused, uint16_t duration, uint8_t id) { return (hid_set_idle(dev, duration, id)); } static int hidbus_set_protocol(device_t dev, device_t child __unused, uint16_t protocol) { return (hid_set_protocol(dev, protocol)); } static int hidbus_ioctl(device_t dev, device_t child __unused, unsigned long cmd, uintptr_t data) { return (hid_ioctl(dev, cmd, data)); } /*------------------------------------------------------------------------* * hidbus_lookup_id * * This functions takes an array of "struct hid_device_id" and tries * to match the entries with the information in "struct hid_device_info". * * Return values: * NULL: No match found. * Else: Pointer to matching entry. *------------------------------------------------------------------------*/ const struct hid_device_id * hidbus_lookup_id(device_t dev, const struct hid_device_id *id, int nitems_id) { const struct hid_device_id *id_end; const struct hid_device_info *info; int32_t usage; bool is_child; if (id == NULL) { goto done; } id_end = id + nitems_id; info = hid_get_device_info(dev); is_child = device_get_devclass(dev) != devclass_find("hidbus"); if (is_child) usage = hidbus_get_usage(dev); /* * Keep on matching array entries until we find a match or * until we reach the end of the matching array: */ for (; id != id_end; id++) { if (is_child && (id->match_flag_page) && (id->page != HID_GET_USAGE_PAGE(usage))) { continue; } if (is_child && (id->match_flag_usage) && (id->usage != HID_GET_USAGE(usage))) { continue; } if ((id->match_flag_bus) && (id->idBus != info->idBus)) { continue; } if ((id->match_flag_vendor) && (id->idVendor != info->idVendor)) { continue; } if ((id->match_flag_product) && (id->idProduct != info->idProduct)) { continue; } if ((id->match_flag_ver_lo) && (id->idVersion_lo > info->idVersion)) { continue; } if ((id->match_flag_ver_hi) && (id->idVersion_hi < info->idVersion)) { continue; } if (id->match_flag_pnp && strncmp(id->idPnP, info->idPnP, HID_PNP_ID_SIZE) != 0) { continue; } /* We found a match! */ return (id); } done: return (NULL); } /*------------------------------------------------------------------------* * hidbus_lookup_driver_info - factored out code * * Return values: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ int hidbus_lookup_driver_info(device_t child, const struct hid_device_id *id, int nitems_id) { id = hidbus_lookup_id(child, id, nitems_id); if (id) { /* copy driver info */ hidbus_set_driver_info(child, id->driver_info); return (0); } return (ENXIO); } const struct hid_device_info * hid_get_device_info(device_t dev) { device_t bus; bus = device_get_devclass(dev) == devclass_find("hidbus") ? dev : device_get_parent(dev); return (device_get_ivars(bus)); } static device_method_t hidbus_methods[] = { /* device interface */ DEVMETHOD(device_probe, hidbus_probe), DEVMETHOD(device_attach, hidbus_attach), DEVMETHOD(device_detach, hidbus_detach), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* bus interface */ DEVMETHOD(bus_add_child, hidbus_add_child), DEVMETHOD(bus_child_detached, hidbus_child_detached), DEVMETHOD(bus_child_deleted, hidbus_child_deleted), DEVMETHOD(bus_read_ivar, hidbus_read_ivar), DEVMETHOD(bus_write_ivar, hidbus_write_ivar), DEVMETHOD(bus_child_pnpinfo, hidbus_child_pnpinfo), DEVMETHOD(bus_child_location, hidbus_child_location), /* hid interface */ DEVMETHOD(hid_intr_start, hidbus_intr_start), DEVMETHOD(hid_intr_stop, hidbus_intr_stop), DEVMETHOD(hid_intr_poll, hidbus_intr_poll), DEVMETHOD(hid_get_rdesc, hidbus_get_rdesc), DEVMETHOD(hid_read, hidbus_read), DEVMETHOD(hid_write, hidbus_write), DEVMETHOD(hid_get_report, hidbus_get_report), DEVMETHOD(hid_set_report, hidbus_set_report), DEVMETHOD(hid_set_idle, hidbus_set_idle), DEVMETHOD(hid_set_protocol, hidbus_set_protocol), DEVMETHOD(hid_ioctl, hidbus_ioctl), DEVMETHOD_END }; driver_t hidbus_driver = { "hidbus", hidbus_methods, sizeof(struct hidbus_softc), }; MODULE_DEPEND(hidbus, hid, 1, 1, 1); MODULE_VERSION(hidbus, 1); DRIVER_MODULE(hidbus, atopcase, hidbus_driver, 0, 0); DRIVER_MODULE(hidbus, hvhid, hidbus_driver, 0, 0); DRIVER_MODULE(hidbus, iichid, hidbus_driver, 0, 0); DRIVER_MODULE(hidbus, usbhid, hidbus_driver, 0, 0); diff --git a/sys/dev/ida/ida.c b/sys/dev/ida/ida.c index ea231b041e13..fa135b7fc9ed 100644 --- a/sys/dev/ida/ida.c +++ b/sys/dev/ida/ida.c @@ -1,839 +1,836 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1999,2000 Jonathan Lemon * All rights reserved. * # Derived from the original IDA Compaq RAID driver, which is * Copyright (c) 1996, 1997, 1998, 1999 * Mark Dawson and David James. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * Generic driver for Compaq SMART RAID adapters. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* prototypes */ static int ida_alloc_qcbs(struct ida_softc *ida); static void ida_done(struct ida_softc *ida, struct ida_qcb *qcb); static void ida_start(struct ida_softc *ida); static void ida_startio(struct ida_softc *ida); static void ida_startup(void *arg); static void ida_timeout(void *arg); static int ida_wait(struct ida_softc *ida, struct ida_qcb *qcb); static d_ioctl_t ida_ioctl; static struct cdevsw ida_cdevsw = { .d_version = D_VERSION, .d_ioctl = ida_ioctl, .d_name = "ida", }; void ida_free(struct ida_softc *ida) { int i; if (ida->ih != NULL) bus_teardown_intr(ida->dev, ida->irq, ida->ih); mtx_lock(&ida->lock); callout_stop(&ida->ch); mtx_unlock(&ida->lock); callout_drain(&ida->ch); if (ida->buffer_dmat) { for (i = 0; i < IDA_QCB_MAX; i++) bus_dmamap_destroy(ida->buffer_dmat, ida->qcbs[i].dmamap); bus_dma_tag_destroy(ida->buffer_dmat); } if (ida->hwqcb_dmat) { if (ida->hwqcb_busaddr) bus_dmamap_unload(ida->hwqcb_dmat, ida->hwqcb_dmamap); if (ida->hwqcbs) bus_dmamem_free(ida->hwqcb_dmat, ida->hwqcbs, ida->hwqcb_dmamap); bus_dma_tag_destroy(ida->hwqcb_dmat); } if (ida->qcbs != NULL) free(ida->qcbs, M_DEVBUF); if (ida->irq != NULL) bus_release_resource(ida->dev, ida->irq_res_type, 0, ida->irq); if (ida->parent_dmat != NULL) bus_dma_tag_destroy(ida->parent_dmat); if (ida->regs != NULL) bus_release_resource(ida->dev, ida->regs_res_type, ida->regs_res_id, ida->regs); mtx_destroy(&ida->lock); } /* * record bus address from bus_dmamap_load */ static void ida_dma_map_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *baddr; baddr = (bus_addr_t *)arg; *baddr = segs->ds_addr; } static __inline struct ida_qcb * ida_get_qcb(struct ida_softc *ida) { struct ida_qcb *qcb; if ((qcb = SLIST_FIRST(&ida->free_qcbs)) != NULL) { SLIST_REMOVE_HEAD(&ida->free_qcbs, link.sle); bzero(qcb->hwqcb, sizeof(struct ida_hdr) + sizeof(struct ida_req)); } return (qcb); } static __inline void ida_free_qcb(struct ida_softc *ida, struct ida_qcb *qcb) { qcb->state = QCB_FREE; qcb->buf = NULL; qcb->error = 0; SLIST_INSERT_HEAD(&ida->free_qcbs, qcb, link.sle); } static __inline bus_addr_t idahwqcbvtop(struct ida_softc *ida, struct ida_hardware_qcb *hwqcb) { return (ida->hwqcb_busaddr + ((bus_addr_t)hwqcb - (bus_addr_t)ida->hwqcbs)); } static __inline struct ida_qcb * idahwqcbptov(struct ida_softc *ida, bus_addr_t hwqcb_addr) { struct ida_hardware_qcb *hwqcb; hwqcb = (struct ida_hardware_qcb *) ((bus_addr_t)ida->hwqcbs + (hwqcb_addr - ida->hwqcb_busaddr)); return (hwqcb->qcb); } static int ida_alloc_qcbs(struct ida_softc *ida) { struct ida_qcb *qcb; int error, i; for (i = 0; i < IDA_QCB_MAX; i++) { qcb = &ida->qcbs[i]; error = bus_dmamap_create(ida->buffer_dmat, /*flags*/0, &qcb->dmamap); if (error != 0) return (error); qcb->ida = ida; qcb->flags = QCB_FREE; qcb->hwqcb = &ida->hwqcbs[i]; qcb->hwqcb->qcb = qcb; qcb->hwqcb_busaddr = idahwqcbvtop(ida, qcb->hwqcb); SLIST_INSERT_HEAD(&ida->free_qcbs, qcb, link.sle); } return (0); } int ida_setup(struct ida_softc *ida) { struct ida_controller_info cinfo; device_t child; int error, i, unit; SLIST_INIT(&ida->free_qcbs); STAILQ_INIT(&ida->qcb_queue); bioq_init(&ida->bio_queue); ida->qcbs = (struct ida_qcb *) malloc(IDA_QCB_MAX * sizeof(struct ida_qcb), M_DEVBUF, M_NOWAIT | M_ZERO); if (ida->qcbs == NULL) return (ENOMEM); /* * Create our DMA tags */ /* DMA tag for our hardware QCB structures */ error = bus_dma_tag_create( /* parent */ ida->parent_dmat, /* alignment */ 1, /* boundary */ 0, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ IDA_QCB_MAX * sizeof(struct ida_hardware_qcb), /* nsegments */ 1, /* maxsegsz */ BUS_SPACE_MAXSIZE_32BIT, /* flags */ 0, /* lockfunc */ NULL, /* lockarg */ NULL, &ida->hwqcb_dmat); if (error) return (ENOMEM); /* DMA tag for mapping buffers into device space */ error = bus_dma_tag_create( /* parent */ ida->parent_dmat, /* alignment */ 1, /* boundary */ 0, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ DFLTPHYS, /* nsegments */ IDA_NSEG, /* maxsegsz */ BUS_SPACE_MAXSIZE_32BIT, /* flags */ 0, /* lockfunc */ busdma_lock_mutex, /* lockarg */ &ida->lock, &ida->buffer_dmat); if (error) return (ENOMEM); /* Allocation of hardware QCBs */ /* XXX allocation is rounded to hardware page size */ error = bus_dmamem_alloc(ida->hwqcb_dmat, (void **)&ida->hwqcbs, BUS_DMA_NOWAIT, &ida->hwqcb_dmamap); if (error) return (ENOMEM); /* And permanently map them in */ bus_dmamap_load(ida->hwqcb_dmat, ida->hwqcb_dmamap, ida->hwqcbs, IDA_QCB_MAX * sizeof(struct ida_hardware_qcb), ida_dma_map_cb, &ida->hwqcb_busaddr, /*flags*/0); bzero(ida->hwqcbs, IDA_QCB_MAX * sizeof(struct ida_hardware_qcb)); error = ida_alloc_qcbs(ida); if (error) return (error); mtx_lock(&ida->lock); ida->cmd.int_enable(ida, 0); error = ida_command(ida, CMD_GET_CTRL_INFO, &cinfo, sizeof(cinfo), IDA_CONTROLLER, 0, DMA_DATA_IN); if (error) { mtx_unlock(&ida->lock); device_printf(ida->dev, "CMD_GET_CTRL_INFO failed.\n"); return (error); } device_printf(ida->dev, "drives=%d firm_rev=%c%c%c%c\n", cinfo.num_drvs, cinfo.firm_rev[0], cinfo.firm_rev[1], cinfo.firm_rev[2], cinfo.firm_rev[3]); if (ida->flags & IDA_FIRMWARE) { int data; error = ida_command(ida, CMD_START_FIRMWARE, &data, sizeof(data), IDA_CONTROLLER, 0, DMA_DATA_IN); if (error) { mtx_unlock(&ida->lock); device_printf(ida->dev, "CMD_START_FIRMWARE failed.\n"); return (error); } } ida->cmd.int_enable(ida, 1); ida->flags |= IDA_ATTACHED; mtx_unlock(&ida->lock); for (i = 0; i < cinfo.num_drvs; i++) { child = device_add_child(ida->dev, /*"idad"*/NULL, DEVICE_UNIT_ANY); if (child != NULL) device_set_ivars(child, (void *)(intptr_t)i); } ida->ich.ich_func = ida_startup; ida->ich.ich_arg = ida; if (config_intrhook_establish(&ida->ich) != 0) { device_delete_children(ida->dev); device_printf(ida->dev, "Cannot establish configuration hook\n"); return (error); } unit = device_get_unit(ida->dev); ida->ida_dev_t = make_dev(&ida_cdevsw, unit, UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR, "ida%d", unit); ida->ida_dev_t->si_drv1 = ida; return (0); } static void ida_startup(void *arg) { struct ida_softc *ida; ida = arg; config_intrhook_disestablish(&ida->ich); bus_topo_lock(); bus_attach_children(ida->dev); bus_topo_unlock(); } int ida_detach(device_t dev) { struct ida_softc *ida; int error; ida = (struct ida_softc *)device_get_softc(dev); error = bus_generic_detach(dev); - if (error) - return (error); - error = device_delete_children(dev); if (error) return (error); /* * XXX * before detaching, we must make sure that the system is * quiescent; nothing mounted, no pending activity. */ /* * XXX * now, how are we supposed to maintain a list of our drives? * iterate over our "child devices"? */ destroy_dev(ida->ida_dev_t); ida_free(ida); return (error); } static void ida_data_cb(void *arg, bus_dma_segment_t *segs, int nsegments, int error) { struct ida_hardware_qcb *hwqcb; struct ida_softc *ida; struct ida_qcb *qcb; bus_dmasync_op_t op; int i; qcb = arg; ida = qcb->ida; if (!dumping) mtx_assert(&ida->lock, MA_OWNED); if (error) { qcb->error = error; ida_done(ida, qcb); return; } hwqcb = qcb->hwqcb; hwqcb->hdr.size = htole16((sizeof(struct ida_req) + sizeof(struct ida_sgb) * IDA_NSEG) >> 2); for (i = 0; i < nsegments; i++) { hwqcb->seg[i].addr = htole32(segs[i].ds_addr); hwqcb->seg[i].length = htole32(segs[i].ds_len); } hwqcb->req.sgcount = nsegments; if (qcb->flags & DMA_DATA_TRANSFER) { switch (qcb->flags & DMA_DATA_TRANSFER) { case DMA_DATA_TRANSFER: op = BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE; break; case DMA_DATA_IN: op = BUS_DMASYNC_PREREAD; break; default: KASSERT((qcb->flags & DMA_DATA_TRANSFER) == DMA_DATA_OUT, ("bad DMA data flags")); op = BUS_DMASYNC_PREWRITE; break; } bus_dmamap_sync(ida->buffer_dmat, qcb->dmamap, op); } bus_dmamap_sync(ida->hwqcb_dmat, ida->hwqcb_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); STAILQ_INSERT_TAIL(&ida->qcb_queue, qcb, link.stqe); ida_start(ida); ida->flags &= ~IDA_QFROZEN; } static int ida_map_qcb(struct ida_softc *ida, struct ida_qcb *qcb, void *data, bus_size_t datasize) { int error, flags; if (ida->flags & IDA_INTERRUPTS) flags = BUS_DMA_WAITOK; else flags = BUS_DMA_NOWAIT; error = bus_dmamap_load(ida->buffer_dmat, qcb->dmamap, data, datasize, ida_data_cb, qcb, flags); if (error == EINPROGRESS) { ida->flags |= IDA_QFROZEN; error = 0; } return (error); } int ida_command(struct ida_softc *ida, int command, void *data, int datasize, int drive, u_int32_t pblkno, int flags) { struct ida_hardware_qcb *hwqcb; struct ida_qcb *qcb; int error; if (!dumping) mtx_assert(&ida->lock, MA_OWNED); qcb = ida_get_qcb(ida); if (qcb == NULL) { device_printf(ida->dev, "out of QCBs\n"); return (EAGAIN); } qcb->flags = flags | IDA_COMMAND; hwqcb = qcb->hwqcb; hwqcb->hdr.drive = drive; hwqcb->req.blkno = htole32(pblkno); hwqcb->req.bcount = htole16(howmany(datasize, DEV_BSIZE)); hwqcb->req.command = command; error = ida_map_qcb(ida, qcb, data, datasize); if (error == 0) { error = ida_wait(ida, qcb); /* Don't free QCB on a timeout in case it later completes. */ if (error) return (error); error = qcb->error; } /* XXX should have status returned here? */ /* XXX have "status pointer" area in QCB? */ ida_free_qcb(ida, qcb); return (error); } void ida_submit_buf(struct ida_softc *ida, struct bio *bp) { mtx_lock(&ida->lock); bioq_insert_tail(&ida->bio_queue, bp); ida_startio(ida); mtx_unlock(&ida->lock); } static void ida_startio(struct ida_softc *ida) { struct ida_hardware_qcb *hwqcb; struct ida_qcb *qcb; struct idad_softc *drv; struct bio *bp; int error; mtx_assert(&ida->lock, MA_OWNED); for (;;) { if (ida->flags & IDA_QFROZEN) return; bp = bioq_first(&ida->bio_queue); if (bp == NULL) return; /* no more buffers */ qcb = ida_get_qcb(ida); if (qcb == NULL) return; /* out of resources */ bioq_remove(&ida->bio_queue, bp); qcb->buf = bp; qcb->flags = bp->bio_cmd == BIO_READ ? DMA_DATA_IN : DMA_DATA_OUT; hwqcb = qcb->hwqcb; drv = bp->bio_driver1; hwqcb->hdr.drive = drv->drive; hwqcb->req.blkno = bp->bio_pblkno; hwqcb->req.bcount = howmany(bp->bio_bcount, DEV_BSIZE); hwqcb->req.command = bp->bio_cmd == BIO_READ ? CMD_READ : CMD_WRITE; error = ida_map_qcb(ida, qcb, bp->bio_data, bp->bio_bcount); if (error) { qcb->error = error; ida_done(ida, qcb); } } } static void ida_start(struct ida_softc *ida) { struct ida_qcb *qcb; if (!dumping) mtx_assert(&ida->lock, MA_OWNED); while ((qcb = STAILQ_FIRST(&ida->qcb_queue)) != NULL) { if (ida->cmd.fifo_full(ida)) break; STAILQ_REMOVE_HEAD(&ida->qcb_queue, link.stqe); /* * XXX * place the qcb on an active list? */ /* Set a timeout. */ if (!ida->qactive && !dumping) callout_reset(&ida->ch, hz * 5, ida_timeout, ida); ida->qactive++; qcb->state = QCB_ACTIVE; ida->cmd.submit(ida, qcb); } } static int ida_wait(struct ida_softc *ida, struct ida_qcb *qcb) { struct ida_qcb *qcb_done = NULL; bus_addr_t completed; int delay; if (!dumping) mtx_assert(&ida->lock, MA_OWNED); if (ida->flags & IDA_INTERRUPTS) { if (mtx_sleep(qcb, &ida->lock, PRIBIO, "idacmd", 5 * hz)) { qcb->state = QCB_TIMEDOUT; return (ETIMEDOUT); } return (0); } again: delay = 5 * 1000 * 100; /* 5 sec delay */ while ((completed = ida->cmd.done(ida)) == 0) { if (delay-- == 0) { qcb->state = QCB_TIMEDOUT; return (ETIMEDOUT); } DELAY(10); } qcb_done = idahwqcbptov(ida, completed & ~3); if (qcb_done != qcb) goto again; ida_done(ida, qcb); return (0); } void ida_intr(void *data) { struct ida_softc *ida; struct ida_qcb *qcb; bus_addr_t completed; ida = (struct ida_softc *)data; mtx_lock(&ida->lock); if (ida->cmd.int_pending(ida) == 0) { mtx_unlock(&ida->lock); return; /* not our interrupt */ } while ((completed = ida->cmd.done(ida)) != 0) { qcb = idahwqcbptov(ida, completed & ~3); if (qcb == NULL || qcb->state != QCB_ACTIVE) { device_printf(ida->dev, "ignoring completion %jx\n", (intmax_t)completed); continue; } /* Handle "Bad Command List" errors. */ if ((completed & 3) && (qcb->hwqcb->req.error == 0)) qcb->hwqcb->req.error = CMD_REJECTED; ida_done(ida, qcb); } ida_startio(ida); mtx_unlock(&ida->lock); } /* * should switch out command type; may be status, not just I/O. */ static void ida_done(struct ida_softc *ida, struct ida_qcb *qcb) { bus_dmasync_op_t op; int active, error = 0; /* * finish up command */ if (!dumping) mtx_assert(&ida->lock, MA_OWNED); active = (qcb->state != QCB_FREE); if (qcb->flags & DMA_DATA_TRANSFER && active) { switch (qcb->flags & DMA_DATA_TRANSFER) { case DMA_DATA_TRANSFER: op = BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE; break; case DMA_DATA_IN: op = BUS_DMASYNC_POSTREAD; break; default: KASSERT((qcb->flags & DMA_DATA_TRANSFER) == DMA_DATA_OUT, ("bad DMA data flags")); op = BUS_DMASYNC_POSTWRITE; break; } bus_dmamap_sync(ida->buffer_dmat, qcb->dmamap, op); bus_dmamap_unload(ida->buffer_dmat, qcb->dmamap); } if (active) bus_dmamap_sync(ida->hwqcb_dmat, ida->hwqcb_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); if (qcb->hwqcb->req.error & SOFT_ERROR) { if (qcb->buf) device_printf(ida->dev, "soft %s error\n", qcb->buf->bio_cmd == BIO_READ ? "read" : "write"); else device_printf(ida->dev, "soft error\n"); } if (qcb->hwqcb->req.error & HARD_ERROR) { error = 1; if (qcb->buf) device_printf(ida->dev, "hard %s error\n", qcb->buf->bio_cmd == BIO_READ ? "read" : "write"); else device_printf(ida->dev, "hard error\n"); } if (qcb->hwqcb->req.error & CMD_REJECTED) { error = 1; device_printf(ida->dev, "invalid request\n"); } if (qcb->error) { error = 1; device_printf(ida->dev, "request failed to map: %d\n", qcb->error); } if (qcb->flags & IDA_COMMAND) { if (ida->flags & IDA_INTERRUPTS) wakeup(qcb); if (qcb->state == QCB_TIMEDOUT) ida_free_qcb(ida, qcb); } else { KASSERT(qcb->buf != NULL, ("ida_done(): qcb->buf is NULL!")); if (error) qcb->buf->bio_flags |= BIO_ERROR; idad_intr(qcb->buf); ida_free_qcb(ida, qcb); } if (!active) return; ida->qactive--; /* Reschedule or cancel timeout */ if (ida->qactive) callout_reset(&ida->ch, hz * 5, ida_timeout, ida); else callout_stop(&ida->ch); } static void ida_timeout(void *arg) { struct ida_softc *ida; ida = (struct ida_softc *)arg; device_printf(ida->dev, "%s() qactive %d\n", __func__, ida->qactive); if (ida->flags & IDA_INTERRUPTS) device_printf(ida->dev, "IDA_INTERRUPTS\n"); device_printf(ida->dev, "\t R_CMD_FIFO: %08x\n" "\t R_DONE_FIFO: %08x\n" "\t R_INT_MASK: %08x\n" "\t R_STATUS: %08x\n" "\tR_INT_PENDING: %08x\n", ida_inl(ida, R_CMD_FIFO), ida_inl(ida, R_DONE_FIFO), ida_inl(ida, R_INT_MASK), ida_inl(ida, R_STATUS), ida_inl(ida, R_INT_PENDING)); return; } /* * IOCTL stuff follows. */ struct cmd_info { int cmd; int len; int flags; }; static struct cmd_info *ida_cmd_lookup(int); static int ida_ioctl (struct cdev *dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *td) { struct ida_softc *sc; struct ida_user_command *uc; struct cmd_info *ci; int len; int flags; int error; int data; void *daddr; sc = (struct ida_softc *)dev->si_drv1; uc = (struct ida_user_command *)addr; error = 0; switch (cmd) { case IDAIO_COMMAND: ci = ida_cmd_lookup(uc->command); if (ci == NULL) { error = EINVAL; break; } len = ci->len; flags = ci->flags; if (len) daddr = &uc->d.buf; else { daddr = &data; len = sizeof(data); } mtx_lock(&sc->lock); error = ida_command(sc, uc->command, daddr, len, uc->drive, uc->blkno, flags); mtx_unlock(&sc->lock); break; default: error = ENOIOCTL; break; } return (error); } static struct cmd_info ci_list[] = { { CMD_GET_LOG_DRV_INFO, sizeof(struct ida_drive_info), DMA_DATA_IN }, { CMD_GET_CTRL_INFO, sizeof(struct ida_controller_info), DMA_DATA_IN }, { CMD_SENSE_DRV_STATUS, sizeof(struct ida_drive_status), DMA_DATA_IN }, { CMD_START_RECOVERY, 0, 0 }, { CMD_GET_PHYS_DRV_INFO, sizeof(struct ida_phys_drv_info), DMA_DATA_TRANSFER }, { CMD_BLINK_DRV_LEDS, sizeof(struct ida_blink_drv_leds), DMA_DATA_OUT }, { CMD_SENSE_DRV_LEDS, sizeof(struct ida_blink_drv_leds), DMA_DATA_IN }, { CMD_GET_LOG_DRV_EXT, sizeof(struct ida_drive_info_ext), DMA_DATA_IN }, { CMD_RESET_CTRL, 0, 0 }, { CMD_GET_CONFIG, 0, 0 }, { CMD_SET_CONFIG, 0, 0 }, { CMD_LABEL_LOG_DRV, sizeof(struct ida_label_logical), DMA_DATA_OUT }, { CMD_SET_SURFACE_DELAY, 0, 0 }, { CMD_SENSE_BUS_PARAMS, 0, 0 }, { CMD_SENSE_SUBSYS_INFO, 0, 0 }, { CMD_SENSE_SURFACE_ATS, 0, 0 }, { CMD_PASSTHROUGH, 0, 0 }, { CMD_RESET_SCSI_DEV, 0, 0 }, { CMD_PAUSE_BG_ACT, 0, 0 }, { CMD_RESUME_BG_ACT, 0, 0 }, { CMD_START_FIRMWARE, 0, 0 }, { CMD_SENSE_DRV_ERR_LOG, 0, 0 }, { CMD_START_CPM, 0, 0 }, { CMD_SENSE_CP, 0, 0 }, { CMD_STOP_CPM, 0, 0 }, { CMD_FLUSH_CACHE, 0, 0 }, { CMD_ACCEPT_MEDIA_EXCH, 0, 0 }, { 0, 0, 0 } }; static struct cmd_info * ida_cmd_lookup (int command) { struct cmd_info *ci; ci = ci_list; while (ci->cmd) { if (ci->cmd == command) return (ci); ci++; } return (NULL); } diff --git a/sys/dev/iicbus/controller/opencores/iicoc_pci.c b/sys/dev/iicbus/controller/opencores/iicoc_pci.c index cecfa888349b..f01b4514936e 100644 --- a/sys/dev/iicbus/controller/opencores/iicoc_pci.c +++ b/sys/dev/iicbus/controller/opencores/iicoc_pci.c @@ -1,138 +1,137 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2003-2012 Broadcom Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" #include "iicoc.h" static int iicoc_detach(device_t dev) { struct iicoc_softc *sc; sc = device_get_softc(dev); - device_delete_children(dev); bus_generic_detach(dev); bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); mtx_destroy(&sc->sc_mtx); return (0); } /* * We add all the devices which we know about. * The generic attach routine will attach them if they are alive. */ static int iicoc_attach(device_t dev) { struct iicoc_softc *sc; sc = device_get_softc(dev); sc->dev = dev; mtx_init(&sc->sc_mtx, "iicoc", "iicoc", MTX_DEF); sc->mem_rid = 0; sc->mem_res = bus_alloc_resource_anywhere(dev, SYS_RES_MEMORY, &sc->mem_rid, 0x100, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "Could not allocate bus resource.\n"); return (-1); } iicoc_init(dev); sc->iicbus = device_add_child(dev, "iicbus", DEVICE_UNIT_ANY); if (sc->iicbus == NULL) { device_printf(dev, "Could not allocate iicbus instance.\n"); bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); mtx_destroy(&sc->sc_mtx); return (-1); } bus_attach_children(dev); return (0); } static int iicoc_probe(device_t dev) { struct iicoc_softc *sc; sc = device_get_softc(dev); if ((pci_get_vendor(dev) == 0x184e) && (pci_get_device(dev) == 0x1011)) { sc->clockfreq = XLP_I2C_CLKFREQ; sc->i2cfreq = XLP_I2C_FREQ; sc->reg_shift = 2; device_set_desc(dev, "Netlogic XLP I2C Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static device_method_t iicoc_methods[] = { /* device interface */ DEVMETHOD(device_probe, iicoc_probe), DEVMETHOD(device_attach, iicoc_attach), DEVMETHOD(device_detach, iicoc_detach), /* iicbus interface */ DEVMETHOD(iicbus_callback, iicbus_null_callback), DEVMETHOD(iicbus_repeated_start, iicoc_iicbus_repeated_start), DEVMETHOD(iicbus_start, iicoc_iicbus_start), DEVMETHOD(iicbus_stop, iicoc_iicbus_stop), DEVMETHOD(iicbus_reset, iicoc_iicbus_reset), DEVMETHOD(iicbus_write, iicoc_iicbus_write), DEVMETHOD(iicbus_read, iicoc_iicbus_read), DEVMETHOD(iicbus_transfer, iicbus_transfer_gen), DEVMETHOD_END }; static driver_t iicoc_driver = { "iicoc", iicoc_methods, sizeof(struct iicoc_softc), }; DRIVER_MODULE(iicoc, pci, iicoc_driver, 0, 0); diff --git a/sys/dev/iicbus/iicsmb.c b/sys/dev/iicbus/iicsmb.c index e502ad904070..a5885648632e 100644 --- a/sys/dev/iicbus/iicsmb.c +++ b/sys/dev/iicbus/iicsmb.c @@ -1,478 +1,477 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998, 2001 Nicolas Souchu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * I2C to SMB bridge * * Example: * * smb bttv * \ / * smbus * / \ * iicsmb bti2c * | * iicbus * / | \ * iicbb pcf ... * | * lpbb */ #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" #include "smbus_if.h" struct iicsmb_softc { #define SMB_WAITING_ADDR 0x0 #define SMB_WAITING_LOW 0x1 #define SMB_WAITING_HIGH 0x2 #define SMB_DONE 0x3 int state; u_char devaddr; /* slave device address */ char low; /* low byte received first */ char high; /* high byte */ struct mtx lock; device_t smbus; }; static int iicsmb_probe(device_t); static int iicsmb_attach(device_t); static int iicsmb_detach(device_t); static void iicsmb_identify(driver_t *driver, device_t parent); static int iicsmb_intr(device_t dev, int event, char *buf); static int iicsmb_callback(device_t dev, int index, void *data); static int iicsmb_quick(device_t dev, u_char slave, int how); static int iicsmb_sendb(device_t dev, u_char slave, char byte); static int iicsmb_recvb(device_t dev, u_char slave, char *byte); static int iicsmb_writeb(device_t dev, u_char slave, char cmd, char byte); static int iicsmb_writew(device_t dev, u_char slave, char cmd, short word); static int iicsmb_readb(device_t dev, u_char slave, char cmd, char *byte); static int iicsmb_readw(device_t dev, u_char slave, char cmd, short *word); static int iicsmb_pcall(device_t dev, u_char slave, char cmd, short sdata, short *rdata); static int iicsmb_bwrite(device_t dev, u_char slave, char cmd, u_char count, char *buf); static int iicsmb_bread(device_t dev, u_char slave, char cmd, u_char *count, char *buf); static device_method_t iicsmb_methods[] = { /* device interface */ DEVMETHOD(device_identify, iicsmb_identify), DEVMETHOD(device_probe, iicsmb_probe), DEVMETHOD(device_attach, iicsmb_attach), DEVMETHOD(device_detach, iicsmb_detach), /* iicbus interface */ DEVMETHOD(iicbus_intr, iicsmb_intr), /* smbus interface */ DEVMETHOD(smbus_callback, iicsmb_callback), DEVMETHOD(smbus_quick, iicsmb_quick), DEVMETHOD(smbus_sendb, iicsmb_sendb), DEVMETHOD(smbus_recvb, iicsmb_recvb), DEVMETHOD(smbus_writeb, iicsmb_writeb), DEVMETHOD(smbus_writew, iicsmb_writew), DEVMETHOD(smbus_readb, iicsmb_readb), DEVMETHOD(smbus_readw, iicsmb_readw), DEVMETHOD(smbus_pcall, iicsmb_pcall), DEVMETHOD(smbus_bwrite, iicsmb_bwrite), DEVMETHOD(smbus_bread, iicsmb_bread), DEVMETHOD_END }; static driver_t iicsmb_driver = { "iicsmb", iicsmb_methods, sizeof(struct iicsmb_softc), }; static void iicsmb_identify(driver_t *driver, device_t parent) { if (device_find_child(parent, "iicsmb", -1) == NULL) BUS_ADD_CHILD(parent, 0, "iicsmb", DEVICE_UNIT_ANY); } static int iicsmb_probe(device_t dev) { device_set_desc(dev, "SMBus over I2C bridge"); return (BUS_PROBE_NOWILDCARD); } static int iicsmb_attach(device_t dev) { struct iicsmb_softc *sc = (struct iicsmb_softc *)device_get_softc(dev); mtx_init(&sc->lock, "iicsmb", NULL, MTX_DEF); sc->smbus = device_add_child(dev, "smbus", DEVICE_UNIT_ANY); /* probe and attach the smbus */ bus_attach_children(dev); return (0); } static int iicsmb_detach(device_t dev) { struct iicsmb_softc *sc = (struct iicsmb_softc *)device_get_softc(dev); bus_generic_detach(dev); - device_delete_children(dev); mtx_destroy(&sc->lock); return (0); } /* * iicsmb_intr() * * iicbus interrupt handler */ static int iicsmb_intr(device_t dev, int event, char *buf) { struct iicsmb_softc *sc = (struct iicsmb_softc *)device_get_softc(dev); mtx_lock(&sc->lock); switch (event) { case INTR_GENERAL: case INTR_START: sc->state = SMB_WAITING_ADDR; break; case INTR_STOP: /* call smbus intr handler */ smbus_intr(sc->smbus, sc->devaddr, sc->low, sc->high, SMB_ENOERR); break; case INTR_RECEIVE: switch (sc->state) { case SMB_DONE: /* XXX too much data, discard */ printf("%s: too much data from 0x%x\n", __func__, sc->devaddr & 0xff); goto end; case SMB_WAITING_ADDR: sc->devaddr = (u_char)*buf; sc->state = SMB_WAITING_LOW; break; case SMB_WAITING_LOW: sc->low = *buf; sc->state = SMB_WAITING_HIGH; break; case SMB_WAITING_HIGH: sc->high = *buf; sc->state = SMB_DONE; break; } end: break; case INTR_TRANSMIT: case INTR_NOACK: break; case INTR_ERROR: switch (*buf) { case IIC_EBUSERR: smbus_intr(sc->smbus, sc->devaddr, 0, 0, SMB_EBUSERR); break; default: printf("%s unknown error 0x%x!\n", __func__, (int)*buf); break; } break; default: panic("%s: unknown event (%d)!", __func__, event); } mtx_unlock(&sc->lock); return (0); } static int iicsmb_callback(device_t dev, int index, void *data) { device_t parent = device_get_parent(dev); int error = 0; int how; switch (index) { case SMB_REQUEST_BUS: /* request underlying iicbus */ how = *(int *)data; error = iicbus_request_bus(parent, dev, how); break; case SMB_RELEASE_BUS: /* release underlying iicbus */ error = iicbus_release_bus(parent, dev); break; default: error = EINVAL; } return (error); } static int iic2smb_error(int error) { switch (error) { case IIC_NOERR: return (SMB_ENOERR); case IIC_EBUSERR: return (SMB_EBUSERR); case IIC_ENOACK: return (SMB_ENOACK); case IIC_ETIMEOUT: return (SMB_ETIMEOUT); case IIC_EBUSBSY: return (SMB_EBUSY); case IIC_ESTATUS: return (SMB_EBUSERR); case IIC_EUNDERFLOW: return (SMB_EBUSERR); case IIC_EOVERFLOW: return (SMB_EBUSERR); case IIC_ENOTSUPP: return (SMB_ENOTSUPP); case IIC_ENOADDR: return (SMB_EBUSERR); case IIC_ERESOURCE: return (SMB_EBUSERR); default: return (SMB_EBUSERR); } } #define TRANSFER_MSGS(dev, msgs) iicbus_transfer(dev, msgs, nitems(msgs)) static int iicsmb_quick(device_t dev, u_char slave, int how) { struct iic_msg msgs[] = { { slave, how == SMB_QWRITE ? IIC_M_WR : IIC_M_RD, 0, NULL }, }; int error; switch (how) { case SMB_QWRITE: case SMB_QREAD: break; default: return (SMB_EINVAL); } error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_sendb(device_t dev, u_char slave, char byte) { struct iic_msg msgs[] = { { slave, IIC_M_WR, 1, &byte }, }; int error; error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_recvb(device_t dev, u_char slave, char *byte) { struct iic_msg msgs[] = { { slave, IIC_M_RD, 1, byte }, }; int error; error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_writeb(device_t dev, u_char slave, char cmd, char byte) { uint8_t bytes[] = { cmd, byte }; struct iic_msg msgs[] = { { slave, IIC_M_WR, nitems(bytes), bytes }, }; int error; error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_writew(device_t dev, u_char slave, char cmd, short word) { uint8_t bytes[] = { cmd, word & 0xff, word >> 8 }; struct iic_msg msgs[] = { { slave, IIC_M_WR, nitems(bytes), bytes }, }; int error; error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_readb(device_t dev, u_char slave, char cmd, char *byte) { struct iic_msg msgs[] = { { slave, IIC_M_WR | IIC_M_NOSTOP, 1, &cmd }, { slave, IIC_M_RD, 1, byte }, }; int error; error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_readw(device_t dev, u_char slave, char cmd, short *word) { uint8_t buf[2]; struct iic_msg msgs[] = { { slave, IIC_M_WR | IIC_M_NOSTOP, 1, &cmd }, { slave, IIC_M_RD, nitems(buf), buf }, }; int error; error = TRANSFER_MSGS(dev, msgs); if (error == 0) *word = ((uint16_t)buf[1] << 8) | buf[0]; return (iic2smb_error(error)); } static int iicsmb_pcall(device_t dev, u_char slave, char cmd, short sdata, short *rdata) { uint8_t in[3] = { cmd, sdata & 0xff, sdata >> 8 }; uint8_t out[2]; struct iic_msg msgs[] = { { slave, IIC_M_WR | IIC_M_NOSTOP, nitems(in), in }, { slave, IIC_M_RD, nitems(out), out }, }; int error; error = TRANSFER_MSGS(dev, msgs); if (error == 0) *rdata = ((uint16_t)out[1] << 8) | out[0]; return (iic2smb_error(error)); } static int iicsmb_bwrite(device_t dev, u_char slave, char cmd, u_char count, char *buf) { uint8_t bytes[2] = { cmd, count }; struct iic_msg msgs[] = { { slave, IIC_M_WR | IIC_M_NOSTOP, nitems(bytes), bytes }, { slave, IIC_M_WR | IIC_M_NOSTART, count, buf }, }; int error; if (count > SMB_MAXBLOCKSIZE || count == 0) return (SMB_EINVAL); error = TRANSFER_MSGS(dev, msgs); return (iic2smb_error(error)); } static int iicsmb_bread(device_t dev, u_char slave, char cmd, u_char *count, char *buf) { struct iic_msg msgs[] = { { slave, IIC_M_WR | IIC_M_NOSTOP, 1, &cmd }, { slave, IIC_M_RD | IIC_M_NOSTOP, 1, count }, }; struct iic_msg block_msg[] = { { slave, IIC_M_RD | IIC_M_NOSTART, 0, buf }, }; device_t parent = device_get_parent(dev); int error; /* Have to do this because the command is split in two transfers. */ error = iicbus_request_bus(parent, dev, IIC_WAIT | IIC_RECURSIVE); if (error == 0) error = TRANSFER_MSGS(dev, msgs); if (error == 0) { /* * If the slave offers an empty or a too long reply, * read one byte to generate the stop or abort. */ if (*count > SMB_MAXBLOCKSIZE || *count == 0) block_msg[0].len = 1; else block_msg[0].len = *count; error = TRANSFER_MSGS(dev, block_msg); if (*count > SMB_MAXBLOCKSIZE || *count == 0) error = SMB_EINVAL; } (void)iicbus_release_bus(parent, dev); return (iic2smb_error(error)); } DRIVER_MODULE(iicsmb, iicbus, iicsmb_driver, 0, 0); DRIVER_MODULE(smbus, iicsmb, smbus_driver, 0, 0); MODULE_DEPEND(iicsmb, iicbus, IICBUS_MINVER, IICBUS_PREFVER, IICBUS_MAXVER); MODULE_DEPEND(iicsmb, smbus, SMBUS_MINVER, SMBUS_PREFVER, SMBUS_MAXVER); MODULE_VERSION(iicsmb, 1); diff --git a/sys/dev/nvdimm/nvdimm_acpi.c b/sys/dev/nvdimm/nvdimm_acpi.c index b9803c2c3a05..39697e52eebb 100644 --- a/sys/dev/nvdimm/nvdimm_acpi.c +++ b/sys/dev/nvdimm/nvdimm_acpi.c @@ -1,279 +1,274 @@ /*- * Copyright (c) 2017 The FreeBSD Foundation * Copyright (c) 2018, 2019 Intel Corporation * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_acpi.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define _COMPONENT ACPI_OEM ACPI_MODULE_NAME("NVDIMM_ACPI") struct nvdimm_root_dev { SLIST_HEAD(, SPA_mapping) spas; }; static MALLOC_DEFINE(M_NVDIMM_ACPI, "nvdimm_acpi", "NVDIMM ACPI bus memory"); static ACPI_STATUS find_dimm(ACPI_HANDLE handle, UINT32 nesting_level, void *context, void **return_value) { ACPI_DEVICE_INFO *device_info; ACPI_STATUS status; device_info = NULL; status = AcpiGetObjectInfo(handle, &device_info); if (ACPI_FAILURE(status)) return_ACPI_STATUS(AE_ERROR); if (device_info->Address == (uintptr_t)context) { *(ACPI_HANDLE *)return_value = handle; status = AE_CTRL_TERMINATE; } else status = AE_OK; AcpiOsFree(device_info); return_ACPI_STATUS(status); } static ACPI_HANDLE get_dimm_acpi_handle(ACPI_HANDLE root_handle, nfit_handle_t adr) { ACPI_HANDLE res; ACPI_STATUS status; res = NULL; status = AcpiWalkNamespace(ACPI_TYPE_DEVICE, root_handle, 1, find_dimm, NULL, (void *)(uintptr_t)adr, &res); if (ACPI_FAILURE(status)) res = NULL; return (res); } static int nvdimm_root_create_devs(device_t dev, ACPI_TABLE_NFIT *nfitbl) { ACPI_HANDLE root_handle, dimm_handle; device_t child; nfit_handle_t *dimm_ids, *dimm; uintptr_t *ivars; int num_dimm_ids; root_handle = acpi_get_handle(dev); acpi_nfit_get_dimm_ids(nfitbl, &dimm_ids, &num_dimm_ids); for (dimm = dimm_ids; dimm < dimm_ids + num_dimm_ids; dimm++) { dimm_handle = get_dimm_acpi_handle(root_handle, *dimm); if (dimm_handle == NULL) continue; child = BUS_ADD_CHILD(dev, 100, "nvdimm", DEVICE_UNIT_ANY); if (child == NULL) { device_printf(dev, "failed to create nvdimm\n"); return (ENXIO); } ivars = mallocarray(NVDIMM_ROOT_IVAR_MAX, sizeof(uintptr_t), M_NVDIMM_ACPI, M_ZERO | M_WAITOK); device_set_ivars(child, ivars); nvdimm_root_set_acpi_handle(child, dimm_handle); nvdimm_root_set_device_handle(child, *dimm); } free(dimm_ids, M_NVDIMM_ACPI); return (0); } static int nvdimm_root_create_spas(struct nvdimm_root_dev *dev, ACPI_TABLE_NFIT *nfitbl) { ACPI_NFIT_SYSTEM_ADDRESS **spas, **spa; struct SPA_mapping *spa_mapping; enum SPA_mapping_type spa_type; int error, num_spas; error = 0; acpi_nfit_get_spa_ranges(nfitbl, &spas, &num_spas); for (spa = spas; spa < spas + num_spas; spa++) { spa_type = nvdimm_spa_type_from_uuid( (struct uuid *)(*spa)->RangeGuid); if (spa_type == SPA_TYPE_UNKNOWN) continue; spa_mapping = malloc(sizeof(struct SPA_mapping), M_NVDIMM_ACPI, M_WAITOK | M_ZERO); error = nvdimm_spa_init(spa_mapping, *spa, spa_type); if (error != 0) { nvdimm_spa_fini(spa_mapping); free(spa_mapping, M_NVDIMM_ACPI); break; } if (nvdimm_spa_type_user_accessible(spa_type) && spa_type != SPA_TYPE_CONTROL_REGION) nvdimm_create_namespaces(spa_mapping, nfitbl); SLIST_INSERT_HEAD(&dev->spas, spa_mapping, link); } free(spas, M_NVDIMM_ACPI); return (error); } static char *nvdimm_root_id[] = {"ACPI0012", NULL}; static int nvdimm_root_probe(device_t dev) { int rv; if (acpi_disabled("nvdimm")) return (ENXIO); rv = ACPI_ID_PROBE(device_get_parent(dev), dev, nvdimm_root_id, NULL); if (rv <= 0) device_set_desc(dev, "ACPI NVDIMM root device"); return (rv); } static int nvdimm_root_attach(device_t dev) { struct nvdimm_root_dev *root; ACPI_TABLE_NFIT *nfitbl; ACPI_STATUS status; int error; status = AcpiGetTable(ACPI_SIG_NFIT, 1, (ACPI_TABLE_HEADER **)&nfitbl); if (ACPI_FAILURE(status)) { device_printf(dev, "cannot get NFIT\n"); return (ENXIO); } error = nvdimm_root_create_devs(dev, nfitbl); if (error != 0) return (error); bus_attach_children(dev); root = device_get_softc(dev); error = nvdimm_root_create_spas(root, nfitbl); AcpiPutTable(&nfitbl->Header); return (error); } static void nvdimm_root_child_deleted(device_t dev, device_t child) { free(device_get_ivars(child), M_NVDIMM_ACPI); } static int nvdimm_root_detach(device_t dev) { struct nvdimm_root_dev *root; struct SPA_mapping *spa, *next; - int error; root = device_get_softc(dev); SLIST_FOREACH_SAFE(spa, &root->spas, link, next) { nvdimm_destroy_namespaces(spa); nvdimm_spa_fini(spa); SLIST_REMOVE_HEAD(&root->spas, link); free(spa, M_NVDIMM_ACPI); } - error = bus_generic_detach(dev); - if (error != 0) - return (error); - error = device_delete_children(dev); - return (error); + return (bus_generic_detach(dev)); } static int nvdimm_root_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { if (index < 0 || index >= NVDIMM_ROOT_IVAR_MAX) return (ENOENT); *result = ((uintptr_t *)device_get_ivars(child))[index]; return (0); } static int nvdimm_root_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { if (index < 0 || index >= NVDIMM_ROOT_IVAR_MAX) return (ENOENT); ((uintptr_t *)device_get_ivars(child))[index] = value; return (0); } static int nvdimm_root_child_location(device_t dev, device_t child, struct sbuf *sb) { ACPI_HANDLE handle; handle = nvdimm_root_get_acpi_handle(child); if (handle != NULL) sbuf_printf(sb, "handle=%s", acpi_name(handle)); return (0); } static device_method_t nvdimm_acpi_methods[] = { DEVMETHOD(device_probe, nvdimm_root_probe), DEVMETHOD(device_attach, nvdimm_root_attach), DEVMETHOD(device_detach, nvdimm_root_detach), DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_child_deleted, nvdimm_root_child_deleted), DEVMETHOD(bus_read_ivar, nvdimm_root_read_ivar), DEVMETHOD(bus_write_ivar, nvdimm_root_write_ivar), DEVMETHOD(bus_child_location, nvdimm_root_child_location), DEVMETHOD(bus_get_device_path, acpi_get_acpi_device_path), DEVMETHOD_END }; static driver_t nvdimm_acpi_driver = { "nvdimm_acpi_root", nvdimm_acpi_methods, sizeof(struct nvdimm_root_dev), }; DRIVER_MODULE(nvdimm_acpi_root, acpi, nvdimm_acpi_driver, NULL, NULL); MODULE_DEPEND(nvdimm_acpi_root, acpi, 1, 1, 1); diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c index 64bce9f01ea8..251680590747 100644 --- a/sys/dev/pci/pci.c +++ b/sys/dev/pci/pci.c @@ -1,6935 +1,6933 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1997, Stefan Esser * Copyright (c) 2000, Michael Smith * Copyright (c) 2000, BSDi * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include "opt_acpi.h" #include "opt_iommu.h" #include "opt_bus.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) #include #endif #include #include #include #include #ifdef PCI_IOV #include #include #endif #include #include #include #include #include #include "pcib_if.h" #include "pci_if.h" #define PCIR_IS_BIOS(cfg, reg) \ (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \ ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1)) static device_probe_t pci_probe; static bus_reset_post_t pci_reset_post; static bus_reset_prepare_t pci_reset_prepare; static bus_reset_child_t pci_reset_child; static bus_hint_device_unit_t pci_hint_device_unit; static bus_remap_intr_t pci_remap_intr_method; static pci_get_id_t pci_get_id_method; static int pci_has_quirk(uint32_t devid, int quirk); static pci_addr_t pci_mapbase(uint64_t mapreg); static const char *pci_maptype(uint64_t mapreg); static int pci_maprange(uint64_t mapreg); static pci_addr_t pci_rombase(uint64_t mapreg); static int pci_romsize(uint64_t testval); static void pci_fixancient(pcicfgregs *cfg); static int pci_printf(pcicfgregs *cfg, const char *fmt, ...); static int pci_porten(device_t dev); static int pci_memen(device_t dev); static void pci_assign_interrupt(device_t bus, device_t dev, int force_route); static int pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, int force, int prefetch); static void pci_load_vendor_data(void); static int pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc); static char *pci_describe_device(device_t dev); static int pci_modevent(module_t mod, int what, void *arg); static void pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg); static void pci_read_cap(device_t pcib, pcicfgregs *cfg); static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data); #if 0 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data); #endif static void pci_read_vpd(device_t pcib, pcicfgregs *cfg); static void pci_mask_msix(device_t dev, u_int index); static void pci_unmask_msix(device_t dev, u_int index); static int pci_msi_blacklisted(void); static int pci_msix_blacklisted(void); static void pci_resume_msi(device_t dev); static void pci_resume_msix(device_t dev); static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f, uint16_t vid, uint16_t did); static device_method_t pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, pci_probe), DEVMETHOD(device_attach, pci_attach), DEVMETHOD(device_detach, pci_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, pci_resume), /* Bus interface */ DEVMETHOD(bus_print_child, pci_print_child), DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch), DEVMETHOD(bus_read_ivar, pci_read_ivar), DEVMETHOD(bus_write_ivar, pci_write_ivar), DEVMETHOD(bus_driver_added, pci_driver_added), DEVMETHOD(bus_setup_intr, pci_setup_intr), DEVMETHOD(bus_teardown_intr, pci_teardown_intr), DEVMETHOD(bus_reset_prepare, pci_reset_prepare), DEVMETHOD(bus_reset_post, pci_reset_post), DEVMETHOD(bus_reset_child, pci_reset_child), DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag), DEVMETHOD(bus_get_resource_list,pci_get_resource_list), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_delete_resource, pci_delete_resource), DEVMETHOD(bus_alloc_resource, pci_alloc_resource), DEVMETHOD(bus_adjust_resource, pci_adjust_resource), DEVMETHOD(bus_release_resource, pci_release_resource), DEVMETHOD(bus_activate_resource, pci_activate_resource), DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource), DEVMETHOD(bus_map_resource, pci_map_resource), DEVMETHOD(bus_unmap_resource, pci_unmap_resource), DEVMETHOD(bus_child_deleted, pci_child_deleted), DEVMETHOD(bus_child_detached, pci_child_detached), DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method), DEVMETHOD(bus_child_location, pci_child_location_method), DEVMETHOD(bus_get_device_path, pci_get_device_path_method), DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit), DEVMETHOD(bus_remap_intr, pci_remap_intr_method), DEVMETHOD(bus_suspend_child, pci_suspend_child), DEVMETHOD(bus_resume_child, pci_resume_child), DEVMETHOD(bus_rescan, pci_rescan_method), /* PCI interface */ DEVMETHOD(pci_read_config, pci_read_config_method), DEVMETHOD(pci_write_config, pci_write_config_method), DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method), DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method), DEVMETHOD(pci_enable_io, pci_enable_io_method), DEVMETHOD(pci_disable_io, pci_disable_io_method), DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method), DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method), DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method), DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method), DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method), DEVMETHOD(pci_find_cap, pci_find_cap_method), DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method), DEVMETHOD(pci_find_extcap, pci_find_extcap_method), DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method), DEVMETHOD(pci_find_htcap, pci_find_htcap_method), DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method), DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method), DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method), DEVMETHOD(pci_enable_msi, pci_enable_msi_method), DEVMETHOD(pci_enable_msix, pci_enable_msix_method), DEVMETHOD(pci_disable_msi, pci_disable_msi_method), DEVMETHOD(pci_remap_msix, pci_remap_msix_method), DEVMETHOD(pci_release_msi, pci_release_msi_method), DEVMETHOD(pci_msi_count, pci_msi_count_method), DEVMETHOD(pci_msix_count, pci_msix_count_method), DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method), DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method), DEVMETHOD(pci_get_id, pci_get_id_method), DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method), DEVMETHOD(pci_child_added, pci_child_added_method), #ifdef PCI_IOV DEVMETHOD(pci_iov_attach, pci_iov_attach_method), DEVMETHOD(pci_iov_detach, pci_iov_detach_method), DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method), #endif DEVMETHOD_END }; DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc)); EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS); MODULE_VERSION(pci, 1); static char *pci_vendordata; static size_t pci_vendordata_size; struct pci_quirk { uint32_t devid; /* Vendor/device of the card */ int type; #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */ #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */ #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */ #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */ #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */ #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */ #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */ int arg1; int arg2; }; static const struct pci_quirk pci_quirks[] = { /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */ { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 }, { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 }, /* As does the Serverworks OSB4 (the SMBus mapping register) */ { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 }, /* * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge * or the CMIC-SL (AKA ServerWorks GC_LE). */ { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * MSI doesn't work on earlier Intel chipsets including * E7500, E7501, E7505, 845, 865, 875/E7210, and 855. */ { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * MSI doesn't work with devices behind the AMD 8131 HT-PCIX * bridge. */ { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * Some virtualization environments emulate an older chipset * but support MSI just fine. QEMU uses the Intel 82440. */ { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 }, /* * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus * controller depending on SoftPciRst register (PM_IO 0x55 [7]). * It prevents us from attaching hpet(4) when the bit is unset. * Note this quirk only affects SB600 revision A13 and earlier. * For SB600 A21 and later, firmware must set the bit to hide it. * For SB700 and later, it is unused and hardcoded to zero. */ { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 }, /* * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit * of the command register is set. */ { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't * issue MSI interrupts with PCIM_CMD_INTxDIS set either. */ { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */ { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */ { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */ { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */ { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */ { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */ /* * HPE Gen 10 VGA has a memory range that can't be allocated in the * expected place. */ { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 }, { 0 } }; /* map register information */ #define PCI_MAPMEM 0x01 /* memory map */ #define PCI_MAPMEMP 0x02 /* prefetchable memory map */ #define PCI_MAPPORT 0x04 /* port map */ struct devlist pci_devq; uint32_t pci_generation; uint32_t pci_numdevs = 0; static int pcie_chipset, pcix_chipset; /* sysctl vars */ SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "PCI bus tuning parameters"); static int pci_enable_io_modes = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN, &pci_enable_io_modes, 1, "Enable I/O and memory bits in the config register. Some BIOSes do not" " enable these bits correctly. We'd like to do this all the time, but" " there are some peripherals that this causes problems with."); static int pci_do_realloc_bars = 1; SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN, &pci_do_realloc_bars, 0, "Attempt to allocate a new range for any BARs whose original " "firmware-assigned ranges fail to allocate during the initial device scan."); static int pci_do_power_nodriver = 0; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN, &pci_do_power_nodriver, 0, "Place a function into D3 state when no driver attaches to it. 0 means" " disable. 1 means conservatively place devices into D3 state. 2 means" " aggressively place devices into D3 state. 3 means put absolutely" " everything in D3 state."); int pci_do_power_resume = 1; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN, &pci_do_power_resume, 1, "Transition from D3 -> D0 on resume."); int pci_do_power_suspend = 1; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN, &pci_do_power_suspend, 1, "Transition from D0 -> D3 on suspend."); static int pci_do_msi = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1, "Enable support for MSI interrupts"); static int pci_do_msix = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1, "Enable support for MSI-X interrupts"); static int pci_msix_rewrite_table = 0; SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN, &pci_msix_rewrite_table, 0, "Rewrite entire MSI-X table when updating MSI-X entries"); static int pci_honor_msi_blacklist = 1; SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN, &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X"); #if defined(__i386__) || defined(__amd64__) static int pci_usb_takeover = 1; #else static int pci_usb_takeover = 0; #endif SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN, &pci_usb_takeover, 1, "Enable early takeover of USB controllers. Disable this if you depend on" " BIOS emulation of USB devices, that is you use USB devices (like" " keyboard or mouse) but do not load USB drivers"); static int pci_clear_bars; SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0, "Ignore firmware-assigned resources for BARs."); static int pci_clear_buses; SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0, "Ignore firmware-assigned bus numbers."); static int pci_enable_ari = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari, 0, "Enable support for PCIe Alternative RID Interpretation"); int pci_enable_aspm = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm, 0, "Enable support for PCIe Active State Power Management"); static int pci_clear_aer_on_attach = 0; SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN, &pci_clear_aer_on_attach, 0, "Clear port and device AER state on driver attach"); static bool pci_enable_mps_tune = true; SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN, &pci_enable_mps_tune, 1, "Enable tuning of MPS(maximum payload size)." ); static int pci_has_quirk(uint32_t devid, int quirk) { const struct pci_quirk *q; for (q = &pci_quirks[0]; q->devid; q++) { if (q->devid == devid && q->type == quirk) return (1); } return (0); } /* Find a device_t by bus/slot/function in domain 0 */ device_t pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func) { return (pci_find_dbsf(0, bus, slot, func)); } /* Find a device_t by domain/bus/slot/function */ device_t pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func) { struct pci_devinfo *dinfo = NULL; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if ((dinfo->cfg.domain == domain) && (dinfo->cfg.bus == bus) && (dinfo->cfg.slot == slot) && (dinfo->cfg.func == func)) { break; } } return (dinfo != NULL ? dinfo->cfg.dev : NULL); } /* Find a device_t by vendor/device ID */ device_t pci_find_device(uint16_t vendor, uint16_t device) { struct pci_devinfo *dinfo; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if ((dinfo->cfg.vendor == vendor) && (dinfo->cfg.device == device)) { return (dinfo->cfg.dev); } } return (NULL); } device_t pci_find_class(uint8_t class, uint8_t subclass) { struct pci_devinfo *dinfo; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if (dinfo->cfg.baseclass == class && dinfo->cfg.subclass == subclass) { return (dinfo->cfg.dev); } } return (NULL); } device_t pci_find_class_from(uint8_t class, uint8_t subclass, device_t from) { struct pci_devinfo *dinfo; bool found = false; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if (from != NULL && found == false) { if (from != dinfo->cfg.dev) continue; found = true; continue; } if (dinfo->cfg.baseclass == class && dinfo->cfg.subclass == subclass) { return (dinfo->cfg.dev); } } return (NULL); } static int pci_printf(pcicfgregs *cfg, const char *fmt, ...) { va_list ap; int retval; retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot, cfg->func); va_start(ap, fmt); retval += vprintf(fmt, ap); va_end(ap); return (retval); } /* return base address of memory or port map */ static pci_addr_t pci_mapbase(uint64_t mapreg) { if (PCI_BAR_MEM(mapreg)) return (mapreg & PCIM_BAR_MEM_BASE); else return (mapreg & PCIM_BAR_IO_BASE); } /* return map type of memory or port map */ static const char * pci_maptype(uint64_t mapreg) { if (PCI_BAR_IO(mapreg)) return ("I/O Port"); if (mapreg & PCIM_BAR_MEM_PREFETCH) return ("Prefetchable Memory"); return ("Memory"); } /* return log2 of map size decoded for memory or port map */ int pci_mapsize(uint64_t testval) { int ln2size; testval = pci_mapbase(testval); ln2size = 0; if (testval != 0) { while ((testval & 1) == 0) { ln2size++; testval >>= 1; } } return (ln2size); } /* return base address of device ROM */ static pci_addr_t pci_rombase(uint64_t mapreg) { return (mapreg & PCIM_BIOS_ADDR_MASK); } /* return log2 of map size decided for device ROM */ static int pci_romsize(uint64_t testval) { int ln2size; testval = pci_rombase(testval); ln2size = 0; if (testval != 0) { while ((testval & 1) == 0) { ln2size++; testval >>= 1; } } return (ln2size); } /* return log2 of address range supported by map register */ static int pci_maprange(uint64_t mapreg) { int ln2range = 0; if (PCI_BAR_IO(mapreg)) ln2range = 32; else switch (mapreg & PCIM_BAR_MEM_TYPE) { case PCIM_BAR_MEM_32: ln2range = 32; break; case PCIM_BAR_MEM_1MB: ln2range = 20; break; case PCIM_BAR_MEM_64: ln2range = 64; break; } return (ln2range); } /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */ static void pci_fixancient(pcicfgregs *cfg) { if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) return; /* PCI to PCI bridges use header type 1 */ if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI) cfg->hdrtype = PCIM_HDRTYPE_BRIDGE; } /* extract header type specific config data */ static void pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: cfg->subvendor = REG(PCIR_SUBVEND_0, 2); cfg->subdevice = REG(PCIR_SUBDEV_0, 2); cfg->mingnt = REG(PCIR_MINGNT, 1); cfg->maxlat = REG(PCIR_MAXLAT, 1); cfg->nummaps = PCI_MAXMAPS_0; break; case PCIM_HDRTYPE_BRIDGE: cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1); cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1); cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1); cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1); cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2); cfg->nummaps = PCI_MAXMAPS_1; break; case PCIM_HDRTYPE_CARDBUS: cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1); cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1); cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1); cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1); cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2); cfg->subvendor = REG(PCIR_SUBVEND_2, 2); cfg->subdevice = REG(PCIR_SUBDEV_2, 2); cfg->nummaps = PCI_MAXMAPS_2; break; } #undef REG } /* read configuration header into pcicfgregs structure */ struct pci_devinfo * pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f) { #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) uint16_t vid, did; vid = REG(PCIR_VENDOR, 2); if (vid == PCIV_INVALID) return (NULL); did = REG(PCIR_DEVICE, 2); return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did)); } struct pci_devinfo * pci_alloc_devinfo_method(device_t dev) { return (malloc(sizeof(struct pci_devinfo), M_DEVBUF, M_WAITOK | M_ZERO)); } static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f, uint16_t vid, uint16_t did) { struct pci_devinfo *devlist_entry; pcicfgregs *cfg; devlist_entry = PCI_ALLOC_DEVINFO(bus); cfg = &devlist_entry->cfg; cfg->domain = d; cfg->bus = b; cfg->slot = s; cfg->func = f; cfg->vendor = vid; cfg->device = did; cfg->cmdreg = REG(PCIR_COMMAND, 2); cfg->statreg = REG(PCIR_STATUS, 2); cfg->baseclass = REG(PCIR_CLASS, 1); cfg->subclass = REG(PCIR_SUBCLASS, 1); cfg->progif = REG(PCIR_PROGIF, 1); cfg->revid = REG(PCIR_REVID, 1); cfg->hdrtype = REG(PCIR_HDRTYPE, 1); cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1); cfg->lattimer = REG(PCIR_LATTIMER, 1); cfg->intpin = REG(PCIR_INTPIN, 1); cfg->intline = REG(PCIR_INTLINE, 1); cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0; cfg->hdrtype &= ~PCIM_MFDEV; STAILQ_INIT(&cfg->maps); cfg->iov = NULL; pci_fixancient(cfg); pci_hdrtypedata(pcib, b, s, f, cfg); if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT) pci_read_cap(pcib, cfg); STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links); devlist_entry->conf.pc_sel.pc_domain = cfg->domain; devlist_entry->conf.pc_sel.pc_bus = cfg->bus; devlist_entry->conf.pc_sel.pc_dev = cfg->slot; devlist_entry->conf.pc_sel.pc_func = cfg->func; devlist_entry->conf.pc_hdr = cfg->hdrtype; devlist_entry->conf.pc_subvendor = cfg->subvendor; devlist_entry->conf.pc_subdevice = cfg->subdevice; devlist_entry->conf.pc_vendor = cfg->vendor; devlist_entry->conf.pc_device = cfg->device; devlist_entry->conf.pc_class = cfg->baseclass; devlist_entry->conf.pc_subclass = cfg->subclass; devlist_entry->conf.pc_progif = cfg->progif; devlist_entry->conf.pc_revid = cfg->revid; pci_numdevs++; pci_generation++; return (devlist_entry); } #undef REG static void pci_ea_fill_info(device_t pcib, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \ cfg->ea.ea_location + (n), w) int num_ent; int ptr; int a, b; uint32_t val; int ent_size; uint32_t dw[4]; uint64_t base, max_offset; struct pci_ea_entry *eae; if (cfg->ea.ea_location == 0) return; STAILQ_INIT(&cfg->ea.ea_entries); /* Determine the number of entries */ num_ent = REG(PCIR_EA_NUM_ENT, 2); num_ent &= PCIM_EA_NUM_ENT_MASK; /* Find the first entry to care of */ ptr = PCIR_EA_FIRST_ENT; /* Skip DWORD 2 for type 1 functions */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) ptr += 4; for (a = 0; a < num_ent; a++) { eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO); eae->eae_cfg_offset = cfg->ea.ea_location + ptr; /* Read a number of dwords in the entry */ val = REG(ptr, 4); ptr += 4; ent_size = (val & PCIM_EA_ES); for (b = 0; b < ent_size; b++) { dw[b] = REG(ptr, 4); ptr += 4; } eae->eae_flags = val; eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET; base = dw[0] & PCIM_EA_FIELD_MASK; max_offset = dw[1] | ~PCIM_EA_FIELD_MASK; b = 2; if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) { base |= (uint64_t)dw[b] << 32UL; b++; } if (((dw[1] & PCIM_EA_IS_64) != 0) && (b < ent_size)) { max_offset |= (uint64_t)dw[b] << 32UL; b++; } eae->eae_base = base; eae->eae_max_offset = max_offset; STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link); if (bootverbose) { printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n", cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags, (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset); } } } #undef REG static void pci_read_cap(device_t pcib, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w) #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w) #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) uint64_t addr; #endif uint32_t val; int ptr, nextptr, ptrptr; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: case PCIM_HDRTYPE_BRIDGE: ptrptr = PCIR_CAP_PTR; break; case PCIM_HDRTYPE_CARDBUS: ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */ break; default: return; /* no extended capabilities support */ } nextptr = REG(ptrptr, 1); /* sanity check? */ /* * Read capability entries. */ while (nextptr != 0) { /* Sanity check */ if (nextptr > 255) { printf("illegal PCI extended capability offset %d\n", nextptr); return; } /* Find the next entry */ ptr = nextptr; nextptr = REG(ptr + PCICAP_NEXTPTR, 1); /* Process this entry */ switch (REG(ptr + PCICAP_ID, 1)) { case PCIY_PMG: /* PCI power management */ if (cfg->pp.pp_cap == 0) { cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2); cfg->pp.pp_status = ptr + PCIR_POWER_STATUS; cfg->pp.pp_bse = ptr + PCIR_POWER_BSE; if ((nextptr - ptr) > PCIR_POWER_DATA) cfg->pp.pp_data = ptr + PCIR_POWER_DATA; } break; case PCIY_HT: /* HyperTransport */ /* Determine HT-specific capability type. */ val = REG(ptr + PCIR_HT_COMMAND, 2); if ((val & 0xe000) == PCIM_HTCAP_SLAVE) cfg->ht.ht_slave = ptr; #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) switch (val & PCIM_HTCMD_CAP_MASK) { case PCIM_HTCAP_MSI_MAPPING: if (!(val & PCIM_HTCMD_MSI_FIXED)) { /* Sanity check the mapping window. */ addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI, 4); addr <<= 32; addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO, 4); if (addr != MSI_INTEL_ADDR_BASE) device_printf(pcib, "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n", cfg->domain, cfg->bus, cfg->slot, cfg->func, (long long)addr); } else addr = MSI_INTEL_ADDR_BASE; cfg->ht.ht_msimap = ptr; cfg->ht.ht_msictrl = val; cfg->ht.ht_msiaddr = addr; break; } #endif break; case PCIY_MSI: /* PCI MSI */ cfg->msi.msi_location = ptr; cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2); cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl & PCIM_MSICTRL_MMC_MASK)>>1); break; case PCIY_MSIX: /* PCI MSI-X */ cfg->msix.msix_location = ptr; cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2); cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl & PCIM_MSIXCTRL_TABLE_SIZE) + 1; val = REG(ptr + PCIR_MSIX_TABLE, 4); cfg->msix.msix_table_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK); cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK; val = REG(ptr + PCIR_MSIX_PBA, 4); cfg->msix.msix_pba_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK); cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK; break; case PCIY_VPD: /* PCI Vital Product Data */ cfg->vpd.vpd_reg = ptr; break; case PCIY_SUBVENDOR: /* Should always be true. */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) { val = REG(ptr + PCIR_SUBVENDCAP_ID, 4); cfg->subvendor = val & 0xffff; cfg->subdevice = val >> 16; } break; case PCIY_PCIX: /* PCI-X */ /* * Assume we have a PCI-X chipset if we have * at least one PCI-PCI bridge with a PCI-X * capability. Note that some systems with * PCI-express or HT chipsets might match on * this check as well. */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) pcix_chipset = 1; cfg->pcix.pcix_location = ptr; break; case PCIY_EXPRESS: /* PCI-express */ /* * Assume we have a PCI-express chipset if we have * at least one PCI-express device. */ pcie_chipset = 1; cfg->pcie.pcie_location = ptr; val = REG(ptr + PCIER_FLAGS, 2); cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE; break; case PCIY_EA: /* Enhanced Allocation */ cfg->ea.ea_location = ptr; pci_ea_fill_info(pcib, cfg); break; default: break; } } #if defined(__powerpc__) /* * Enable the MSI mapping window for all HyperTransport * slaves. PCI-PCI bridges have their windows enabled via * PCIB_MAP_MSI(). */ if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 && !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) { device_printf(pcib, "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n", cfg->domain, cfg->bus, cfg->slot, cfg->func); cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl, 2); } #endif /* REG and WREG use carry through to next functions */ } /* * PCI Vital Product Data */ #define PCI_VPD_TIMEOUT 1000000 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data) { int count = PCI_VPD_TIMEOUT; KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2); while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) { if (--count < 0) return (ENXIO); DELAY(1); /* limit looping */ } *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4)); return (0); } #if 0 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data) { int count = PCI_VPD_TIMEOUT; KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4); WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2); while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) { if (--count < 0) return (ENXIO); DELAY(1); /* limit looping */ } return (0); } #endif #undef PCI_VPD_TIMEOUT struct vpd_readstate { device_t pcib; pcicfgregs *cfg; uint32_t val; int bytesinval; int off; uint8_t cksum; }; /* return 0 and one byte in *data if no read error, -1 else */ static int vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data) { uint32_t reg; uint8_t byte; if (vrs->bytesinval == 0) { if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®)) return (-1); vrs->val = le32toh(reg); vrs->off += 4; byte = vrs->val & 0xff; vrs->bytesinval = 3; } else { vrs->val = vrs->val >> 8; byte = vrs->val & 0xff; vrs->bytesinval--; } vrs->cksum += byte; *data = byte; return (0); } /* return 0 on match, -1 and "unget" byte on no match */ static int vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected) { uint8_t data; if (vpd_nextbyte(vrs, &data) != 0) return (-1); if (data == expected) return (0); vrs->cksum -= data; vrs->val = (vrs->val << 8) + data; vrs->bytesinval++; return (-1); } /* return size if tag matches, -1 on no match, -2 on read error */ static int vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag) { uint8_t byte1, byte2; if (vpd_expectbyte(vrs, vpd_tag) != 0) return (-1); if ((vpd_tag & 0x80) == 0) return (vpd_tag & 0x07); if (vpd_nextbyte(vrs, &byte1) != 0) return (-2); if (vpd_nextbyte(vrs, &byte2) != 0) return (-2); return ((byte2 << 8) + byte1); } /* (re)allocate buffer in multiples of 8 elements */ static void* alloc_buffer(void* buffer, size_t element_size, int needed) { int alloc, new_alloc; alloc = roundup2(needed, 8); new_alloc = roundup2(needed + 1, 8); if (alloc != new_alloc) { buffer = reallocf(buffer, new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO); } return (buffer); } /* read VPD keyword and return element size, return -1 on read error */ static int vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2]) { uint8_t data; if (vpd_nextbyte(vrs, &keyword[0]) != 0) return (-1); if (vpd_nextbyte(vrs, &keyword[1]) != 0) return (-1); if (vpd_nextbyte(vrs, &data) != 0) return (-1); return (data); } /* read VPD data element of given size into allocated buffer */ static char * vpd_read_value(struct vpd_readstate *vrs, int size) { int i; char char1; char *value; value = malloc(size + 1, M_DEVBUF, M_WAITOK); for (i = 0; i < size; i++) { if (vpd_nextbyte(vrs, &char1) != 0) { free(value, M_DEVBUF); return (NULL); } value[i] = char1; } value[size] = '\0'; return (value); } /* read VPD into *keyword and *value, return length of data element */ static int vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen) { int len; len = vpd_read_elem_head(vrs, keyword); if (len < 0 || len > maxlen) return (-1); *value = vpd_read_value(vrs, len); return (len); } /* subtract all data following first byte from checksum of RV element */ static void vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len) { int i; uint8_t fixup; fixup = 0; for (i = 1; i < len; i++) fixup += rvstring[i]; vrs->cksum -= fixup; } /* fetch one read-only element and return size of heading + data */ static int next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize) { struct pcicfg_vpd *vpd; pcicfgregs *cfg; struct vpd_readonly *vpd_ros; int len; cfg = vrs->cfg; vpd = &cfg->vpd; if (maxsize < 3) return (-1); vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt); vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt]; maxsize -= 3; len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize); if (vpd_ros->value == NULL) return (-1); vpd_ros->len = len; if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') { vpd_fixup_cksum(vrs, vpd_ros->value, len); if (vrs->cksum != 0) { pci_printf(cfg, "invalid VPD checksum %#hhx\n", vrs->cksum); return (-1); } } vpd->vpd_rocnt++; return (len + 3); } /* fetch one writable element and return size of heading + data */ static int next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize) { struct pcicfg_vpd *vpd; pcicfgregs *cfg; struct vpd_write *vpd_w; int len; cfg = vrs->cfg; vpd = &cfg->vpd; if (maxsize < 3) return (-1); vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt); if (vpd->vpd_w == NULL) { pci_printf(cfg, "out of memory"); return (-1); } vpd_w = &vpd->vpd_w[vpd->vpd_wcnt]; maxsize -= 3; vpd_w->start = vrs->off + 3 - vrs->bytesinval; len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize); if (vpd_w->value == NULL) return (-1); vpd_w->len = len; vpd->vpd_wcnt++; return (len + 3); } /* free all memory allocated for VPD data */ static void vpd_free(struct pcicfg_vpd *vpd) { int i; free(vpd->vpd_ident, M_DEVBUF); for (i = 0; i < vpd->vpd_rocnt; i++) free(vpd->vpd_ros[i].value, M_DEVBUF); free(vpd->vpd_ros, M_DEVBUF); vpd->vpd_rocnt = 0; for (i = 0; i < vpd->vpd_wcnt; i++) free(vpd->vpd_w[i].value, M_DEVBUF); free(vpd->vpd_w, M_DEVBUF); vpd->vpd_wcnt = 0; } #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */ #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */ #define VPD_TAG_RO (0x10 | 0x80) /* large tag */ #define VPD_TAG_RW (0x11 | 0x80) /* large tag */ static int pci_parse_vpd(device_t pcib, pcicfgregs *cfg) { struct vpd_readstate vrs; int cksumvalid; int size, elem_size; /* init vpd reader */ vrs.bytesinval = 0; vrs.off = 0; vrs.pcib = pcib; vrs.cfg = cfg; vrs.cksum = 0; /* read VPD ident element - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT); if (size <= 0) { pci_printf(cfg, "no VPD ident found\n"); return (0); } cfg->vpd.vpd_ident = vpd_read_value(&vrs, size); if (cfg->vpd.vpd_ident == NULL) { pci_printf(cfg, "error accessing VPD ident data\n"); return (0); } /* read VPD RO elements - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_RO); if (size <= 0) { pci_printf(cfg, "no read-only VPD data found\n"); return (0); } while (size > 0) { elem_size = next_vpd_ro_elem(&vrs, size); if (elem_size < 0) { pci_printf(cfg, "error accessing read-only VPD data\n"); return (-1); } size -= elem_size; } cksumvalid = (vrs.cksum == 0); if (!cksumvalid) return (-1); /* read VPD RW elements - optional */ size = vpd_read_tag_size(&vrs, VPD_TAG_RW); if (size == -2) return (-1); while (size > 0) { elem_size = next_vpd_rw_elem(&vrs, size); if (elem_size < 0) { pci_printf(cfg, "error accessing writeable VPD data\n"); return (-1); } size -= elem_size; } /* read empty END tag - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_END); if (size != 0) { pci_printf(cfg, "No valid VPD end tag found\n"); } return (0); } static void pci_read_vpd(device_t pcib, pcicfgregs *cfg) { int status; status = pci_parse_vpd(pcib, cfg); if (status < 0) vpd_free(&cfg->vpd); cfg->vpd.vpd_cached = 1; #undef REG #undef WREG } int pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(dev), cfg); *identptr = cfg->vpd.vpd_ident; if (*identptr == NULL) return (ENXIO); return (0); } int pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw, const char **vptr) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; int i; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(dev), cfg); for (i = 0; i < cfg->vpd.vpd_rocnt; i++) if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword, sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) { *vptr = cfg->vpd.vpd_ros[i].value; return (0); } *vptr = NULL; return (ENXIO); } struct pcicfg_vpd * pci_fetch_vpd_list(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg); return (&cfg->vpd); } /* * Find the requested HyperTransport capability and return the offset * in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg) { int ptr, error; uint16_t val; error = pci_find_cap(child, PCIY_HT, &ptr); if (error) return (error); /* * Traverse the capabilities list checking each HT capability * to see if it matches the requested HT capability. */ for (;;) { val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); if (capability == PCIM_HTCAP_SLAVE || capability == PCIM_HTCAP_HOST) val &= 0xe000; else val &= PCIM_HTCMD_CAP_MASK; if (val == capability) { if (capreg != NULL) *capreg = ptr; return (0); } /* Skip to the next HT capability. */ if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) break; } return (ENOENT); } /* * Find the next requested HyperTransport capability after start and return * the offset in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_next_htcap_method(device_t dev, device_t child, int capability, int start, int *capreg) { int ptr; uint16_t val; KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT, ("start capability is not HyperTransport capability")); ptr = start; /* * Traverse the capabilities list checking each HT capability * to see if it matches the requested HT capability. */ for (;;) { /* Skip to the next HT capability. */ if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) break; val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); if (capability == PCIM_HTCAP_SLAVE || capability == PCIM_HTCAP_HOST) val &= 0xe000; else val &= PCIM_HTCMD_CAP_MASK; if (val == capability) { if (capreg != NULL) *capreg = ptr; return (0); } } return (ENOENT); } /* * Find the requested capability and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_cap_method(device_t dev, device_t child, int capability, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t status; uint8_t ptr; /* * Check the CAP_LIST bit of the PCI status register first. */ status = pci_read_config(child, PCIR_STATUS, 2); if (!(status & PCIM_STATUS_CAPPRESENT)) return (ENXIO); /* * Determine the start pointer of the capabilities list. */ switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: case PCIM_HDRTYPE_BRIDGE: ptr = PCIR_CAP_PTR; break; case PCIM_HDRTYPE_CARDBUS: ptr = PCIR_CAP_PTR_2; break; default: /* XXX: panic? */ return (ENXIO); /* no extended capabilities support */ } ptr = pci_read_config(child, ptr, 1); /* * Traverse the capabilities list. */ while (ptr != 0) { if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); } return (ENOENT); } /* * Find the next requested capability after start and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_next_cap_method(device_t dev, device_t child, int capability, int start, int *capreg) { uint8_t ptr; KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability, ("start capability is not expected capability")); ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1); while (ptr != 0) { if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); } return (ENOENT); } /* * Find the requested extended capability and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_extcap_method(device_t dev, device_t child, int capability, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t ecap; uint16_t ptr; /* Only supported for PCI-express devices. */ if (cfg->pcie.pcie_location == 0) return (ENXIO); ptr = PCIR_EXTCAP; ecap = pci_read_config(child, ptr, 4); if (ecap == 0xffffffff || ecap == 0) return (ENOENT); for (;;) { if (PCI_EXTCAP_ID(ecap) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = PCI_EXTCAP_NEXTPTR(ecap); if (ptr == 0) break; ecap = pci_read_config(child, ptr, 4); } return (ENOENT); } /* * Find the next requested extended capability after start and return the * offset in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_next_extcap_method(device_t dev, device_t child, int capability, int start, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t ecap; uint16_t ptr; /* Only supported for PCI-express devices. */ if (cfg->pcie.pcie_location == 0) return (ENXIO); ecap = pci_read_config(child, start, 4); KASSERT(PCI_EXTCAP_ID(ecap) == capability, ("start extended capability is not expected capability")); ptr = PCI_EXTCAP_NEXTPTR(ecap); while (ptr != 0) { ecap = pci_read_config(child, ptr, 4); if (PCI_EXTCAP_ID(ecap) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = PCI_EXTCAP_NEXTPTR(ecap); } return (ENOENT); } /* * Support for MSI-X message interrupts. */ static void pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset; KASSERT(msix->msix_table_len > index, ("bogus index")); offset = msix->msix_table_offset + index * 16; bus_write_4(msix->msix_table_res, offset, address & 0xffffffff); bus_write_4(msix->msix_table_res, offset + 4, address >> 32); bus_write_4(msix->msix_table_res, offset + 8, data); } void pci_enable_msix_method(device_t dev, device_t child, u_int index, uint64_t address, uint32_t data) { if (pci_msix_rewrite_table) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; /* * Some VM hosts require MSIX to be disabled in the * control register before updating the MSIX table * entries are allowed. It is not enough to only * disable MSIX while updating a single entry. MSIX * must be disabled while updating all entries in the * table. */ pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2); pci_resume_msix(child); } else pci_write_msix_entry(child, index, address, data); /* Enable MSI -> HT mapping. */ pci_ht_map_msi(child, address); } void pci_mask_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, val; KASSERT(msix->msix_msgnum > index, ("bogus index")); offset = msix->msix_table_offset + index * 16 + 12; val = bus_read_4(msix->msix_table_res, offset); val |= PCIM_MSIX_VCTRL_MASK; /* * Some devices (e.g. Samsung PM961) do not support reads of this * register, so always write the new value. */ bus_write_4(msix->msix_table_res, offset, val); } void pci_unmask_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, val; KASSERT(msix->msix_table_len > index, ("bogus index")); offset = msix->msix_table_offset + index * 16 + 12; val = bus_read_4(msix->msix_table_res, offset); val &= ~PCIM_MSIX_VCTRL_MASK; /* * Some devices (e.g. Samsung PM961) do not support reads of this * register, so always write the new value. */ bus_write_4(msix->msix_table_res, offset, val); } int pci_pending_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, bit; KASSERT(msix->msix_table_len > index, ("bogus index")); offset = msix->msix_pba_offset + (index / 32) * 4; bit = 1 << index % 32; return (bus_read_4(msix->msix_pba_res, offset) & bit); } /* * Restore MSI-X registers and table during resume. If MSI-X is * enabled then walk the virtual table to restore the actual MSI-X * table. */ static void pci_resume_msix(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct msix_table_entry *mte; struct msix_vector *mv; int i; if (msix->msix_alloc > 0) { /* First, mask all vectors. */ for (i = 0; i < msix->msix_msgnum; i++) pci_mask_msix(dev, i); /* Second, program any messages with at least one handler. */ for (i = 0; i < msix->msix_table_len; i++) { mte = &msix->msix_table[i]; if (mte->mte_vector == 0 || mte->mte_handlers == 0) continue; mv = &msix->msix_vectors[mte->mte_vector - 1]; pci_write_msix_entry(dev, i, mv->mv_address, mv->mv_data); pci_unmask_msix(dev, i); } } pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl, 2); } /* * Attempt to allocate *count MSI-X messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at rid 1. */ int pci_alloc_msix_method(device_t dev, device_t child, int *count) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; int actual, error, i, irq, max; /* Don't let count == 0 get us into trouble. */ if (*count == 0) return (EINVAL); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) return (ENXIO); /* Already have allocated messages? */ if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) return (ENXIO); /* If MSI-X is blacklisted for this system, fail. */ if (pci_msix_blacklisted()) return (ENXIO); /* MSI-X capability present? */ if (cfg->msix.msix_location == 0 || !pci_do_msix) return (ENODEV); /* Make sure the appropriate BARs are mapped. */ rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_table_bar); if (rle == NULL || rle->res == NULL || !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); cfg->msix.msix_table_res = rle->res; if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) { rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_pba_bar); if (rle == NULL || rle->res == NULL || !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); } cfg->msix.msix_pba_res = rle->res; if (bootverbose) device_printf(child, "attempting to allocate %d MSI-X vectors (%d supported)\n", *count, cfg->msix.msix_msgnum); max = min(*count, cfg->msix.msix_msgnum); for (i = 0; i < max; i++) { /* Allocate a message. */ error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq); if (error) { if (i == 0) return (error); break; } resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, irq, 1); } actual = i; if (bootverbose) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1); if (actual == 1) device_printf(child, "using IRQ %ju for MSI-X\n", rle->start); else { int run; /* * Be fancy and try to print contiguous runs of * IRQ values as ranges. 'irq' is the previous IRQ. * 'run' is true if we are in a range. */ device_printf(child, "using IRQs %ju", rle->start); irq = rle->start; run = 0; for (i = 1; i < actual; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); /* Still in a run? */ if (rle->start == irq + 1) { run = 1; irq++; continue; } /* Finish previous range. */ if (run) { printf("-%d", irq); run = 0; } /* Start new range. */ printf(",%ju", rle->start); irq = rle->start; } /* Unfinished range? */ if (run) printf("-%d", irq); printf(" for MSI-X\n"); } } /* Mask all vectors. */ for (i = 0; i < cfg->msix.msix_msgnum; i++) pci_mask_msix(child, i); /* Allocate and initialize vector data and virtual table. */ cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual, M_DEVBUF, M_WAITOK | M_ZERO); cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < actual; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); cfg->msix.msix_vectors[i].mv_irq = rle->start; cfg->msix.msix_table[i].mte_vector = i + 1; } /* Update control register to enable MSI-X. */ cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL, cfg->msix.msix_ctrl, 2); /* Update counts of alloc'd messages. */ cfg->msix.msix_alloc = actual; cfg->msix.msix_table_len = actual; *count = actual; return (0); } /* * By default, pci_alloc_msix() will assign the allocated IRQ * resources consecutively to the first N messages in the MSI-X table. * However, device drivers may want to use different layouts if they * either receive fewer messages than they asked for, or they wish to * populate the MSI-X table sparsely. This method allows the driver * to specify what layout it wants. It must be called after a * successful pci_alloc_msix() but before any of the associated * SYS_RES_IRQ resources are allocated via bus_alloc_resource(). * * The 'vectors' array contains 'count' message vectors. The array * maps directly to the MSI-X table in that index 0 in the array * specifies the vector for the first message in the MSI-X table, etc. * The vector value in each array index can either be 0 to indicate * that no vector should be assigned to a message slot, or it can be a * number from 1 to N (where N is the count returned from a * succcessful call to pci_alloc_msix()) to indicate which message * vector (IRQ) to be used for the corresponding message. * * On successful return, each message with a non-zero vector will have * an associated SYS_RES_IRQ whose rid is equal to the array index + * 1. Additionally, if any of the IRQs allocated via the previous * call to pci_alloc_msix() are not used in the mapping, those IRQs * will be freed back to the system automatically. * * For example, suppose a driver has a MSI-X table with 6 messages and * asks for 6 messages, but pci_alloc_msix() only returns a count of * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and * C. After the call to pci_alloc_msix(), the device will be setup to * have an MSI-X table of ABC--- (where - means no vector assigned). * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 }, * then the MSI-X table will look like A-AB-B, and the 'C' vector will * be freed back to the system. This device will also have valid * SYS_RES_IRQ rids of 1, 3, 4, and 6. * * In any case, the SYS_RES_IRQ rid X will always map to the message * at MSI-X table index X - 1 and will only be valid if a vector is * assigned to that table entry. */ int pci_remap_msix_method(device_t dev, device_t child, int count, const u_int *vectors) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct resource_list_entry *rle; int i, irq, j, *used; /* * Have to have at least one message in the table but the * table can't be bigger than the actual MSI-X table in the * device. */ if (count == 0 || count > msix->msix_msgnum) return (EINVAL); /* Sanity check the vectors. */ for (i = 0; i < count; i++) if (vectors[i] > msix->msix_alloc) return (EINVAL); /* * Make sure there aren't any holes in the vectors to be used. * It's a big pain to support it, and it doesn't really make * sense anyway. Also, at least one vector must be used. */ used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) if (vectors[i] != 0) used[vectors[i] - 1] = 1; for (i = 0; i < msix->msix_alloc - 1; i++) if (used[i] == 0 && used[i + 1] == 1) { free(used, M_DEVBUF); return (EINVAL); } if (used[0] != 1) { free(used, M_DEVBUF); return (EINVAL); } /* Make sure none of the resources are allocated. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; if (msix->msix_table[i].mte_handlers > 0) { free(used, M_DEVBUF); return (EBUSY); } rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing resource")); if (rle->res != NULL) { free(used, M_DEVBUF); return (EBUSY); } } /* Free the existing resource list entries. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); } /* * Build the new virtual table keeping track of which vectors are * used. */ free(msix->msix_table, M_DEVBUF); msix->msix_table = malloc(sizeof(struct msix_table_entry) * count, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) msix->msix_table[i].mte_vector = vectors[i]; msix->msix_table_len = count; /* Free any unused IRQs and resize the vectors array if necessary. */ j = msix->msix_alloc - 1; if (used[j] == 0) { struct msix_vector *vec; while (used[j] == 0) { PCIB_RELEASE_MSIX(device_get_parent(dev), child, msix->msix_vectors[j].mv_irq); j--; } vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF, M_WAITOK); bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) * (j + 1)); free(msix->msix_vectors, M_DEVBUF); msix->msix_vectors = vec; msix->msix_alloc = j + 1; } free(used, M_DEVBUF); /* Map the IRQs onto the rids. */ for (i = 0; i < count; i++) { if (vectors[i] == 0) continue; irq = msix->msix_vectors[vectors[i] - 1].mv_irq; resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, irq, 1); } if (bootverbose) { device_printf(child, "Remapped MSI-X IRQs as: "); for (i = 0; i < count; i++) { if (i != 0) printf(", "); if (vectors[i] == 0) printf("---"); else printf("%d", msix->msix_vectors[vectors[i] - 1].mv_irq); } printf("\n"); } return (0); } static int pci_release_msix(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct resource_list_entry *rle; int i; /* Do we have any messages to release? */ if (msix->msix_alloc == 0) return (ENODEV); /* Make sure none of the resources are allocated. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; if (msix->msix_table[i].mte_handlers > 0) return (EBUSY); rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing resource")); if (rle->res != NULL) return (EBUSY); } /* Update control register to disable MSI-X. */ msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE; pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl, 2); /* Free the resource list entries. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); } free(msix->msix_table, M_DEVBUF); msix->msix_table_len = 0; /* Release the IRQs. */ for (i = 0; i < msix->msix_alloc; i++) PCIB_RELEASE_MSIX(device_get_parent(dev), child, msix->msix_vectors[i].mv_irq); free(msix->msix_vectors, M_DEVBUF); msix->msix_alloc = 0; return (0); } /* * Return the max supported MSI-X messages this device supports. * Basically, assuming the MD code can alloc messages, this function * should return the maximum value that pci_alloc_msix() can return. * Thus, it is subject to the tunables, etc. */ int pci_msix_count_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; if (pci_do_msix && msix->msix_location != 0) return (msix->msix_msgnum); return (0); } int pci_msix_pba_bar_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; if (pci_do_msix && msix->msix_location != 0) return (msix->msix_pba_bar); return (-1); } int pci_msix_table_bar_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; if (pci_do_msix && msix->msix_location != 0) return (msix->msix_table_bar); return (-1); } /* * HyperTransport MSI mapping control */ void pci_ht_map_msi(device_t dev, uint64_t addr) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_ht *ht = &dinfo->cfg.ht; if (!ht->ht_msimap) return; if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) && ht->ht_msiaddr >> 20 == addr >> 20) { /* Enable MSI -> HT mapping. */ ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, ht->ht_msictrl, 2); } if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) { /* Disable MSI -> HT mapping. */ ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE; pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, ht->ht_msictrl, 2); } } int pci_get_relaxed_ordering_enabled(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_RELAXED_ORD_ENABLE; return (val != 0); } int pci_get_max_payload(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_MAX_PAYLOAD; val >>= 5; return (1 << (val + 7)); } int pci_get_max_read_req(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_MAX_READ_REQUEST; val >>= 12; return (1 << (val + 7)); } int pci_set_max_read_req(device_t dev, int size) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); if (size < 128) size = 128; if (size > 4096) size = 4096; size = (1 << (fls(size) - 1)); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= ~PCIEM_CTL_MAX_READ_REQUEST; val |= (fls(size) - 8) << 12; pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2); return (size); } uint32_t pcie_read_config(device_t dev, int reg, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) { if (width == 2) return (0xffff); return (0xffffffff); } return (pci_read_config(dev, cap + reg, width)); } void pcie_write_config(device_t dev, int reg, uint32_t value, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return; pci_write_config(dev, cap + reg, value, width); } /* * Adjusts a PCI-e capability register by clearing the bits in mask * and setting the bits in (value & mask). Bits not set in mask are * not adjusted. * * Returns the old value on success or all ones on failure. */ uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint32_t old, new; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) { if (width == 2) return (0xffff); return (0xffffffff); } old = pci_read_config(dev, cap + reg, width); new = old & ~mask; new |= (value & mask); pci_write_config(dev, cap + reg, new, width); return (old); } /* * Support for MSI message signalled interrupts. */ void pci_enable_msi_method(device_t dev, device_t child, uint64_t address, uint16_t data) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; /* Write data and address values. */ pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR, address & 0xffffffff, 4); if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH, address >> 32, 4); pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT, data, 2); } else pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data, 2); /* Enable MSI in the control register. */ msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); /* Enable MSI -> HT mapping. */ pci_ht_map_msi(child, address); } void pci_disable_msi_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; /* Disable MSI -> HT mapping. */ pci_ht_map_msi(child, 0); /* Disable MSI in the control register. */ msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); } /* * Restore MSI registers during resume. If MSI is enabled then * restore the data and address registers in addition to the control * register. */ static void pci_resume_msi(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msi *msi = &dinfo->cfg.msi; uint64_t address; uint16_t data; if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) { address = msi->msi_addr; data = msi->msi_data; pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR, address & 0xffffffff, 4); if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR_HIGH, address >> 32, 4); pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA_64BIT, data, 2); } else pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA, data, 2); } pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); } static int pci_remap_intr_method(device_t bus, device_t dev, u_int irq) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; struct msix_table_entry *mte; struct msix_vector *mv; uint64_t addr; uint32_t data; int error, i, j; /* * Handle MSI first. We try to find this IRQ among our list * of MSI IRQs. If we find it, we request updated address and * data registers and apply the results. */ if (cfg->msi.msi_alloc > 0) { /* If we don't have any active handlers, nothing to do. */ if (cfg->msi.msi_handlers == 0) return (0); for (i = 0; i < cfg->msi.msi_alloc; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); if (rle->start == irq) { error = PCIB_MAP_MSI(device_get_parent(bus), dev, irq, &addr, &data); if (error) return (error); pci_disable_msi(dev); dinfo->cfg.msi.msi_addr = addr; dinfo->cfg.msi.msi_data = data; pci_enable_msi(dev, addr, data); return (0); } } return (ENOENT); } /* * For MSI-X, we check to see if we have this IRQ. If we do, * we request the updated mapping info. If that works, we go * through all the slots that use this IRQ and update them. */ if (cfg->msix.msix_alloc > 0) { bool found = false; for (i = 0; i < cfg->msix.msix_alloc; i++) { mv = &cfg->msix.msix_vectors[i]; if (mv->mv_irq == irq) { error = PCIB_MAP_MSI(device_get_parent(bus), dev, irq, &addr, &data); if (error) return (error); mv->mv_address = addr; mv->mv_data = data; for (j = 0; j < cfg->msix.msix_table_len; j++) { mte = &cfg->msix.msix_table[j]; if (mte->mte_vector != i + 1) continue; if (mte->mte_handlers == 0) continue; pci_mask_msix(dev, j); pci_enable_msix(dev, j, addr, data); pci_unmask_msix(dev, j); } found = true; } } return (found ? 0 : ENOENT); } return (ENOENT); } /* * Returns true if the specified device is blacklisted because MSI * doesn't work. */ int pci_msi_device_blacklisted(device_t dev) { if (!pci_honor_msi_blacklist) return (0); return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI)); } /* * Determine if MSI is blacklisted globally on this system. Currently, * we just check for blacklisted chipsets as represented by the * host-PCI bridge at device 0:0:0. In the future, it may become * necessary to check other system attributes, such as the kenv values * that give the motherboard manufacturer and model number. */ static int pci_msi_blacklisted(void) { device_t dev; if (!pci_honor_msi_blacklist) return (0); /* Blacklist all non-PCI-express and non-PCI-X chipsets. */ if (!(pcie_chipset || pcix_chipset)) { if (vm_guest != VM_GUEST_NO) { /* * Whitelist older chipsets in virtual * machines known to support MSI. */ dev = pci_find_bsf(0, 0, 0); if (dev != NULL) return (!pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_ENABLE_MSI_VM)); } return (1); } dev = pci_find_bsf(0, 0, 0); if (dev != NULL) return (pci_msi_device_blacklisted(dev)); return (0); } /* * Returns true if the specified device is blacklisted because MSI-X * doesn't work. Note that this assumes that if MSI doesn't work, * MSI-X doesn't either. */ int pci_msix_device_blacklisted(device_t dev) { if (!pci_honor_msi_blacklist) return (0); if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) return (1); return (pci_msi_device_blacklisted(dev)); } /* * Determine if MSI-X is blacklisted globally on this system. If MSI * is blacklisted, assume that MSI-X is as well. Check for additional * chipsets where MSI works but MSI-X does not. */ static int pci_msix_blacklisted(void) { device_t dev; if (!pci_honor_msi_blacklist) return (0); dev = pci_find_bsf(0, 0, 0); if (dev != NULL && pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) return (1); return (pci_msi_blacklisted()); } /* * Attempt to allocate *count MSI messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at a rid 1. */ int pci_alloc_msi_method(device_t dev, device_t child, int *count) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; int actual, error, i, irqs[32]; uint16_t ctrl; /* Don't let count == 0 get us into trouble. */ if (*count == 0) return (EINVAL); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) return (ENXIO); /* Already have allocated messages? */ if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) return (ENXIO); /* If MSI is blacklisted for this system, fail. */ if (pci_msi_blacklisted()) return (ENXIO); /* MSI capability present? */ if (cfg->msi.msi_location == 0 || !pci_do_msi) return (ENODEV); if (bootverbose) device_printf(child, "attempting to allocate %d MSI vectors (%d supported)\n", *count, cfg->msi.msi_msgnum); /* Don't ask for more than the device supports. */ actual = min(*count, cfg->msi.msi_msgnum); /* Don't ask for more than 32 messages. */ actual = min(actual, 32); /* MSI requires power of 2 number of messages. */ if (!powerof2(actual)) return (EINVAL); for (;;) { /* Try to allocate N messages. */ error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual, actual, irqs); if (error == 0) break; if (actual == 1) return (error); /* Try N / 2. */ actual >>= 1; } /* * We now have N actual messages mapped onto SYS_RES_IRQ * resources in the irqs[] array, so add new resources * starting at rid 1. */ for (i = 0; i < actual; i++) resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irqs[i], irqs[i], 1); if (bootverbose) { if (actual == 1) device_printf(child, "using IRQ %d for MSI\n", irqs[0]); else { int run; /* * Be fancy and try to print contiguous runs * of IRQ values as ranges. 'run' is true if * we are in a range. */ device_printf(child, "using IRQs %d", irqs[0]); run = 0; for (i = 1; i < actual; i++) { /* Still in a run? */ if (irqs[i] == irqs[i - 1] + 1) { run = 1; continue; } /* Finish previous range. */ if (run) { printf("-%d", irqs[i - 1]); run = 0; } /* Start new range. */ printf(",%d", irqs[i]); } /* Unfinished range? */ if (run) printf("-%d", irqs[actual - 1]); printf(" for MSI\n"); } } /* Update control register with actual count. */ ctrl = cfg->msi.msi_ctrl; ctrl &= ~PCIM_MSICTRL_MME_MASK; ctrl |= (ffs(actual) - 1) << 4; cfg->msi.msi_ctrl = ctrl; pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2); /* Update counts of alloc'd messages. */ cfg->msi.msi_alloc = actual; cfg->msi.msi_handlers = 0; *count = actual; return (0); } /* Release the MSI messages associated with this device. */ int pci_release_msi_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; struct resource_list_entry *rle; int error, i, irqs[32]; /* Try MSI-X first. */ error = pci_release_msix(dev, child); if (error != ENODEV) return (error); /* Do we have any messages to release? */ if (msi->msi_alloc == 0) return (ENODEV); KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages")); /* Make sure none of the resources are allocated. */ if (msi->msi_handlers > 0) return (EBUSY); for (i = 0; i < msi->msi_alloc; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing MSI resource")); if (rle->res != NULL) return (EBUSY); irqs[i] = rle->start; } /* Update control register with 0 count. */ KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE), ("%s: MSI still enabled", __func__)); msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); /* Release the messages. */ PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs); for (i = 0; i < msi->msi_alloc; i++) resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); /* Update alloc count. */ msi->msi_alloc = 0; msi->msi_addr = 0; msi->msi_data = 0; return (0); } /* * Return the max supported MSI messages this device supports. * Basically, assuming the MD code can alloc messages, this function * should return the maximum value that pci_alloc_msi() can return. * Thus, it is subject to the tunables, etc. */ int pci_msi_count_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; if (pci_do_msi && msi->msi_location != 0) return (msi->msi_msgnum); return (0); } /* free pcicfgregs structure and all depending data structures */ int pci_freecfg(struct pci_devinfo *dinfo) { struct devlist *devlist_head; struct pci_map *pm, *next; devlist_head = &pci_devq; if (dinfo->cfg.vpd.vpd_reg) vpd_free(&dinfo->cfg.vpd); STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) { free(pm, M_DEVBUF); } STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links); free(dinfo, M_DEVBUF); /* increment the generation count */ pci_generation++; /* we're losing one device */ pci_numdevs--; return (0); } /* * PCI power manangement */ int pci_set_powerstate_method(device_t dev, device_t child, int state) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint16_t status; int oldstate, highest, delay; if (cfg->pp.pp_cap == 0) return (EOPNOTSUPP); /* * Optimize a no state change request away. While it would be OK to * write to the hardware in theory, some devices have shown odd * behavior when going from D3 -> D3. */ oldstate = pci_get_powerstate(child); if (oldstate == state) return (0); /* * The PCI power management specification states that after a state * transition between PCI power states, system software must * guarantee a minimal delay before the function accesses the device. * Compute the worst case delay that we need to guarantee before we * access the device. Many devices will be responsive much more * quickly than this delay, but there are some that don't respond * instantly to state changes. Transitions to/from D3 state require * 10ms, while D2 requires 200us, and D0/1 require none. The delay * is done below with DELAY rather than a sleeper function because * this function can be called from contexts where we cannot sleep. */ highest = (oldstate > state) ? oldstate : state; if (highest == PCI_POWERSTATE_D3) delay = 10000; else if (highest == PCI_POWERSTATE_D2) delay = 200; else delay = 0; status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2) & ~PCIM_PSTAT_DMASK; switch (state) { case PCI_POWERSTATE_D0: status |= PCIM_PSTAT_D0; break; case PCI_POWERSTATE_D1: if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0) return (EOPNOTSUPP); status |= PCIM_PSTAT_D1; break; case PCI_POWERSTATE_D2: if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0) return (EOPNOTSUPP); status |= PCIM_PSTAT_D2; break; case PCI_POWERSTATE_D3: status |= PCIM_PSTAT_D3; break; default: return (EINVAL); } if (bootverbose) pci_printf(cfg, "Transition from D%d to D%d\n", oldstate, state); PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2); if (delay) DELAY(delay); return (0); } int pci_get_powerstate_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint16_t status; int result; if (cfg->pp.pp_cap != 0) { status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2); switch (status & PCIM_PSTAT_DMASK) { case PCIM_PSTAT_D0: result = PCI_POWERSTATE_D0; break; case PCIM_PSTAT_D1: result = PCI_POWERSTATE_D1; break; case PCIM_PSTAT_D2: result = PCI_POWERSTATE_D2; break; case PCIM_PSTAT_D3: result = PCI_POWERSTATE_D3; break; default: result = PCI_POWERSTATE_UNKNOWN; break; } } else { /* No support, device is always at D0 */ result = PCI_POWERSTATE_D0; } return (result); } /* * Some convenience functions for PCI device drivers. */ static __inline void pci_set_command_bit(device_t dev, device_t child, uint16_t bit) { uint16_t command; command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); command |= bit; PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); } static __inline void pci_clear_command_bit(device_t dev, device_t child, uint16_t bit) { uint16_t command; command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); command &= ~bit; PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); } int pci_enable_busmaster_method(device_t dev, device_t child) { pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); return (0); } int pci_disable_busmaster_method(device_t dev, device_t child) { pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); return (0); } int pci_enable_io_method(device_t dev, device_t child, int space) { uint16_t bit; switch(space) { case SYS_RES_IOPORT: bit = PCIM_CMD_PORTEN; break; case SYS_RES_MEMORY: bit = PCIM_CMD_MEMEN; break; default: return (EINVAL); } pci_set_command_bit(dev, child, bit); return (0); } int pci_disable_io_method(device_t dev, device_t child, int space) { uint16_t bit; switch(space) { case SYS_RES_IOPORT: bit = PCIM_CMD_PORTEN; break; case SYS_RES_MEMORY: bit = PCIM_CMD_MEMEN; break; default: return (EINVAL); } pci_clear_command_bit(dev, child, bit); return (0); } /* * New style pci driver. Parent device is either a pci-host-bridge or a * pci-pci-bridge. Both kinds are represented by instances of pcib. */ void pci_print_verbose(struct pci_devinfo *dinfo) { if (bootverbose) { pcicfgregs *cfg = &dinfo->cfg; printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n", cfg->vendor, cfg->device, cfg->revid); printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n", cfg->domain, cfg->bus, cfg->slot, cfg->func); printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n", cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype, cfg->mfdev); printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n", cfg->cmdreg, cfg->statreg, cfg->cachelnsz); printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n", cfg->lattimer, cfg->lattimer * 30, cfg->mingnt, cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250); if (cfg->intpin > 0) printf("\tintpin=%c, irq=%d\n", cfg->intpin +'a' -1, cfg->intline); if (cfg->pp.pp_cap) { uint16_t status; status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2); printf("\tpowerspec %d supports D0%s%s D3 current D%d\n", cfg->pp.pp_cap & PCIM_PCAP_SPEC, cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "", cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "", status & PCIM_PSTAT_DMASK); } if (cfg->msi.msi_location) { int ctrl; ctrl = cfg->msi.msi_ctrl; printf("\tMSI supports %d message%s%s%s\n", cfg->msi.msi_msgnum, (cfg->msi.msi_msgnum == 1) ? "" : "s", (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "", (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":""); } if (cfg->msix.msix_location) { printf("\tMSI-X supports %d message%s ", cfg->msix.msix_msgnum, (cfg->msix.msix_msgnum == 1) ? "" : "s"); if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar) printf("in map 0x%x\n", cfg->msix.msix_table_bar); else printf("in maps 0x%x and 0x%x\n", cfg->msix.msix_table_bar, cfg->msix.msix_pba_bar); } } } static int pci_porten(device_t dev) { return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0; } static int pci_memen(device_t dev) { return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0; } void pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp, int *bar64) { struct pci_devinfo *dinfo; pci_addr_t map, testval; int ln2range; uint16_t cmd; /* * The device ROM BAR is special. It is always a 32-bit * memory BAR. Bit 0 is special and should not be set when * sizing the BAR. */ dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, reg)) { map = pci_read_config(dev, reg, 4); pci_write_config(dev, reg, 0xfffffffe, 4); testval = pci_read_config(dev, reg, 4); pci_write_config(dev, reg, map, 4); *mapp = map; *testvalp = testval; if (bar64 != NULL) *bar64 = 0; return; } map = pci_read_config(dev, reg, 4); ln2range = pci_maprange(map); if (ln2range == 64) map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; /* * Disable decoding via the command register before * determining the BAR's length since we will be placing it in * a weird state. */ cmd = pci_read_config(dev, PCIR_COMMAND, 2); pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); /* * Determine the BAR's length by writing all 1's. The bottom * log_2(size) bits of the BAR will stick as 0 when we read * the value back. * * NB: according to the PCI Local Bus Specification, rev. 3.0: * "Software writes 0FFFFFFFFh to both registers, reads them back, * and combines the result into a 64-bit value." (section 6.2.5.1) * * Writes to both registers must be performed before attempting to * read back the size value. */ testval = 0; pci_write_config(dev, reg, 0xffffffff, 4); if (ln2range == 64) { pci_write_config(dev, reg + 4, 0xffffffff, 4); testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; } testval |= pci_read_config(dev, reg, 4); /* * Restore the original value of the BAR. We may have reprogrammed * the BAR of the low-level console device and when booting verbose, * we need the console device addressable. */ pci_write_config(dev, reg, map, 4); if (ln2range == 64) pci_write_config(dev, reg + 4, map >> 32, 4); pci_write_config(dev, PCIR_COMMAND, cmd, 2); *mapp = map; *testvalp = testval; if (bar64 != NULL) *bar64 = (ln2range == 64); } static void pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base) { struct pci_devinfo *dinfo; int ln2range; /* The device ROM BAR is always a 32-bit memory BAR. */ dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) ln2range = 32; else ln2range = pci_maprange(pm->pm_value); pci_write_config(dev, pm->pm_reg, base, 4); if (ln2range == 64) pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4); pm->pm_value = pci_read_config(dev, pm->pm_reg, 4); if (ln2range == 64) pm->pm_value |= (pci_addr_t)pci_read_config(dev, pm->pm_reg + 4, 4) << 32; } struct pci_map * pci_find_bar(device_t dev, int reg) { struct pci_devinfo *dinfo; struct pci_map *pm; dinfo = device_get_ivars(dev); STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { if (pm->pm_reg == reg) return (pm); } return (NULL); } struct pci_map * pci_first_bar(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); return (STAILQ_FIRST(&dinfo->cfg.maps)); } struct pci_map * pci_next_bar(struct pci_map *pm) { return (STAILQ_NEXT(pm, pm_link)); } int pci_bar_enabled(device_t dev, struct pci_map *pm) { struct pci_devinfo *dinfo; uint16_t cmd; dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) && !(pm->pm_value & PCIM_BIOS_ENABLE)) return (0); #ifdef PCI_IOV if ((dinfo->cfg.flags & PCICFG_VF) != 0) { struct pcicfg_iov *iov; iov = dinfo->cfg.iov; cmd = pci_read_config(iov->iov_pf, iov->iov_pos + PCIR_SRIOV_CTL, 2); return ((cmd & PCIM_SRIOV_VF_MSE) != 0); } #endif cmd = pci_read_config(dev, PCIR_COMMAND, 2); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value)) return ((cmd & PCIM_CMD_MEMEN) != 0); else return ((cmd & PCIM_CMD_PORTEN) != 0); } struct pci_map * pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size) { struct pci_devinfo *dinfo; struct pci_map *pm, *prev; dinfo = device_get_ivars(dev); pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO); pm->pm_reg = reg; pm->pm_value = value; pm->pm_size = size; STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) { KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x", reg)); if (STAILQ_NEXT(prev, pm_link) == NULL || STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg) break; } if (prev != NULL) STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link); else STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link); return (pm); } static void pci_restore_bars(device_t dev) { struct pci_devinfo *dinfo; struct pci_map *pm; int ln2range; dinfo = device_get_ivars(dev); STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) ln2range = 32; else ln2range = pci_maprange(pm->pm_value); pci_write_config(dev, pm->pm_reg, pm->pm_value, 4); if (ln2range == 64) pci_write_config(dev, pm->pm_reg + 4, pm->pm_value >> 32, 4); } } /* * Add a resource based on a pci map register. Return 1 if the map * register is a 32bit map register or 2 if it is a 64bit register. */ static int pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, int force, int prefetch) { struct pci_map *pm; pci_addr_t base, map, testval; pci_addr_t start, end, count; int barlen, basezero, flags, maprange, mapsize, type; uint16_t cmd; struct resource *res; /* * The BAR may already exist if the device is a CardBus card * whose CIS is stored in this BAR. */ pm = pci_find_bar(dev, reg); if (pm != NULL) { maprange = pci_maprange(pm->pm_value); barlen = maprange == 64 ? 2 : 1; return (barlen); } pci_read_bar(dev, reg, &map, &testval, NULL); if (PCI_BAR_MEM(map)) { type = SYS_RES_MEMORY; if (map & PCIM_BAR_MEM_PREFETCH) prefetch = 1; } else type = SYS_RES_IOPORT; mapsize = pci_mapsize(testval); base = pci_mapbase(map); #ifdef __PCI_BAR_ZERO_VALID basezero = 0; #else basezero = base == 0; #endif maprange = pci_maprange(map); barlen = maprange == 64 ? 2 : 1; /* * For I/O registers, if bottom bit is set, and the next bit up * isn't clear, we know we have a BAR that doesn't conform to the * spec, so ignore it. Also, sanity check the size of the data * areas to the type of memory involved. Memory must be at least * 16 bytes in size, while I/O ranges must be at least 4. */ if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0) return (barlen); if ((type == SYS_RES_MEMORY && mapsize < 4) || (type == SYS_RES_IOPORT && mapsize < 2)) return (barlen); /* Save a record of this BAR. */ pm = pci_add_bar(dev, reg, map, mapsize); if (bootverbose) { printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d", reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize); if (type == SYS_RES_IOPORT && !pci_porten(dev)) printf(", port disabled\n"); else if (type == SYS_RES_MEMORY && !pci_memen(dev)) printf(", memory disabled\n"); else printf(", enabled\n"); } /* * If base is 0, then we have problems if this architecture does * not allow that. It is best to ignore such entries for the * moment. These will be allocated later if the driver specifically * requests them. However, some removable buses look better when * all resources are allocated, so allow '0' to be overridden. * * Similarly treat maps whose values is the same as the test value * read back. These maps have had all f's written to them by the * BIOS in an attempt to disable the resources. */ if (!force && (basezero || map == testval)) return (barlen); if ((u_long)base != base) { device_printf(bus, "pci%d:%d:%d:%d bar %#x too many address bits", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), reg); return (barlen); } /* * This code theoretically does the right thing, but has * undesirable side effects in some cases where peripherals * respond oddly to having these bits enabled. Let the user * be able to turn them off (since pci_enable_io_modes is 1 by * default). */ if (pci_enable_io_modes) { /* Turn on resources that have been left off by a lazy BIOS */ if (type == SYS_RES_IOPORT && !pci_porten(dev)) { cmd = pci_read_config(dev, PCIR_COMMAND, 2); cmd |= PCIM_CMD_PORTEN; pci_write_config(dev, PCIR_COMMAND, cmd, 2); } if (type == SYS_RES_MEMORY && !pci_memen(dev)) { cmd = pci_read_config(dev, PCIR_COMMAND, 2); cmd |= PCIM_CMD_MEMEN; pci_write_config(dev, PCIR_COMMAND, cmd, 2); } } else { if (type == SYS_RES_IOPORT && !pci_porten(dev)) return (barlen); if (type == SYS_RES_MEMORY && !pci_memen(dev)) return (barlen); } count = (pci_addr_t)1 << mapsize; flags = RF_ALIGNMENT_LOG2(mapsize); if (prefetch) flags |= RF_PREFETCHABLE; if (basezero || base == pci_mapbase(testval) || pci_clear_bars) { start = 0; /* Let the parent decide. */ end = ~0; } else { start = base; end = base + count - 1; } resource_list_add(rl, type, reg, start, end, count); /* * Try to allocate the resource for this BAR from our parent * so that this resource range is already reserved. The * driver for this device will later inherit this resource in * pci_alloc_resource(). */ res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count, flags); if ((pci_do_realloc_bars || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR)) && res == NULL && (start != 0 || end != ~0)) { /* * If the allocation fails, try to allocate a resource for * this BAR using any available range. The firmware felt * it was important enough to assign a resource, so don't * disable decoding if we can help it. */ resource_list_delete(rl, type, reg); resource_list_add(rl, type, reg, 0, ~0, count); res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0, count, flags); } if (res == NULL) { /* * If the allocation fails, delete the resource list entry * and disable decoding for this device. * * If the driver requests this resource in the future, * pci_reserve_map() will try to allocate a fresh * resource range. */ resource_list_delete(rl, type, reg); pci_disable_io(dev, type); if (bootverbose) device_printf(bus, "pci%d:%d:%d:%d bar %#x failed to allocate\n", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), reg); } else { start = rman_get_start(res); pci_write_bar(dev, pm, start); } return (barlen); } /* * For ATA devices we need to decide early what addressing mode to use. * Legacy demands that the primary and secondary ATA ports sits on the * same addresses that old ISA hardware did. This dictates that we use * those addresses and ignore the BAR's if we cannot set PCI native * addressing mode. */ static void pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force, uint32_t prefetchmask) { int rid, type, progif; #if 0 /* if this device supports PCI native addressing use it */ progif = pci_read_config(dev, PCIR_PROGIF, 1); if ((progif & 0x8a) == 0x8a) { if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) && pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) { printf("Trying ATA native PCI addressing mode\n"); pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1); } } #endif progif = pci_read_config(dev, PCIR_PROGIF, 1); type = SYS_RES_IOPORT; if (progif & PCIP_STORAGE_IDE_MODEPRIM) { pci_add_map(bus, dev, PCIR_BAR(0), rl, force, prefetchmask & (1 << 0)); pci_add_map(bus, dev, PCIR_BAR(1), rl, force, prefetchmask & (1 << 1)); } else { rid = PCIR_BAR(0); resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0, 0x1f7, 8, 0); rid = PCIR_BAR(1); resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6, 0x3f6, 1, 0); } if (progif & PCIP_STORAGE_IDE_MODESEC) { pci_add_map(bus, dev, PCIR_BAR(2), rl, force, prefetchmask & (1 << 2)); pci_add_map(bus, dev, PCIR_BAR(3), rl, force, prefetchmask & (1 << 3)); } else { rid = PCIR_BAR(2); resource_list_add(rl, type, rid, 0x170, 0x177, 8); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170, 0x177, 8, 0); rid = PCIR_BAR(3); resource_list_add(rl, type, rid, 0x376, 0x376, 1); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376, 0x376, 1, 0); } pci_add_map(bus, dev, PCIR_BAR(4), rl, force, prefetchmask & (1 << 4)); pci_add_map(bus, dev, PCIR_BAR(5), rl, force, prefetchmask & (1 << 5)); } static void pci_assign_interrupt(device_t bus, device_t dev, int force_route) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; char tunable_name[64]; int irq; /* Has to have an intpin to have an interrupt. */ if (cfg->intpin == 0) return; /* Let the user override the IRQ with a tunable. */ irq = PCI_INVALID_IRQ; snprintf(tunable_name, sizeof(tunable_name), "hw.pci%d.%d.%d.INT%c.irq", cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1); if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0)) irq = PCI_INVALID_IRQ; /* * If we didn't get an IRQ via the tunable, then we either use the * IRQ value in the intline register or we ask the bus to route an * interrupt for us. If force_route is true, then we only use the * value in the intline register if the bus was unable to assign an * IRQ. */ if (!PCI_INTERRUPT_VALID(irq)) { if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route) irq = PCI_ASSIGN_INTERRUPT(bus, dev); if (!PCI_INTERRUPT_VALID(irq)) irq = cfg->intline; } /* If after all that we don't have an IRQ, just bail. */ if (!PCI_INTERRUPT_VALID(irq)) return; /* Update the config register if it changed. */ if (irq != cfg->intline) { cfg->intline = irq; pci_write_config(dev, PCIR_INTLINE, irq, 1); } /* Add this IRQ as rid 0 interrupt resource. */ resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1); } /* Perform early OHCI takeover from SMM. */ static void ohci_early_takeover(device_t self) { struct resource *res; uint32_t ctl; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; ctl = bus_read_4(res, OHCI_CONTROL); if (ctl & OHCI_IR) { if (bootverbose) printf("ohci early: " "SMM active, request owner change\n"); bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR); for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) { DELAY(1000); ctl = bus_read_4(res, OHCI_CONTROL); } if (ctl & OHCI_IR) { if (bootverbose) printf("ohci early: " "SMM does not respond, resetting\n"); bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET); } /* Disable interrupts */ bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } /* Perform early UHCI takeover from SMM. */ static void uhci_early_takeover(device_t self) { struct resource *res; int rid; /* * Set the PIRQD enable bit and switch off all the others. We don't * want legacy support to interfere with us XXX Does this also mean * that the BIOS won't touch the keyboard anymore if it is connected * to the ports of the root hub? */ pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2); /* Disable interrupts */ rid = PCI_UHCI_BASE_REG; res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE); if (res != NULL) { bus_write_2(res, UHCI_INTR, 0); bus_release_resource(self, SYS_RES_IOPORT, rid, res); } } /* Perform early EHCI takeover from SMM. */ static void ehci_early_takeover(device_t self) { struct resource *res; uint32_t cparams; uint32_t eec; uint8_t eecp; uint8_t bios_sem; uint8_t offs; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; cparams = bus_read_4(res, EHCI_HCCPARAMS); /* Synchronise with the BIOS if it owns the controller. */ for (eecp = EHCI_HCC_EECP(cparams); eecp != 0; eecp = EHCI_EECP_NEXT(eec)) { eec = pci_read_config(self, eecp, 4); if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) { continue; } bios_sem = pci_read_config(self, eecp + EHCI_LEGSUP_BIOS_SEM, 1); if (bios_sem == 0) { continue; } if (bootverbose) printf("ehci early: " "SMM active, request owner change\n"); pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1); for (i = 0; (i < 100) && (bios_sem != 0); i++) { DELAY(1000); bios_sem = pci_read_config(self, eecp + EHCI_LEGSUP_BIOS_SEM, 1); } if (bios_sem != 0) { if (bootverbose) printf("ehci early: " "SMM does not respond\n"); } /* Disable interrupts */ offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION)); bus_write_4(res, offs + EHCI_USBINTR, 0); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } /* Perform early XHCI takeover from SMM. */ static void xhci_early_takeover(device_t self) { struct resource *res; uint32_t cparams; uint32_t eec; uint8_t eecp; uint8_t bios_sem; uint8_t offs; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; cparams = bus_read_4(res, XHCI_HCSPARAMS0); eec = -1; /* Synchronise with the BIOS if it owns the controller. */ for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec); eecp += XHCI_XECP_NEXT(eec) << 2) { eec = bus_read_4(res, eecp); if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY) continue; bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM); if (bios_sem == 0) continue; if (bootverbose) printf("xhci early: " "SMM active, request owner change\n"); bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1); /* wait a maximum of 5 second */ for (i = 0; (i < 5000) && (bios_sem != 0); i++) { DELAY(1000); bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM); } if (bios_sem != 0) { if (bootverbose) printf("xhci early: " "SMM does not respond\n"); } /* Disable interrupts */ offs = bus_read_1(res, XHCI_CAPLENGTH); bus_write_4(res, offs + XHCI_USBCMD, 0); bus_read_4(res, offs + XHCI_USBSTS); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } static void pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg, struct resource_list *rl) { struct resource *res; char *cp; rman_res_t start, end, count; int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_BRIDGE: sec_reg = PCIR_SECBUS_1; sub_reg = PCIR_SUBBUS_1; break; case PCIM_HDRTYPE_CARDBUS: sec_reg = PCIR_SECBUS_2; sub_reg = PCIR_SUBBUS_2; break; default: return; } /* * If the existing bus range is valid, attempt to reserve it * from our parent. If this fails for any reason, clear the * secbus and subbus registers. * * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus? * This would at least preserve the existing sec_bus if it is * valid. */ sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1); sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1); /* Quirk handling. */ switch (pci_get_devid(dev)) { case 0x12258086: /* Intel 82454KX/GX (Orion) */ sup_bus = pci_read_config(dev, 0x41, 1); if (sup_bus != 0xff) { sec_bus = sup_bus + 1; sub_bus = sup_bus + 1; PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1); PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); } break; case 0x00dd10de: /* Compaq R3000 BIOS sets wrong subordinate bus number. */ if ((cp = kern_getenv("smbios.planar.maker")) == NULL) break; if (strncmp(cp, "Compal", 6) != 0) { freeenv(cp); break; } freeenv(cp); if ((cp = kern_getenv("smbios.planar.product")) == NULL) break; if (strncmp(cp, "08A0", 4) != 0) { freeenv(cp); break; } freeenv(cp); if (sub_bus < 0xa) { sub_bus = 0xa; PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); } break; } if (bootverbose) printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus); if (sec_bus > 0 && sub_bus >= sec_bus) { start = sec_bus; end = sub_bus; count = end - start + 1; resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count); /* * If requested, clear secondary bus registers in * bridge devices to force a complete renumbering * rather than reserving the existing range. However, * preserve the existing size. */ if (pci_clear_buses) goto clear; rid = 0; res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid, start, end, count, 0); if (res != NULL) return; if (bootverbose) device_printf(bus, "pci%d:%d:%d:%d secbus failed to allocate\n", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); } clear: PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1); PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1); } static struct resource * pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct pci_devinfo *dinfo; pcicfgregs *cfg; struct resource_list *rl; struct resource *res; int sec_reg, sub_reg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; rl = &dinfo->resources; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_BRIDGE: sec_reg = PCIR_SECBUS_1; sub_reg = PCIR_SUBBUS_1; break; case PCIM_HDRTYPE_CARDBUS: sec_reg = PCIR_SECBUS_2; sub_reg = PCIR_SUBBUS_2; break; default: return (NULL); } if (*rid != 0) return (NULL); if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL) resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count); if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) { res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid, start, end, count, flags & ~RF_ACTIVE); if (res == NULL) { resource_list_delete(rl, PCI_RES_BUS, *rid); device_printf(child, "allocating %ju bus%s failed\n", count, count == 1 ? "" : "es"); return (NULL); } if (bootverbose) device_printf(child, "Lazy allocation of %ju bus%s at %ju\n", count, count == 1 ? "" : "es", rman_get_start(res)); PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1); PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1); } return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start, end, count, flags)); } static int pci_ea_bei_to_rid(device_t dev, int bei) { #ifdef PCI_IOV struct pci_devinfo *dinfo; int iov_pos; struct pcicfg_iov *iov; dinfo = device_get_ivars(dev); iov = dinfo->cfg.iov; if (iov != NULL) iov_pos = iov->iov_pos; else iov_pos = 0; #endif /* Check if matches BAR */ if ((bei >= PCIM_EA_BEI_BAR_0) && (bei <= PCIM_EA_BEI_BAR_5)) return (PCIR_BAR(bei)); /* Check ROM */ if (bei == PCIM_EA_BEI_ROM) return (PCIR_BIOS); #ifdef PCI_IOV /* Check if matches VF_BAR */ if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) && (bei <= PCIM_EA_BEI_VF_BAR_5)) return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) + iov_pos); #endif return (-1); } int pci_ea_is_enabled(device_t dev, int rid) { struct pci_ea_entry *ea; struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid) return ((ea->eae_flags & PCIM_EA_ENABLE) > 0); } return (0); } void pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov) { struct pci_ea_entry *ea; struct pci_devinfo *dinfo; pci_addr_t start, end, count; struct resource_list *rl; int type, flags, rid; struct resource *res; uint32_t tmp; #ifdef PCI_IOV struct pcicfg_iov *iov; #endif dinfo = device_get_ivars(dev); rl = &dinfo->resources; flags = 0; #ifdef PCI_IOV iov = dinfo->cfg.iov; #endif if (dinfo->cfg.ea.ea_location == 0) return; STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { /* * TODO: Ignore EA-BAR if is not enabled. * Currently the EA implementation supports * only situation, where EA structure contains * predefined entries. In case they are not enabled * leave them unallocated and proceed with * a legacy-BAR mechanism. */ if ((ea->eae_flags & PCIM_EA_ENABLE) == 0) continue; switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) { case PCIM_EA_P_MEM_PREFETCH: case PCIM_EA_P_VF_MEM_PREFETCH: flags = RF_PREFETCHABLE; /* FALLTHROUGH */ case PCIM_EA_P_VF_MEM: case PCIM_EA_P_MEM: type = SYS_RES_MEMORY; break; case PCIM_EA_P_IO: type = SYS_RES_IOPORT; break; default: continue; } if (alloc_iov != 0) { #ifdef PCI_IOV /* Allocating IOV, confirm BEI matches */ if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) || (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5)) continue; #else continue; #endif } else { /* Allocating BAR, confirm BEI matches */ if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) || (ea->eae_bei > PCIM_EA_BEI_BAR_5)) && (ea->eae_bei != PCIM_EA_BEI_ROM)) continue; } rid = pci_ea_bei_to_rid(dev, ea->eae_bei); if (rid < 0) continue; /* Skip resources already allocated by EA */ if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) || (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL)) continue; start = ea->eae_base; count = ea->eae_max_offset + 1; #ifdef PCI_IOV if (iov != NULL) count = count * iov->iov_num_vfs; #endif end = start + count - 1; if (count == 0) continue; resource_list_add(rl, type, rid, start, end, count); res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count, flags); if (res == NULL) { resource_list_delete(rl, type, rid); /* * Failed to allocate using EA, disable entry. * Another attempt to allocation will be performed * further, but this time using legacy BAR registers */ tmp = pci_read_config(dev, ea->eae_cfg_offset, 4); tmp &= ~PCIM_EA_ENABLE; pci_write_config(dev, ea->eae_cfg_offset, tmp, 4); /* * Disabling entry might fail in case it is hardwired. * Read flags again to match current status. */ ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4); continue; } /* As per specification, fill BAR with zeros */ pci_write_config(dev, rid, 0, 4); } } void pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask) { struct pci_devinfo *dinfo; pcicfgregs *cfg; struct resource_list *rl; const struct pci_quirk *q; uint32_t devid; int i; dinfo = device_get_ivars(dev); cfg = &dinfo->cfg; rl = &dinfo->resources; devid = (cfg->device << 16) | cfg->vendor; /* Allocate resources using Enhanced Allocation */ pci_add_resources_ea(bus, dev, 0); /* ATA devices needs special map treatment */ if ((pci_get_class(dev) == PCIC_STORAGE) && (pci_get_subclass(dev) == PCIS_STORAGE_IDE) && ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) || (!pci_read_config(dev, PCIR_BAR(0), 4) && !pci_read_config(dev, PCIR_BAR(2), 4))) ) pci_ata_maps(bus, dev, rl, force, prefetchmask); else for (i = 0; i < cfg->nummaps;) { /* Skip resources already managed by EA */ if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) || (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) || pci_ea_is_enabled(dev, PCIR_BAR(i))) { i++; continue; } /* * Skip quirked resources. */ for (q = &pci_quirks[0]; q->devid != 0; q++) if (q->devid == devid && q->type == PCI_QUIRK_UNMAP_REG && q->arg1 == PCIR_BAR(i)) break; if (q->devid != 0) { i++; continue; } i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force, prefetchmask & (1 << i)); } /* * Add additional, quirked resources. */ for (q = &pci_quirks[0]; q->devid != 0; q++) if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG) pci_add_map(bus, dev, q->arg1, rl, force, 0); if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) { #ifdef __PCI_REROUTE_INTERRUPT /* * Try to re-route interrupts. Sometimes the BIOS or * firmware may leave bogus values in these registers. * If the re-route fails, then just stick with what we * have. */ pci_assign_interrupt(bus, dev, 1); #else pci_assign_interrupt(bus, dev, 0); #endif } if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS && pci_get_subclass(dev) == PCIS_SERIALBUS_USB) { if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI) xhci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI) ehci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI) ohci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI) uhci_early_takeover(dev); } /* * Reserve resources for secondary bus ranges behind bridge * devices. */ pci_reserve_secbus(bus, dev, cfg, rl); } static struct pci_devinfo * pci_identify_function(device_t pcib, device_t dev, int domain, int busno, int slot, int func) { struct pci_devinfo *dinfo; dinfo = pci_read_device(pcib, dev, domain, busno, slot, func); if (dinfo != NULL) pci_add_child(dev, dinfo); return (dinfo); } void pci_add_children(device_t dev, int domain, int busno) { #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) device_t pcib = device_get_parent(dev); struct pci_devinfo *dinfo; int maxslots; int s, f, pcifunchigh; uint8_t hdrtype; int first_func; /* * Try to detect a device at slot 0, function 0. If it exists, try to * enable ARI. We must enable ARI before detecting the rest of the * functions on this bus as ARI changes the set of slots and functions * that are legal on this bus. */ dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0); if (dinfo != NULL && pci_enable_ari) PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev); /* * Start looking for new devices on slot 0 at function 1 because we * just identified the device at slot 0, function 0. */ first_func = 1; maxslots = PCIB_MAXSLOTS(pcib); for (s = 0; s <= maxslots; s++, first_func = 0) { pcifunchigh = 0; f = 0; DELAY(1); /* If function 0 is not present, skip to the next slot. */ if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; hdrtype = REG(PCIR_HDRTYPE, 1); if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) continue; if (hdrtype & PCIM_MFDEV) pcifunchigh = PCIB_MAXFUNCS(pcib); for (f = first_func; f <= pcifunchigh; f++) pci_identify_function(pcib, dev, domain, busno, s, f); } #undef REG } int pci_rescan_method(device_t dev) { #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) device_t pcib = device_get_parent(dev); device_t child, *devlist, *unchanged; int devcount, error, i, j, maxslots, oldcount; int busno, domain, s, f, pcifunchigh; uint8_t hdrtype; /* No need to check for ARI on a rescan. */ error = device_get_children(dev, &devlist, &devcount); if (error) return (error); if (devcount != 0) { unchanged = malloc(devcount * sizeof(device_t), M_TEMP, M_NOWAIT | M_ZERO); if (unchanged == NULL) { free(devlist, M_TEMP); return (ENOMEM); } } else unchanged = NULL; domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); maxslots = PCIB_MAXSLOTS(pcib); for (s = 0; s <= maxslots; s++) { /* If function 0 is not present, skip to the next slot. */ f = 0; if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; pcifunchigh = 0; hdrtype = REG(PCIR_HDRTYPE, 1); if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) continue; if (hdrtype & PCIM_MFDEV) pcifunchigh = PCIB_MAXFUNCS(pcib); for (f = 0; f <= pcifunchigh; f++) { if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; /* * Found a valid function. Check if a * device_t for this device already exists. */ for (i = 0; i < devcount; i++) { child = devlist[i]; if (child == NULL) continue; if (pci_get_slot(child) == s && pci_get_function(child) == f) { unchanged[i] = child; goto next_func; } } pci_identify_function(pcib, dev, domain, busno, s, f); next_func:; } } /* Remove devices that are no longer present. */ for (i = 0; i < devcount; i++) { if (unchanged[i] != NULL) continue; device_delete_child(dev, devlist[i]); } free(devlist, M_TEMP); oldcount = devcount; /* Try to attach the devices just added. */ error = device_get_children(dev, &devlist, &devcount); if (error) { free(unchanged, M_TEMP); return (error); } for (i = 0; i < devcount; i++) { for (j = 0; j < oldcount; j++) { if (devlist[i] == unchanged[j]) goto next_device; } device_probe_and_attach(devlist[i]); next_device:; } free(unchanged, M_TEMP); free(devlist, M_TEMP); return (0); #undef REG } #ifdef PCI_IOV device_t pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid, uint16_t did) { struct pci_devinfo *vf_dinfo; device_t pcib; int busno, slot, func; pcib = device_get_parent(bus); PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func); vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno, slot, func, vid, did); vf_dinfo->cfg.flags |= PCICFG_VF; pci_add_child(bus, vf_dinfo); return (vf_dinfo->cfg.dev); } device_t pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid, uint16_t vid, uint16_t did) { return (pci_add_iov_child(bus, pf, rid, vid, did)); } #endif /* * For PCIe device set Max_Payload_Size to match PCIe root's. */ static void pcie_setup_mps(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); device_t root; uint16_t rmps, mmps, mps; if (dinfo->cfg.pcie.pcie_location == 0) return; root = pci_find_pcie_root_port(dev); if (root == NULL) return; /* Check whether the MPS is already configured. */ rmps = pcie_read_config(root, PCIER_DEVICE_CTL, 2) & PCIEM_CTL_MAX_PAYLOAD; mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) & PCIEM_CTL_MAX_PAYLOAD; if (mps == rmps) return; /* Check whether the device is capable of the root's MPS. */ mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) & PCIEM_CAP_MAX_PAYLOAD) << 5; if (rmps > mmps) { /* * The device is unable to handle root's MPS. Limit root. * XXX: We should traverse through all the tree, applying * it to all the devices. */ pcie_adjust_config(root, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD, mmps, 2); } else { pcie_adjust_config(dev, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD, rmps, 2); } } static void pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo) { int aer; uint32_t r; uint16_t r2; if (dinfo->cfg.pcie.pcie_location != 0 && dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_ROOT_CTL, 2); r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR | PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL); pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_ROOT_CTL, r2, 2); } if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); if (r != 0 && bootverbose) { pci_printf(&dinfo->cfg, "clearing AER UC 0x%08x -> 0x%08x\n", r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4)); } r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4); r &= ~(PCIM_AER_UC_TRAINING_ERROR | PCIM_AER_UC_DL_PROTOCOL_ERROR | PCIM_AER_UC_SURPRISE_LINK_DOWN | PCIM_AER_UC_POISONED_TLP | PCIM_AER_UC_FC_PROTOCOL_ERROR | PCIM_AER_UC_COMPLETION_TIMEOUT | PCIM_AER_UC_COMPLETER_ABORT | PCIM_AER_UC_UNEXPECTED_COMPLETION | PCIM_AER_UC_RECEIVER_OVERFLOW | PCIM_AER_UC_MALFORMED_TLP | PCIM_AER_UC_ECRC_ERROR | PCIM_AER_UC_UNSUPPORTED_REQUEST | PCIM_AER_UC_ACS_VIOLATION | PCIM_AER_UC_INTERNAL_ERROR | PCIM_AER_UC_MC_BLOCKED_TLP | PCIM_AER_UC_ATOMIC_EGRESS_BLK | PCIM_AER_UC_TLP_PREFIX_BLOCKED); pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4); r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); if (r != 0 && bootverbose) { pci_printf(&dinfo->cfg, "clearing AER COR 0x%08x -> 0x%08x\n", r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4)); } r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4); r &= ~(PCIM_AER_COR_RECEIVER_ERROR | PCIM_AER_COR_BAD_TLP | PCIM_AER_COR_BAD_DLLP | PCIM_AER_COR_REPLAY_ROLLOVER | PCIM_AER_COR_REPLAY_TIMEOUT | PCIM_AER_COR_ADVISORY_NF_ERROR | PCIM_AER_COR_INTERNAL_ERROR | PCIM_AER_COR_HEADER_LOG_OVFLOW); pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4); r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, 2); r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE | PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE; pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, r, 2); } } void pci_add_child(device_t bus, struct pci_devinfo *dinfo) { device_t dev; dinfo->cfg.dev = dev = device_add_child(bus, NULL, DEVICE_UNIT_ANY); device_set_ivars(dev, dinfo); resource_list_init(&dinfo->resources); pci_cfg_save(dev, dinfo, 0); pci_cfg_restore(dev, dinfo); pci_print_verbose(dinfo); pci_add_resources(bus, dev, 0, 0); if (pci_enable_mps_tune) pcie_setup_mps(dev); pci_child_added(dinfo->cfg.dev); if (pci_clear_aer_on_attach) pci_add_child_clear_aer(dev, dinfo); EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev); } void pci_child_added_method(device_t dev, device_t child) { } static int pci_probe(device_t dev) { device_set_desc(dev, "PCI bus"); /* Allow other subclasses to override this driver. */ return (BUS_PROBE_GENERIC); } int pci_attach_common(device_t dev) { struct pci_softc *sc; int busno, domain; int rid; sc = device_get_softc(dev); domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); rid = 0; sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno, 1, 0); if (sc->sc_bus == NULL) { device_printf(dev, "failed to allocate bus number\n"); return (ENXIO); } if (bootverbose) device_printf(dev, "domain=%d, physical bus=%d\n", domain, busno); sc->sc_dma_tag = bus_get_dma_tag(dev); return (0); } int pci_attach(device_t dev) { int busno, domain, error; error = pci_attach_common(dev); if (error) return (error); /* * Since there can be multiple independently numbered PCI * buses on systems with multiple PCI domains, we can't use * the unit number to decide which bus we are probing. We ask * the parent pcib what our domain and bus numbers are. */ domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); pci_add_children(dev, domain, busno); bus_attach_children(dev); return (0); } int pci_detach(device_t dev) { struct pci_softc *sc; int error; error = bus_generic_detach(dev); if (error) return (error); sc = device_get_softc(dev); error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus); - if (error) - return (error); - return (device_delete_children(dev)); + return (error); } static void pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp) { int line, unit; const char *at; char me1[24], me2[32]; uint8_t b, s, f; uint32_t d; device_location_cache_t *cache; d = pci_get_domain(child); b = pci_get_bus(child); s = pci_get_slot(child); f = pci_get_function(child); snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f); snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f); line = 0; cache = dev_wired_cache_init(); while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) { resource_string_value(name, unit, "at", &at); if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) { *unitp = unit; break; } if (dev_wired_cache_match(cache, child, at)) { *unitp = unit; break; } } dev_wired_cache_fini(cache); } static void pci_set_power_child(device_t dev, device_t child, int state) { device_t pcib; int dstate; /* * Set the device to the given state. If the firmware suggests * a different power state, use it instead. If power management * is not present, the firmware is responsible for managing * device power. Skip children who aren't attached since they * are handled separately. */ pcib = device_get_parent(dev); dstate = state; if (device_is_attached(child) && PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0) pci_set_powerstate(child, dstate); } int pci_suspend_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list_entry *rle; int error; dinfo = device_get_ivars(child); /* * Save the PCI configuration space for the child and set the * device in the appropriate power state for this sleep state. */ pci_cfg_save(child, dinfo, 0); /* Suspend devices before potentially powering them down. */ error = bus_generic_suspend_child(dev, child); if (error) return (error); if (pci_do_power_suspend) { /* * Make sure this device's interrupt handler is not invoked * in the case the device uses a shared interrupt that can * be raised by some other device. * This is applicable only to regular (legacy) PCI interrupts * as MSI/MSI-X interrupts are never shared. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) (void)bus_suspend_intr(child, rle->res); pci_set_power_child(dev, child, PCI_POWERSTATE_D3); } return (0); } int pci_resume_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list_entry *rle; if (pci_do_power_resume) pci_set_power_child(dev, child, PCI_POWERSTATE_D0); dinfo = device_get_ivars(child); pci_cfg_restore(child, dinfo); if (!device_is_attached(child)) pci_cfg_save(child, dinfo, 1); bus_generic_resume_child(dev, child); /* * Allow interrupts only after fully resuming the driver and hardware. */ if (pci_do_power_suspend) { /* See pci_suspend_child for details. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) (void)bus_resume_intr(child, rle->res); } return (0); } int pci_resume(device_t dev) { device_t child, *devlist; int error, i, numdevs; if ((error = device_get_children(dev, &devlist, &numdevs)) != 0) return (error); /* * Resume critical devices first, then everything else later. */ for (i = 0; i < numdevs; i++) { child = devlist[i]; switch (pci_get_class(child)) { case PCIC_DISPLAY: case PCIC_MEMORY: case PCIC_BRIDGE: case PCIC_BASEPERIPH: BUS_RESUME_CHILD(dev, child); break; } } for (i = 0; i < numdevs; i++) { child = devlist[i]; switch (pci_get_class(child)) { case PCIC_DISPLAY: case PCIC_MEMORY: case PCIC_BRIDGE: case PCIC_BASEPERIPH: break; default: BUS_RESUME_CHILD(dev, child); } } free(devlist, M_TEMP); return (0); } static void pci_load_vendor_data(void) { caddr_t data; void *ptr; size_t sz; data = preload_search_by_type("pci_vendor_data"); if (data != NULL) { ptr = preload_fetch_addr(data); sz = preload_fetch_size(data); if (ptr != NULL && sz != 0) { pci_vendordata = ptr; pci_vendordata_size = sz; /* terminate the database */ pci_vendordata[pci_vendordata_size] = '\n'; } } } void pci_driver_added(device_t dev, driver_t *driver) { int numdevs; device_t *devlist; device_t child; struct pci_devinfo *dinfo; int i; if (bootverbose) device_printf(dev, "driver added\n"); DEVICE_IDENTIFY(driver, dev); if (device_get_children(dev, &devlist, &numdevs) != 0) return; for (i = 0; i < numdevs; i++) { child = devlist[i]; if (device_get_state(child) != DS_NOTPRESENT) continue; dinfo = device_get_ivars(child); pci_print_verbose(dinfo); if (bootverbose) pci_printf(&dinfo->cfg, "reprobing on driver added\n"); pci_cfg_restore(child, dinfo); if (device_probe_and_attach(child) != 0) pci_child_detached(dev, child); } free(devlist, M_TEMP); } int pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { struct pci_devinfo *dinfo; struct msix_table_entry *mte; struct msix_vector *mv; uint64_t addr; uint32_t data; void *cookie; int error, rid; error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr, arg, &cookie); if (error) return (error); /* If this is not a direct child, just bail out. */ if (device_get_parent(child) != dev) { *cookiep = cookie; return(0); } rid = rman_get_rid(irq); if (rid == 0) { /* Make sure that INTx is enabled */ pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { /* * Check to see if the interrupt is MSI or MSI-X. * Ask our parent to map the MSI and give * us the address and data register values. * If we fail for some reason, teardown the * interrupt handler. */ dinfo = device_get_ivars(child); if (dinfo->cfg.msi.msi_alloc > 0) { if (dinfo->cfg.msi.msi_addr == 0) { KASSERT(dinfo->cfg.msi.msi_handlers == 0, ("MSI has handlers, but vectors not mapped")); error = PCIB_MAP_MSI(device_get_parent(dev), child, rman_get_start(irq), &addr, &data); if (error) goto bad; dinfo->cfg.msi.msi_addr = addr; dinfo->cfg.msi.msi_data = data; } if (dinfo->cfg.msi.msi_handlers == 0) pci_enable_msi(child, dinfo->cfg.msi.msi_addr, dinfo->cfg.msi.msi_data); dinfo->cfg.msi.msi_handlers++; } else { KASSERT(dinfo->cfg.msix.msix_alloc > 0, ("No MSI or MSI-X interrupts allocated")); KASSERT(rid <= dinfo->cfg.msix.msix_table_len, ("MSI-X index too high")); mte = &dinfo->cfg.msix.msix_table[rid - 1]; KASSERT(mte->mte_vector != 0, ("no message vector")); mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1]; KASSERT(mv->mv_irq == rman_get_start(irq), ("IRQ mismatch")); if (mv->mv_address == 0) { KASSERT(mte->mte_handlers == 0, ("MSI-X table entry has handlers, but vector not mapped")); error = PCIB_MAP_MSI(device_get_parent(dev), child, rman_get_start(irq), &addr, &data); if (error) goto bad; mv->mv_address = addr; mv->mv_data = data; } /* * The MSIX table entry must be made valid by * incrementing the mte_handlers before * calling pci_enable_msix() and * pci_resume_msix(). Else the MSIX rewrite * table quirk will not work as expected. */ mte->mte_handlers++; if (mte->mte_handlers == 1) { pci_enable_msix(child, rid - 1, mv->mv_address, mv->mv_data); pci_unmask_msix(child, rid - 1); } } /* * Make sure that INTx is disabled if we are using MSI/MSI-X, * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG, * in which case we "enable" INTx so MSI/MSI-X actually works. */ if (!pci_has_quirk(pci_get_devid(child), PCI_QUIRK_MSI_INTX_BUG)) pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); else pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); bad: if (error) { (void)bus_generic_teardown_intr(dev, child, irq, cookie); return (error); } } *cookiep = cookie; return (0); } int pci_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { struct msix_table_entry *mte; struct resource_list_entry *rle; struct pci_devinfo *dinfo; int error, rid; if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE)) return (EINVAL); /* If this isn't a direct child, just bail out */ if (device_get_parent(child) != dev) return(bus_generic_teardown_intr(dev, child, irq, cookie)); rid = rman_get_rid(irq); if (rid == 0) { /* Mask INTx */ pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { /* * Check to see if the interrupt is MSI or MSI-X. If so, * decrement the appropriate handlers count and mask the * MSI-X message, or disable MSI messages if the count * drops to 0. */ dinfo = device_get_ivars(child); rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); if (rle->res != irq) return (EINVAL); if (dinfo->cfg.msi.msi_alloc > 0) { KASSERT(rid <= dinfo->cfg.msi.msi_alloc, ("MSI-X index too high")); if (dinfo->cfg.msi.msi_handlers == 0) return (EINVAL); dinfo->cfg.msi.msi_handlers--; if (dinfo->cfg.msi.msi_handlers == 0) pci_disable_msi(child); } else { KASSERT(dinfo->cfg.msix.msix_alloc > 0, ("No MSI or MSI-X interrupts allocated")); KASSERT(rid <= dinfo->cfg.msix.msix_table_len, ("MSI-X index too high")); mte = &dinfo->cfg.msix.msix_table[rid - 1]; if (mte->mte_handlers == 0) return (EINVAL); mte->mte_handlers--; if (mte->mte_handlers == 0) pci_mask_msix(child, rid - 1); } } error = bus_generic_teardown_intr(dev, child, irq, cookie); if (rid > 0) KASSERT(error == 0, ("%s: generic teardown failed for MSI/MSI-X", __func__)); return (error); } int pci_print_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list *rl; int retval = 0; dinfo = device_get_ivars(child); rl = &dinfo->resources; retval += bus_print_child_header(dev, child); retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx"); retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd"); if (device_get_flags(dev)) retval += printf(" flags %#x", device_get_flags(dev)); retval += printf(" at device %d.%d", pci_get_slot(child), pci_get_function(child)); retval += bus_print_child_domain(dev, child); retval += bus_print_child_footer(dev, child); return (retval); } static const struct { int class; int subclass; int report; /* 0 = bootverbose, 1 = always */ const char *desc; } pci_nomatch_tab[] = { {PCIC_OLD, -1, 1, "old"}, {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"}, {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"}, {PCIC_STORAGE, -1, 1, "mass storage"}, {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"}, {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"}, {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"}, {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"}, {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"}, {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"}, {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"}, {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"}, {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"}, {PCIC_NETWORK, -1, 1, "network"}, {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"}, {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"}, {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"}, {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"}, {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"}, {PCIC_DISPLAY, -1, 1, "display"}, {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"}, {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"}, {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"}, {PCIC_MULTIMEDIA, -1, 1, "multimedia"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"}, {PCIC_MEMORY, -1, 1, "memory"}, {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"}, {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"}, {PCIC_BRIDGE, -1, 1, "bridge"}, {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"}, {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"}, {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"}, {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"}, {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"}, {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"}, {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"}, {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"}, {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"}, {PCIC_SIMPLECOMM, -1, 1, "simple comms"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */ {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"}, {PCIC_BASEPERIPH, -1, 0, "base peripheral"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"}, {PCIC_INPUTDEV, -1, 1, "input device"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"}, {PCIC_DOCKING, -1, 1, "docking station"}, {PCIC_PROCESSOR, -1, 1, "processor"}, {PCIC_SERIALBUS, -1, 1, "serial bus"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"}, {PCIC_WIRELESS, -1, 1, "wireless controller"}, {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"}, {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"}, {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"}, {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"}, {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"}, {PCIC_SATCOM, -1, 1, "satellite communication"}, {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"}, {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"}, {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"}, {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"}, {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"}, {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"}, {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"}, {PCIC_DASP, -1, 0, "dasp"}, {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"}, {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"}, {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"}, {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"}, {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"}, {0, 0, 0, NULL} }; void pci_probe_nomatch(device_t dev, device_t child) { int i, report; const char *cp, *scp; char *device; /* * Look for a listing for this device in a loaded device database. */ report = 1; if ((device = pci_describe_device(child)) != NULL) { device_printf(dev, "<%s>", device); free(device, M_DEVBUF); } else { /* * Scan the class/subclass descriptions for a general * description. */ cp = "unknown"; scp = NULL; for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) { if (pci_nomatch_tab[i].class == pci_get_class(child)) { if (pci_nomatch_tab[i].subclass == -1) { cp = pci_nomatch_tab[i].desc; report = pci_nomatch_tab[i].report; } else if (pci_nomatch_tab[i].subclass == pci_get_subclass(child)) { scp = pci_nomatch_tab[i].desc; report = pci_nomatch_tab[i].report; } } } if (report || bootverbose) { device_printf(dev, "<%s%s%s>", cp ? cp : "", ((cp != NULL) && (scp != NULL)) ? ", " : "", scp ? scp : ""); } } if (report || bootverbose) { printf(" at device %d.%d (no driver attached)\n", pci_get_slot(child), pci_get_function(child)); } pci_cfg_save(child, device_get_ivars(child), 1); } void pci_child_detached(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list *rl; dinfo = device_get_ivars(child); rl = &dinfo->resources; /* * Have to deallocate IRQs before releasing any MSI messages and * have to release MSI messages before deallocating any memory * BARs. */ if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0) pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n"); if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) { if (dinfo->cfg.msi.msi_alloc != 0) pci_printf(&dinfo->cfg, "Device leaked %d MSI " "vectors\n", dinfo->cfg.msi.msi_alloc); else pci_printf(&dinfo->cfg, "Device leaked %d MSI-X " "vectors\n", dinfo->cfg.msix.msix_alloc); (void)pci_release_msi(child); } if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0) pci_printf(&dinfo->cfg, "Device leaked memory resources\n"); if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0) pci_printf(&dinfo->cfg, "Device leaked I/O resources\n"); if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0) pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n"); pci_cfg_save(child, dinfo, 1); } /* * Parse the PCI device database, if loaded, and return a pointer to a * description of the device. * * The database is flat text formatted as follows: * * Any line not in a valid format is ignored. * Lines are terminated with newline '\n' characters. * * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then * the vendor name. * * A DEVICE line is entered immediately below the corresponding VENDOR ID. * - devices cannot be listed without a corresponding VENDOR line. * A DEVICE line consists of a TAB, the 4 digit (hex) device code, * another TAB, then the device name. */ /* * Assuming (ptr) points to the beginning of a line in the database, * return the vendor or device and description of the next entry. * The value of (vendor) or (device) inappropriate for the entry type * is set to -1. Returns nonzero at the end of the database. * * Note that this is slightly unrobust in the face of corrupt data; * we attempt to safeguard against this by spamming the end of the * database with a newline when we initialise. */ static int pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc) { char *cp = *ptr; int left; *device = -1; *vendor = -1; **desc = '\0'; for (;;) { left = pci_vendordata_size - (cp - pci_vendordata); if (left <= 0) { *ptr = cp; return(1); } /* vendor entry? */ if (*cp != '\t' && sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2) break; /* device entry? */ if (*cp == '\t' && sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2) break; /* skip to next line */ while (*cp != '\n' && left > 0) { cp++; left--; } if (*cp == '\n') { cp++; left--; } } /* skip to next line */ while (*cp != '\n' && left > 0) { cp++; left--; } if (*cp == '\n' && left > 0) cp++; *ptr = cp; return(0); } static char * pci_describe_device(device_t dev) { int vendor, device; char *desc, *vp, *dp, *line; desc = vp = dp = NULL; /* * If we have no vendor data, we can't do anything. */ if (pci_vendordata == NULL) goto out; /* * Scan the vendor data looking for this device */ line = pci_vendordata; if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) goto out; for (;;) { if (pci_describe_parse_line(&line, &vendor, &device, &vp)) goto out; if (vendor == pci_get_vendor(dev)) break; } if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) goto out; for (;;) { if (pci_describe_parse_line(&line, &vendor, &device, &dp)) { *dp = 0; break; } if (vendor != -1) { *dp = 0; break; } if (device == pci_get_device(dev)) break; } if (dp[0] == '\0') snprintf(dp, 80, "0x%x", pci_get_device(dev)); if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) != NULL) sprintf(desc, "%s, %s", vp, dp); out: if (vp != NULL) free(vp, M_DEVBUF); if (dp != NULL) free(dp, M_DEVBUF); return(desc); } int pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct pci_devinfo *dinfo; pcicfgregs *cfg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; switch (which) { case PCI_IVAR_ETHADDR: /* * The generic accessor doesn't deal with failure, so * we set the return value, then return an error. */ *((uint8_t **) result) = NULL; return (EINVAL); case PCI_IVAR_SUBVENDOR: *result = cfg->subvendor; break; case PCI_IVAR_SUBDEVICE: *result = cfg->subdevice; break; case PCI_IVAR_VENDOR: *result = cfg->vendor; break; case PCI_IVAR_DEVICE: *result = cfg->device; break; case PCI_IVAR_DEVID: *result = (cfg->device << 16) | cfg->vendor; break; case PCI_IVAR_CLASS: *result = cfg->baseclass; break; case PCI_IVAR_SUBCLASS: *result = cfg->subclass; break; case PCI_IVAR_PROGIF: *result = cfg->progif; break; case PCI_IVAR_REVID: *result = cfg->revid; break; case PCI_IVAR_INTPIN: *result = cfg->intpin; break; case PCI_IVAR_IRQ: *result = cfg->intline; break; case PCI_IVAR_DOMAIN: *result = cfg->domain; break; case PCI_IVAR_BUS: *result = cfg->bus; break; case PCI_IVAR_SLOT: *result = cfg->slot; break; case PCI_IVAR_FUNCTION: *result = cfg->func; break; case PCI_IVAR_CMDREG: *result = cfg->cmdreg; break; case PCI_IVAR_CACHELNSZ: *result = cfg->cachelnsz; break; case PCI_IVAR_MINGNT: if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { *result = -1; return (EINVAL); } *result = cfg->mingnt; break; case PCI_IVAR_MAXLAT: if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { *result = -1; return (EINVAL); } *result = cfg->maxlat; break; case PCI_IVAR_LATTIMER: *result = cfg->lattimer; break; default: return (ENOENT); } return (0); } int pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(child); switch (which) { case PCI_IVAR_INTPIN: dinfo->cfg.intpin = value; return (0); case PCI_IVAR_ETHADDR: case PCI_IVAR_SUBVENDOR: case PCI_IVAR_SUBDEVICE: case PCI_IVAR_VENDOR: case PCI_IVAR_DEVICE: case PCI_IVAR_DEVID: case PCI_IVAR_CLASS: case PCI_IVAR_SUBCLASS: case PCI_IVAR_PROGIF: case PCI_IVAR_REVID: case PCI_IVAR_IRQ: case PCI_IVAR_DOMAIN: case PCI_IVAR_BUS: case PCI_IVAR_SLOT: case PCI_IVAR_FUNCTION: return (EINVAL); /* disallow for now */ default: return (ENOENT); } } #include "opt_ddb.h" #ifdef DDB #include #include /* * List resources based on pci map registers, used for within ddb */ DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE) { struct pci_devinfo *dinfo; struct devlist *devlist_head; struct pci_conf *p; const char *name; int i, error, none_count; none_count = 0; /* get the head of the device queue */ devlist_head = &pci_devq; /* * Go through the list of devices and print out devices */ for (error = 0, i = 0, dinfo = STAILQ_FIRST(devlist_head); (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit; dinfo = STAILQ_NEXT(dinfo, pci_links), i++) { /* Populate pd_name and pd_unit */ name = NULL; if (dinfo->cfg.dev) name = device_get_name(dinfo->cfg.dev); p = &dinfo->conf; db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x " "chip=0x%08x rev=0x%02x hdr=0x%02x\n", (name && *name) ? name : "none", (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) : none_count++, p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev, p->pc_sel.pc_func, (p->pc_class << 16) | (p->pc_subclass << 8) | p->pc_progif, (p->pc_subdevice << 16) | p->pc_subvendor, (p->pc_device << 16) | p->pc_vendor, p->pc_revid, p->pc_hdr); } } #endif /* DDB */ struct resource * pci_reserve_map(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int num, u_int flags) { struct pci_devinfo *dinfo = device_get_ivars(child); struct resource_list *rl = &dinfo->resources; struct resource *res; struct pci_map *pm; uint16_t cmd; pci_addr_t map, testval; int mapsize; res = NULL; /* If rid is managed by EA, ignore it */ if (pci_ea_is_enabled(child, *rid)) goto out; pm = pci_find_bar(child, *rid); if (pm != NULL) { /* This is a BAR that we failed to allocate earlier. */ mapsize = pm->pm_size; map = pm->pm_value; } else { /* * Weed out the bogons, and figure out how large the * BAR/map is. BARs that read back 0 here are bogus * and unimplemented. Note: atapci in legacy mode are * special and handled elsewhere in the code. If you * have a atapci device in legacy mode and it fails * here, that other code is broken. */ pci_read_bar(child, *rid, &map, &testval, NULL); /* * Determine the size of the BAR and ignore BARs with a size * of 0. Device ROM BARs use a different mask value. */ if (PCIR_IS_BIOS(&dinfo->cfg, *rid)) mapsize = pci_romsize(testval); else mapsize = pci_mapsize(testval); if (mapsize == 0) goto out; pm = pci_add_bar(child, *rid, map, mapsize); } if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) { if (type != SYS_RES_MEMORY) { if (bootverbose) device_printf(dev, "child %s requested type %d for rid %#x," " but the BAR says it is an memio\n", device_get_nameunit(child), type, *rid); goto out; } } else { if (type != SYS_RES_IOPORT) { if (bootverbose) device_printf(dev, "child %s requested type %d for rid %#x," " but the BAR says it is an ioport\n", device_get_nameunit(child), type, *rid); goto out; } } /* * For real BARs, we need to override the size that * the driver requests, because that's what the BAR * actually uses and we would otherwise have a * situation where we might allocate the excess to * another driver, which won't work. */ count = ((pci_addr_t)1 << mapsize) * num; if (RF_ALIGNMENT(flags) < mapsize) flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize); if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH)) flags |= RF_PREFETCHABLE; /* * Allocate enough resource, and then write back the * appropriate BAR for that resource. */ resource_list_add(rl, type, *rid, start, end, count); res = resource_list_reserve(rl, dev, child, type, rid, start, end, count, flags & ~RF_ACTIVE); if (res == NULL) { resource_list_delete(rl, type, *rid); device_printf(child, "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n", count, *rid, type, start, end); goto out; } if (bootverbose) device_printf(child, "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n", count, *rid, type, rman_get_start(res)); /* Disable decoding via the CMD register before updating the BAR */ cmd = pci_read_config(child, PCIR_COMMAND, 2); pci_write_config(child, PCIR_COMMAND, cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); map = rman_get_start(res); pci_write_bar(child, pm, map); /* Restore the original value of the CMD register */ pci_write_config(child, PCIR_COMMAND, cmd, 2); out: return (res); } struct resource * pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_long num, u_int flags) { struct pci_devinfo *dinfo; struct resource_list *rl; struct resource_list_entry *rle; struct resource *res; pcicfgregs *cfg; /* * Perform lazy resource allocation */ dinfo = device_get_ivars(child); rl = &dinfo->resources; cfg = &dinfo->cfg; switch (type) { case PCI_RES_BUS: return (pci_alloc_secbus(dev, child, rid, start, end, count, flags)); case SYS_RES_IRQ: /* * Can't alloc legacy interrupt once MSI messages have * been allocated. */ if (*rid == 0 && (cfg->msi.msi_alloc > 0 || cfg->msix.msix_alloc > 0)) return (NULL); /* * If the child device doesn't have an interrupt * routed and is deserving of an interrupt, try to * assign it one. */ if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) && (cfg->intpin != 0)) pci_assign_interrupt(dev, child, 0); break; case SYS_RES_IOPORT: case SYS_RES_MEMORY: /* * PCI-PCI bridge I/O window resources are not BARs. * For those allocations just pass the request up the * tree. */ if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) { switch (*rid) { case PCIR_IOBASEL_1: case PCIR_MEMBASE_1: case PCIR_PMBASEL_1: /* * XXX: Should we bother creating a resource * list entry? */ return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); } } /* Reserve resources for this BAR if needed. */ rle = resource_list_find(rl, type, *rid); if (rle == NULL) { res = pci_reserve_map(dev, child, type, rid, start, end, count, num, flags); if (res == NULL) return (NULL); } } return (resource_list_alloc(rl, dev, child, type, rid, start, end, count, flags)); } struct resource * pci_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { #ifdef PCI_IOV struct pci_devinfo *dinfo; #endif if (device_get_parent(child) != dev) return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child, type, rid, start, end, count, flags)); #ifdef PCI_IOV dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (type) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (NULL); case SYS_RES_MEMORY: return (pci_vf_alloc_mem_resource(dev, child, rid, start, end, count, flags)); } /* Fall through for other types of resource allocations. */ } #endif return (pci_alloc_multi_resource(dev, child, type, rid, start, end, count, 1, flags)); } int pci_release_resource(device_t dev, device_t child, struct resource *r) { struct pci_devinfo *dinfo; struct resource_list *rl; pcicfgregs *cfg __unused; if (device_get_parent(child) != dev) return (bus_generic_release_resource(dev, child, r)); dinfo = device_get_ivars(child); cfg = &dinfo->cfg; #ifdef PCI_IOV if (cfg->flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EDOOFUS); case SYS_RES_MEMORY: return (pci_vf_release_mem_resource(dev, child, r)); } /* Fall through for other types of resource allocations. */ } #endif /* * PCI-PCI bridge I/O window resources are not BARs. For * those allocations just pass the request up the tree. */ if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE && (rman_get_type(r) == SYS_RES_IOPORT || rman_get_type(r) == SYS_RES_MEMORY)) { switch (rman_get_rid(r)) { case PCIR_IOBASEL_1: case PCIR_MEMBASE_1: case PCIR_PMBASEL_1: return (bus_generic_release_resource(dev, child, r)); } } rl = &dinfo->resources; return (resource_list_release(rl, dev, child, r)); } int pci_activate_resource(device_t dev, device_t child, struct resource *r) { struct pci_devinfo *dinfo; int error, rid, type; if (device_get_parent(child) != dev) return (bus_generic_activate_resource(dev, child, r)); dinfo = device_get_ivars(child); #ifdef PCI_IOV if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: error = EINVAL; break; case SYS_RES_MEMORY: error = pci_vf_activate_mem_resource(dev, child, r); break; default: error = bus_generic_activate_resource(dev, child, r); break; } } else #endif error = bus_generic_activate_resource(dev, child, r); if (error) return (error); rid = rman_get_rid(r); type = rman_get_type(r); /* Device ROMs need their decoding explicitly enabled. */ if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) pci_write_bar(child, pci_find_bar(child, rid), rman_get_start(r) | PCIM_BIOS_ENABLE); /* Enable decoding in the command register when activating BARs. */ switch (type) { case SYS_RES_IOPORT: case SYS_RES_MEMORY: error = PCI_ENABLE_IO(dev, child, type); break; } return (error); } int pci_deactivate_resource(device_t dev, device_t child, struct resource *r) { struct pci_devinfo *dinfo; int error, rid, type; if (device_get_parent(child) != dev) return (bus_generic_deactivate_resource(dev, child, r)); dinfo = device_get_ivars(child); #ifdef PCI_IOV if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: error = EINVAL; break; case SYS_RES_MEMORY: error = pci_vf_deactivate_mem_resource(dev, child, r); break; default: error = bus_generic_deactivate_resource(dev, child, r); break; } } else #endif error = bus_generic_deactivate_resource(dev, child, r); if (error) return (error); /* Disable decoding for device ROMs. */ rid = rman_get_rid(r); type = rman_get_type(r); if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) pci_write_bar(child, pci_find_bar(child, rid), rman_get_start(r)); return (0); } int pci_adjust_resource(device_t dev, device_t child, struct resource *r, rman_res_t start, rman_res_t end) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_adjust_resource(dev, child, r, start, end)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_adjust_mem_resource(dev, child, r, start, end)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_adjust_resource(dev, child, r, start, end)); } int pci_map_resource(device_t dev, device_t child, struct resource *r, struct resource_map_request *argsp, struct resource_map *map) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_map_resource(dev, child, r, argsp, map)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_map_mem_resource(dev, child, r, argsp, map)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_map_resource(dev, child, r, argsp, map)); } int pci_unmap_resource(device_t dev, device_t child, struct resource *r, struct resource_map *map) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_unmap_resource(dev, child, r, map)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_unmap_mem_resource(dev, child, r, map)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_unmap_resource(dev, child, r, map)); } void pci_child_deleted(device_t dev, device_t child) { struct resource_list_entry *rle; struct resource_list *rl; struct pci_devinfo *dinfo; dinfo = device_get_ivars(child); rl = &dinfo->resources; EVENTHANDLER_INVOKE(pci_delete_device, child); /* Turn off access to resources we're about to free */ if (bus_child_present(child) != 0) { pci_write_config(child, PCIR_COMMAND, pci_read_config(child, PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2); pci_disable_busmaster(child); } /* Free all allocated resources */ STAILQ_FOREACH(rle, rl, link) { if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, rle->type, rle->rid)) { pci_printf(&dinfo->cfg, "Resource still owned, oops. " "(type=%d, rid=%d, addr=%lx)\n", rle->type, rle->rid, rman_get_start(rle->res)); bus_release_resource(child, rle->type, rle->rid, rle->res); } resource_list_unreserve(rl, dev, child, rle->type, rle->rid); } } resource_list_free(rl); pci_freecfg(dinfo); } void pci_delete_resource(device_t dev, device_t child, int type, int rid) { struct pci_devinfo *dinfo; struct resource_list *rl; struct resource_list_entry *rle; if (device_get_parent(child) != dev) return; dinfo = device_get_ivars(child); rl = &dinfo->resources; rle = resource_list_find(rl, type, rid); if (rle == NULL) return; if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, type, rid)) { device_printf(dev, "delete_resource: " "Resource still owned by child, oops. " "(type=%d, rid=%d, addr=%jx)\n", type, rid, rman_get_start(rle->res)); return; } resource_list_unreserve(rl, dev, child, type, rid); } resource_list_delete(rl, type, rid); } struct resource_list * pci_get_resource_list (device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); return (&dinfo->resources); } #ifdef IOMMU bus_dma_tag_t pci_get_dma_tag(device_t bus, device_t dev) { bus_dma_tag_t tag; struct pci_softc *sc; if (device_get_parent(dev) == bus) { /* try iommu and return if it works */ tag = iommu_get_dma_tag(bus, dev); } else tag = NULL; if (tag == NULL) { sc = device_get_softc(bus); tag = sc->sc_dma_tag; } return (tag); } #else bus_dma_tag_t pci_get_dma_tag(device_t bus, device_t dev) { struct pci_softc *sc = device_get_softc(bus); return (sc->sc_dma_tag); } #endif uint32_t pci_read_config_method(device_t dev, device_t child, int reg, int width) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; #ifdef PCI_IOV /* * SR-IOV VFs don't implement the VID or DID registers, so we have to * emulate them here. */ if (cfg->flags & PCICFG_VF) { if (reg == PCIR_VENDOR) { switch (width) { case 4: return (cfg->device << 16 | cfg->vendor); case 2: return (cfg->vendor); case 1: return (cfg->vendor & 0xff); default: return (0xffffffff); } } else if (reg == PCIR_DEVICE) { switch (width) { /* Note that an unaligned 4-byte read is an error. */ case 2: return (cfg->device); case 1: return (cfg->device & 0xff); default: return (0xffffffff); } } } #endif return (PCIB_READ_CONFIG(device_get_parent(dev), cfg->bus, cfg->slot, cfg->func, reg, width)); } void pci_write_config_method(device_t dev, device_t child, int reg, uint32_t val, int width) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; PCIB_WRITE_CONFIG(device_get_parent(dev), cfg->bus, cfg->slot, cfg->func, reg, val, width); } int pci_child_location_method(device_t dev, device_t child, struct sbuf *sb) { sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d", pci_get_slot(child), pci_get_function(child), pci_get_domain(child), pci_get_bus(child), pci_get_slot(child), pci_get_function(child)); return (0); } int pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb) { struct pci_devinfo *dinfo; pcicfgregs *cfg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x " "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device, cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass, cfg->progif); return (0); } int pci_get_device_path_method(device_t bus, device_t child, const char *locator, struct sbuf *sb) { device_t parent = device_get_parent(bus); int rv; if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) { rv = bus_generic_get_device_path(parent, bus, locator, sb); if (rv == 0) { sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child), pci_get_function(child)); } return (0); } return (bus_generic_get_device_path(bus, child, locator, sb)); } int pci_assign_interrupt_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child, cfg->intpin)); } static void pci_lookup(void *arg, const char *name, device_t *dev) { long val; char *end; int domain, bus, slot, func; if (*dev != NULL) return; /* * Accept pciconf-style selectors of either pciD:B:S:F or * pciB:S:F. In the latter case, the domain is assumed to * be zero. */ if (strncmp(name, "pci", 3) != 0) return; val = strtol(name + 3, &end, 10); if (val < 0 || val > INT_MAX || *end != ':') return; domain = val; val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX || *end != ':') return; bus = val; val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX) return; slot = val; if (*end == ':') { val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX || *end != '\0') return; func = val; } else if (*end == '\0') { func = slot; slot = bus; bus = domain; domain = 0; } else return; if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX || func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX)) return; *dev = pci_find_dbsf(domain, bus, slot, func); } static int pci_modevent(module_t mod, int what, void *arg) { static struct cdev *pci_cdev; static eventhandler_tag tag; switch (what) { case MOD_LOAD: STAILQ_INIT(&pci_devq); pci_generation = 0; pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644, "pci"); pci_load_vendor_data(); tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL, 1000); break; case MOD_UNLOAD: if (tag != NULL) EVENTHANDLER_DEREGISTER(dev_lookup, tag); destroy_dev(pci_cdev); break; } return (0); } static void pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo) { #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2) struct pcicfg_pcie *cfg; int version, pos; cfg = &dinfo->cfg.pcie; pos = cfg->pcie_location; version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ENDPOINT || cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl); if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ROOT_EC) WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl); if (version > 1) { WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2); WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2); WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2); } #undef WREG } static void pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo) { pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, dinfo->cfg.pcix.pcix_command, 2); } void pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo) { /* * Restore the device to full power mode. We must do this * before we restore the registers because moving from D3 to * D0 will cause the chip's BARs and some other registers to * be reset to some unknown power on reset values. Cut down * the noise on boot by doing nothing if we are already in * state D0. */ if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) pci_set_powerstate(dev, PCI_POWERSTATE_D0); pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1); pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1); pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1); pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1); pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1); pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1); switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1); pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1); break; case PCIM_HDRTYPE_BRIDGE: pci_write_config(dev, PCIR_SECLAT_1, dinfo->cfg.bridge.br_seclat, 1); pci_write_config(dev, PCIR_SUBBUS_1, dinfo->cfg.bridge.br_subbus, 1); pci_write_config(dev, PCIR_SECBUS_1, dinfo->cfg.bridge.br_secbus, 1); pci_write_config(dev, PCIR_PRIBUS_1, dinfo->cfg.bridge.br_pribus, 1); pci_write_config(dev, PCIR_BRIDGECTL_1, dinfo->cfg.bridge.br_control, 2); break; case PCIM_HDRTYPE_CARDBUS: pci_write_config(dev, PCIR_SECLAT_2, dinfo->cfg.bridge.br_seclat, 1); pci_write_config(dev, PCIR_SUBBUS_2, dinfo->cfg.bridge.br_subbus, 1); pci_write_config(dev, PCIR_SECBUS_2, dinfo->cfg.bridge.br_secbus, 1); pci_write_config(dev, PCIR_PRIBUS_2, dinfo->cfg.bridge.br_pribus, 1); pci_write_config(dev, PCIR_BRIDGECTL_2, dinfo->cfg.bridge.br_control, 2); break; } pci_restore_bars(dev); if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE) pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2); /* * Restore extended capabilities for PCI-Express and PCI-X */ if (dinfo->cfg.pcie.pcie_location != 0) pci_cfg_restore_pcie(dev, dinfo); if (dinfo->cfg.pcix.pcix_location != 0) pci_cfg_restore_pcix(dev, dinfo); /* Restore MSI and MSI-X configurations if they are present. */ if (dinfo->cfg.msi.msi_location != 0) pci_resume_msi(dev); if (dinfo->cfg.msix.msix_location != 0) pci_resume_msix(dev); #ifdef PCI_IOV if (dinfo->cfg.iov != NULL) pci_iov_cfg_restore(dev, dinfo); #endif } static void pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo) { #define RREG(n) pci_read_config(dev, pos + (n), 2) struct pcicfg_pcie *cfg; int version, pos; cfg = &dinfo->cfg.pcie; pos = cfg->pcie_location; cfg->pcie_flags = RREG(PCIER_FLAGS); version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ENDPOINT || cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL); if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ROOT_EC) cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL); if (version > 1) { cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2); cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2); cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2); } #undef RREG } static void pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo) { dinfo->cfg.pcix.pcix_command = pci_read_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2); } void pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate) { uint32_t cls; int ps; /* * Some drivers apparently write to these registers w/o updating our * cached copy. No harm happens if we update the copy, so do so here * so we can restore them. The COMMAND register is modified by the * bus w/o updating the cache. This should represent the normally * writable portion of the 'defined' part of type 0/1/2 headers. */ dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2); dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2); dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2); dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1); dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1); dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1); dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1); dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1); dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1); switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2); dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2); dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1); dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1); break; case PCIM_HDRTYPE_BRIDGE: dinfo->cfg.bridge.br_seclat = pci_read_config(dev, PCIR_SECLAT_1, 1); dinfo->cfg.bridge.br_subbus = pci_read_config(dev, PCIR_SUBBUS_1, 1); dinfo->cfg.bridge.br_secbus = pci_read_config(dev, PCIR_SECBUS_1, 1); dinfo->cfg.bridge.br_pribus = pci_read_config(dev, PCIR_PRIBUS_1, 1); dinfo->cfg.bridge.br_control = pci_read_config(dev, PCIR_BRIDGECTL_1, 2); break; case PCIM_HDRTYPE_CARDBUS: dinfo->cfg.bridge.br_seclat = pci_read_config(dev, PCIR_SECLAT_2, 1); dinfo->cfg.bridge.br_subbus = pci_read_config(dev, PCIR_SUBBUS_2, 1); dinfo->cfg.bridge.br_secbus = pci_read_config(dev, PCIR_SECBUS_2, 1); dinfo->cfg.bridge.br_pribus = pci_read_config(dev, PCIR_PRIBUS_2, 1); dinfo->cfg.bridge.br_control = pci_read_config(dev, PCIR_BRIDGECTL_2, 2); dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2); dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2); break; } if (dinfo->cfg.pcie.pcie_location != 0) pci_cfg_save_pcie(dev, dinfo); if (dinfo->cfg.pcix.pcix_location != 0) pci_cfg_save_pcix(dev, dinfo); #ifdef PCI_IOV if (dinfo->cfg.iov != NULL) pci_iov_cfg_save(dev, dinfo); #endif /* * don't set the state for display devices, base peripherals and * memory devices since bad things happen when they are powered down. * We should (a) have drivers that can easily detach and (b) use * generic drivers for these devices so that some device actually * attaches. We need to make sure that when we implement (a) we don't * power the device down on a reattach. */ cls = pci_get_class(dev); if (!setstate) return; switch (pci_do_power_nodriver) { case 0: /* NO powerdown at all */ return; case 1: /* Conservative about what to power down */ if (cls == PCIC_STORAGE) return; /*FALLTHROUGH*/ case 2: /* Aggressive about what to power down */ if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY || cls == PCIC_BASEPERIPH) return; /*FALLTHROUGH*/ case 3: /* Power down everything */ break; } /* * PCI spec says we can only go into D3 state from D0 state. * Transition from D[12] into D0 before going to D3 state. */ ps = pci_get_powerstate(dev); if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D0); if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D3); } /* Wrapper APIs suitable for device driver use. */ void pci_save_state(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); pci_cfg_save(dev, dinfo, 0); } void pci_restore_state(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); pci_cfg_restore(dev, dinfo); } static int pci_get_id_method(device_t dev, device_t child, enum pci_id_type type, uintptr_t *id) { return (PCIB_GET_ID(device_get_parent(dev), child, type, id)); } /* Find the upstream port of a given PCI device in a root complex. */ device_t pci_find_pcie_root_port(device_t dev) { struct pci_devinfo *dinfo; devclass_t pci_class; device_t pcib, bus; pci_class = devclass_find("pci"); KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class, ("%s: non-pci device %s", __func__, device_get_nameunit(dev))); /* * Walk the bridge hierarchy until we find a PCI-e root * port or a non-PCI device. */ for (;;) { bus = device_get_parent(dev); KASSERT(bus != NULL, ("%s: null parent of %s", __func__, device_get_nameunit(dev))); pcib = device_get_parent(bus); KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__, device_get_nameunit(bus))); /* * pcib's parent must be a PCI bus for this to be a * PCI-PCI bridge. */ if (device_get_devclass(device_get_parent(pcib)) != pci_class) return (NULL); dinfo = device_get_ivars(pcib); if (dinfo->cfg.pcie.pcie_location != 0 && dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) return (pcib); dev = pcib; } } /* * Wait for pending transactions to complete on a PCI-express function. * * The maximum delay is specified in milliseconds in max_delay. Note * that this function may sleep. * * Returns true if the function is idle and false if the timeout is * exceeded. If dev is not a PCI-express function, this returns true. */ bool pcie_wait_for_pending_transactions(device_t dev, u_int max_delay) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint16_t sta; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (true); sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); while (sta & PCIEM_STA_TRANSACTION_PND) { if (max_delay == 0) return (false); /* Poll once every 100 milliseconds up to the timeout. */ if (max_delay > 100) { pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK); max_delay -= 100; } else { pause_sbt("pcietp", max_delay * SBT_1MS, 0, C_HARDCLOCK); max_delay = 0; } sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); } return (true); } /* * Determine the maximum Completion Timeout in microseconds. * * For non-PCI-express functions this returns 0. */ int pcie_get_max_completion_timeout(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); /* * Functions using the 1.x spec use the default timeout range of * 50 microseconds to 50 milliseconds. Functions that do not * support programmable timeouts also use this range. */ if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 || (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) & PCIEM_CAP2_COMP_TIMO_RANGES) == 0) return (50 * 1000); switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) & PCIEM_CTL2_COMP_TIMO_VAL) { case PCIEM_CTL2_COMP_TIMO_100US: return (100); case PCIEM_CTL2_COMP_TIMO_10MS: return (10 * 1000); case PCIEM_CTL2_COMP_TIMO_55MS: return (55 * 1000); case PCIEM_CTL2_COMP_TIMO_210MS: return (210 * 1000); case PCIEM_CTL2_COMP_TIMO_900MS: return (900 * 1000); case PCIEM_CTL2_COMP_TIMO_3500MS: return (3500 * 1000); case PCIEM_CTL2_COMP_TIMO_13S: return (13 * 1000 * 1000); case PCIEM_CTL2_COMP_TIMO_64S: return (64 * 1000 * 1000); default: return (50 * 1000); } } void pcie_apei_error(device_t dev, int sev, uint8_t *aerp) { struct pci_devinfo *dinfo = device_get_ivars(dev); const char *s; int aer; uint32_t r, r1; uint16_t rs; if (sev == PCIEM_STA_CORRECTABLE_ERROR) s = "Correctable"; else if (sev == PCIEM_STA_NON_FATAL_ERROR) s = "Uncorrectable (Non-Fatal)"; else s = "Uncorrectable (Fatal)"; device_printf(dev, "%s PCIe error reported by APEI\n", s); if (aerp) { if (sev == PCIEM_STA_CORRECTABLE_ERROR) { r = le32dec(aerp + PCIR_AER_COR_STATUS); r1 = le32dec(aerp + PCIR_AER_COR_MASK); } else { r = le32dec(aerp + PCIR_AER_UC_STATUS); r1 = le32dec(aerp + PCIR_AER_UC_MASK); } device_printf(dev, "status 0x%08x mask 0x%08x", r, r1); if (sev != PCIEM_STA_CORRECTABLE_ERROR) { r = le32dec(aerp + PCIR_AER_UC_SEVERITY); rs = le16dec(aerp + PCIR_AER_CAP_CONTROL); printf(" severity 0x%08x first %d\n", r, rs & 0x1f); } else printf("\n"); } /* As kind of recovery just report and clear the error statuses. */ if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); if (r != 0) { pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); device_printf(dev, "Clearing UC AER errors 0x%08x\n", r); } r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r != 0) { pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); device_printf(dev, "Clearing COR AER errors 0x%08x\n", r); } } if (dinfo->cfg.pcie.pcie_location != 0) { rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); if ((rs & (PCIEM_STA_CORRECTABLE_ERROR | PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR | PCIEM_STA_UNSUPPORTED_REQ)) != 0) { pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, rs, 2); device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs); } } } /* * Perform a Function Level Reset (FLR) on a device. * * This function first waits for any pending transactions to complete * within the timeout specified by max_delay. If transactions are * still pending, the function will return false without attempting a * reset. * * If dev is not a PCI-express function or does not support FLR, this * function returns false. * * Note that no registers are saved or restored. The caller is * responsible for saving and restoring any registers including * PCI-standard registers via pci_save_state() and * pci_restore_state(). */ bool pcie_flr(device_t dev, u_int max_delay, bool force) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint16_t cmd, ctl; int compl_delay; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (false); if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR)) return (false); /* * Disable busmastering to prevent generation of new * transactions while waiting for the device to go idle. If * the idle timeout fails, the command register is restored * which will re-enable busmastering. */ cmd = pci_read_config(dev, PCIR_COMMAND, 2); pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2); if (!pcie_wait_for_pending_transactions(dev, max_delay)) { if (!force) { pci_write_config(dev, PCIR_COMMAND, cmd, 2); return (false); } pci_printf(&dinfo->cfg, "Resetting with transactions pending after %d ms\n", max_delay); /* * Extend the post-FLR delay to cover the maximum * Completion Timeout delay of anything in flight * during the FLR delay. Enforce a minimum delay of * at least 10ms. */ compl_delay = pcie_get_max_completion_timeout(dev) / 1000; if (compl_delay < 10) compl_delay = 10; } else compl_delay = 0; /* Initiate the reset. */ ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl | PCIEM_CTL_INITIATE_FLR, 2); /* Wait for 100ms. */ pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK); if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) & PCIEM_STA_TRANSACTION_PND) pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n"); return (true); } /* * Attempt a power-management reset by cycling the device in/out of D3 * state. PCI spec says we can only go into D3 state from D0 state. * Transition from D[12] into D0 before going to D3 state. */ int pci_power_reset(device_t dev) { int ps; ps = pci_get_powerstate(dev); if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D0); pci_set_powerstate(dev, PCI_POWERSTATE_D3); pci_set_powerstate(dev, ps); return (0); } /* * Try link drop and retrain of the downstream port of upstream * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must * cause Conventional Hot reset of the device in the slot. * Alternative, for PCIe, could be the secondary bus reset initiatied * on the upstream switch PCIR_BRIDGECTL_1, bit 6. */ int pcie_link_reset(device_t port, int pcie_location) { uint16_t v; v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2); v |= PCIEM_LINK_CTL_LINK_DIS; pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); pause_sbt("pcier1", mstosbt(20), 0, 0); v &= ~PCIEM_LINK_CTL_LINK_DIS; v |= PCIEM_LINK_CTL_RETRAIN_LINK; pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */ v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2); return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0); } static int pci_reset_post(device_t dev, device_t child) { if (dev == device_get_parent(child)) pci_restore_state(child); return (0); } static int pci_reset_prepare(device_t dev, device_t child) { if (dev == device_get_parent(child)) pci_save_state(child); return (0); } static int pci_reset_child(device_t dev, device_t child, int flags) { int error; if (dev == NULL || device_get_parent(child) != dev) return (0); if ((flags & DEVF_RESET_DETACH) != 0) { error = device_get_state(child) == DS_ATTACHED ? device_detach(child) : 0; } else { error = BUS_SUSPEND_CHILD(dev, child); } if (error == 0) { if (!pcie_flr(child, 1000, false)) { error = BUS_RESET_PREPARE(dev, child); if (error == 0) pci_power_reset(child); BUS_RESET_POST(dev, child); } if ((flags & DEVF_RESET_DETACH) != 0) device_probe_and_attach(child); else BUS_RESUME_CHILD(dev, child); } return (error); } const struct pci_device_table * pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt) { bool match; uint16_t vendor, device, subvendor, subdevice, class, subclass, revid; vendor = pci_get_vendor(child); device = pci_get_device(child); subvendor = pci_get_subvendor(child); subdevice = pci_get_subdevice(child); class = pci_get_class(child); subclass = pci_get_subclass(child); revid = pci_get_revid(child); while (nelt-- > 0) { match = true; if (id->match_flag_vendor) match &= vendor == id->vendor; if (id->match_flag_device) match &= device == id->device; if (id->match_flag_subvendor) match &= subvendor == id->subvendor; if (id->match_flag_subdevice) match &= subdevice == id->subdevice; if (id->match_flag_class) match &= class == id->class_id; if (id->match_flag_subclass) match &= subclass == id->subclass; if (id->match_flag_revid) match &= revid == id->revid; if (match) return (id); id++; } return (NULL); } static void pci_print_faulted_dev_name(const struct pci_devinfo *dinfo) { const char *dev_name; device_t dev; dev = dinfo->cfg.dev; printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus, dinfo->cfg.slot, dinfo->cfg.func); dev_name = device_get_name(dev); if (dev_name != NULL) printf(" (%s%d)", dev_name, device_get_unit(dev)); } void pci_print_faulted_dev(void) { struct pci_devinfo *dinfo; device_t dev; int aer, i; uint32_t r1, r2; uint16_t status; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { dev = dinfo->cfg.dev; status = pci_read_config(dev, PCIR_STATUS, 2); status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | PCIM_STATUS_SERR | PCIM_STATUS_PERR; if (status != 0) { pci_print_faulted_dev_name(dinfo); printf(" error 0x%04x\n", status); } if (dinfo->cfg.pcie.pcie_location != 0) { status = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); if ((status & (PCIEM_STA_CORRECTABLE_ERROR | PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR | PCIEM_STA_UNSUPPORTED_REQ)) != 0) { pci_print_faulted_dev_name(dinfo); printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n", pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, 2), status); } } if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r1 != 0 || r2 != 0) { pci_print_faulted_dev_name(dinfo); printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n" " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n", r1, pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4), pci_read_config(dev, aer + PCIR_AER_UC_SEVERITY, 4), r2, pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4), pci_read_config(dev, aer + PCIR_AER_CAP_CONTROL, 4)); for (i = 0; i < 4; i++) { r1 = pci_read_config(dev, aer + PCIR_AER_HEADER_LOG + i * 4, 4); printf(" HL%d: 0x%08x\n", i, r1); } } } } } #ifdef DDB DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE) { pci_print_faulted_dev(); } static void db_clear_pcie_errors(const struct pci_devinfo *dinfo) { device_t dev; int aer; uint32_t r; dev = dinfo->cfg.dev; r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, r, 2); if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0) return; r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); if (r != 0) pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r != 0) pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); } DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE) { struct pci_devinfo *dinfo; device_t dev; uint16_t status, status1; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { dev = dinfo->cfg.dev; status1 = status = pci_read_config(dev, PCIR_STATUS, 2); status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | PCIM_STATUS_SERR | PCIM_STATUS_PERR; if (status1 != 0) { status &= ~status1; pci_write_config(dev, PCIR_STATUS, status, 2); } if (dinfo->cfg.pcie.pcie_location != 0) db_clear_pcie_errors(dinfo); } } #endif diff --git a/sys/dev/superio/superio.c b/sys/dev/superio/superio.c index 7bed7bb510d3..94e97f197a93 100644 --- a/sys/dev/superio/superio.c +++ b/sys/dev/superio/superio.c @@ -1,1121 +1,1120 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andriy Gapon * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "isa_if.h" typedef void (*sio_conf_enter_f)(struct resource*, uint16_t); typedef void (*sio_conf_exit_f)(struct resource*, uint16_t); struct sio_conf_methods { sio_conf_enter_f enter; sio_conf_exit_f exit; superio_vendor_t vendor; }; struct sio_device { uint8_t ldn; superio_dev_type_t type; }; struct superio_devinfo { STAILQ_ENTRY(superio_devinfo) link; struct resource_list resources; device_t dev; uint8_t ldn; superio_dev_type_t type; uint16_t iobase; uint16_t iobase2; uint8_t irq; uint8_t dma; }; struct siosc { struct mtx conf_lock; STAILQ_HEAD(, superio_devinfo) devlist; struct resource* io_res; struct cdev *chardev; int io_rid; uint16_t io_port; const struct sio_conf_methods *methods; const struct sio_device *known_devices; superio_vendor_t vendor; uint16_t devid; uint8_t revid; int extid; uint8_t current_ldn; uint8_t ldn_reg; uint8_t enable_reg; }; static d_ioctl_t superio_ioctl; static struct cdevsw superio_cdevsw = { .d_version = D_VERSION, .d_ioctl = superio_ioctl, .d_name = "superio", }; #define NUMPORTS 2 static uint8_t sio_read(struct resource* res, uint8_t reg) { bus_write_1(res, 0, reg); return (bus_read_1(res, 1)); } /* Read a word from two one-byte registers, big endian. */ static uint16_t sio_readw(struct resource* res, uint8_t reg) { uint16_t v; v = sio_read(res, reg); v <<= 8; v |= sio_read(res, reg + 1); return (v); } static void sio_write(struct resource* res, uint8_t reg, uint8_t val) { bus_write_1(res, 0, reg); bus_write_1(res, 1, val); } static void sio_ldn_select(struct siosc *sc, uint8_t ldn) { mtx_assert(&sc->conf_lock, MA_OWNED); if (ldn == sc->current_ldn) return; sio_write(sc->io_res, sc->ldn_reg, ldn); sc->current_ldn = ldn; } static uint8_t sio_ldn_read(struct siosc *sc, uint8_t ldn, uint8_t reg) { mtx_assert(&sc->conf_lock, MA_OWNED); if (reg >= sc->enable_reg) { sio_ldn_select(sc, ldn); KASSERT(sc->current_ldn == ldn, ("sio_ldn_select failed")); } return (sio_read(sc->io_res, reg)); } static uint16_t sio_ldn_readw(struct siosc *sc, uint8_t ldn, uint8_t reg) { mtx_assert(&sc->conf_lock, MA_OWNED); if (reg >= sc->enable_reg) { sio_ldn_select(sc, ldn); KASSERT(sc->current_ldn == ldn, ("sio_ldn_select failed")); } return (sio_readw(sc->io_res, reg)); } static void sio_ldn_write(struct siosc *sc, uint8_t ldn, uint8_t reg, uint8_t val) { mtx_assert(&sc->conf_lock, MA_OWNED); if (reg <= sc->ldn_reg) { printf("ignored attempt to write special register 0x%x\n", reg); return; } sio_ldn_select(sc, ldn); KASSERT(sc->current_ldn == ldn, ("sio_ldn_select failed")); sio_write(sc->io_res, reg, val); } static void sio_conf_enter(struct siosc *sc) { mtx_lock(&sc->conf_lock); sc->methods->enter(sc->io_res, sc->io_port); } static void sio_conf_exit(struct siosc *sc) { sc->methods->exit(sc->io_res, sc->io_port); sc->current_ldn = 0xff; mtx_unlock(&sc->conf_lock); } static void ite_conf_enter(struct resource* res, uint16_t port) { bus_write_1(res, 0, 0x87); bus_write_1(res, 0, 0x01); bus_write_1(res, 0, 0x55); bus_write_1(res, 0, port == 0x2e ? 0x55 : 0xaa); } static void ite_conf_exit(struct resource* res, uint16_t port) { sio_write(res, 0x02, 0x02); } static const struct sio_conf_methods ite_conf_methods = { .enter = ite_conf_enter, .exit = ite_conf_exit, .vendor = SUPERIO_VENDOR_ITE }; static void nvt_conf_enter(struct resource* res, uint16_t port) { bus_write_1(res, 0, 0x87); bus_write_1(res, 0, 0x87); } static void nvt_conf_exit(struct resource* res, uint16_t port) { bus_write_1(res, 0, 0xaa); } static const struct sio_conf_methods nvt_conf_methods = { .enter = nvt_conf_enter, .exit = nvt_conf_exit, .vendor = SUPERIO_VENDOR_NUVOTON }; static void fintek_conf_enter(struct resource* res, uint16_t port) { bus_write_1(res, 0, 0x87); bus_write_1(res, 0, 0x87); } static void fintek_conf_exit(struct resource* res, uint16_t port) { bus_write_1(res, 0, 0xaa); } static const struct sio_conf_methods fintek_conf_methods = { .enter = fintek_conf_enter, .exit = fintek_conf_exit, .vendor = SUPERIO_VENDOR_FINTEK }; static const struct sio_conf_methods * const methods_table[] = { &ite_conf_methods, &nvt_conf_methods, &fintek_conf_methods, NULL }; static const uint16_t ports_table[] = { 0x2e, 0x4e, 0 }; const struct sio_device ite_devices[] = { { .ldn = 4, .type = SUPERIO_DEV_HWM }, { .ldn = 7, .type = SUPERIO_DEV_WDT }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device w83627_devices[] = { { .ldn = 8, .type = SUPERIO_DEV_WDT }, { .ldn = 9, .type = SUPERIO_DEV_GPIO }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device nvt_devices[] = { { .ldn = 8, .type = SUPERIO_DEV_WDT }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device nct5104_devices[] = { { .ldn = 7, .type = SUPERIO_DEV_GPIO }, { .ldn = 8, .type = SUPERIO_DEV_WDT }, { .ldn = 15, .type = SUPERIO_DEV_GPIO }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device nct5585_devices[] = { { .ldn = 9, .type = SUPERIO_DEV_GPIO }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device nct611x_devices[] = { { .ldn = 0x7, .type = SUPERIO_DEV_GPIO }, { .ldn = 0x8, .type = SUPERIO_DEV_WDT }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device nct67xx_devices[] = { { .ldn = 0x8, .type = SUPERIO_DEV_WDT }, { .ldn = 0x9, .type = SUPERIO_DEV_GPIO }, { .ldn = 0xb, .type = SUPERIO_DEV_HWM }, { .type = SUPERIO_DEV_NONE }, }; const struct sio_device fintek_devices[] = { { .ldn = 6, .type = SUPERIO_DEV_GPIO }, { .ldn = 7, .type = SUPERIO_DEV_WDT }, { .type = SUPERIO_DEV_NONE }, }; static const struct { superio_vendor_t vendor; uint16_t devid; uint16_t mask; int extid; /* Extra ID: used to handle conflicting devid. */ const char *descr; const struct sio_device *devices; } superio_table[] = { { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8613, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8712, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8716, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8718, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8720, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8721, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8726, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8728, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_ITE, .devid = 0x8771, .devices = ite_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x1061, .mask = 0x00, .descr = "Nuvoton NCT5104D/NCT6102D/NCT6106D (rev. A)", .devices = nct5104_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x5200, .mask = 0xff, .descr = "Winbond 83627HF/F/HG/G", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x5900, .mask = 0xff, .descr = "Winbond 83627S", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x6000, .mask = 0xff, .descr = "Winbond 83697HF", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x6800, .mask = 0xff, .descr = "Winbond 83697UG", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x7000, .mask = 0xff, .descr = "Winbond 83637HF", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x8200, .mask = 0xff, .descr = "Winbond 83627THF", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x8500, .mask = 0xff, .descr = "Winbond 83687THF", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0x8800, .mask = 0xff, .descr = "Winbond 83627EHF", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xa000, .mask = 0xff, .descr = "Winbond 83627DHG", .devices = w83627_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xa200, .mask = 0xff, .descr = "Winbond 83627UHG", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xa500, .mask = 0xff, .descr = "Winbond 83667HG", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xb000, .mask = 0xff, .descr = "Winbond 83627DHG-P", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xb300, .mask = 0xff, .descr = "Winbond 83667HG-B", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xb400, .mask = 0xff, .descr = "Nuvoton NCT6775", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xc300, .mask = 0xff, .descr = "Nuvoton NCT6776", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xc400, .mask = 0xff, .descr = "Nuvoton NCT5104D/NCT6102D/NCT6106D (rev. B+)", .devices = nct5104_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xc500, .mask = 0xff, .descr = "Nuvoton NCT6779D", .devices = nct67xx_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xd42a, .extid = 1, .descr = "Nuvoton NCT6796D-E", .devices = nct67xx_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xd42a, .extid = 2, .descr = "Nuvoton NCT5585D", .devices = nct5585_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xc800, .mask = 0xff, .descr = "Nuvoton NCT6791", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xc900, .mask = 0xff, .descr = "Nuvoton NCT6792", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xd100, .mask = 0xff, .descr = "Nuvoton NCT6793", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xd200, .mask = 0xff, .descr = "Nuvoton NCT6112D/NCT6114D/NCT6116D", .devices = nct611x_devices, }, { .vendor = SUPERIO_VENDOR_NUVOTON, .devid = 0xd300, .mask = 0xff, .descr = "Nuvoton NCT6795", .devices = nvt_devices, }, { .vendor = SUPERIO_VENDOR_FINTEK, .devid = 0x1210, .mask = 0xff, .descr = "Fintek F81803", .devices = fintek_devices, }, { .vendor = SUPERIO_VENDOR_FINTEK, .devid = 0x0704, .descr = "Fintek F81865", .devices = fintek_devices, }, { 0, 0 } }; static const char * devtype_to_str(superio_dev_type_t type) { switch (type) { case SUPERIO_DEV_NONE: return ("none"); case SUPERIO_DEV_HWM: return ("HWM"); case SUPERIO_DEV_WDT: return ("WDT"); case SUPERIO_DEV_GPIO: return ("GPIO"); case SUPERIO_DEV_MAX: return ("invalid"); } return ("invalid"); } static int superio_detect(device_t dev, bool claim, struct siosc *sc) { struct resource *res; rman_res_t port; rman_res_t count; uint16_t devid; uint8_t revid; int error; int rid; int i, m; int prefer; error = bus_get_resource(dev, SYS_RES_IOPORT, 0, &port, &count); if (error != 0) return (error); if (port > UINT16_MAX || count < NUMPORTS) { device_printf(dev, "unexpected I/O range size\n"); return (ENXIO); } /* * Make a temporary resource reservation for hardware probing. * If we can't get the resources we need then * we need to abort. Possibly this indicates * the resources were used by another device * in which case the probe would have failed anyhow. */ rid = 0; res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, RF_ACTIVE); if (res == NULL) { if (claim) device_printf(dev, "failed to allocate I/O resource\n"); return (ENXIO); } prefer = 0; resource_int_value(device_get_name(dev), device_get_unit(dev), "prefer", &prefer); if (bootverbose && prefer > 0) device_printf(dev, "prefer extid %d\n", prefer); for (m = 0; methods_table[m] != NULL; m++) { methods_table[m]->enter(res, port); if (methods_table[m]->vendor == SUPERIO_VENDOR_ITE) { devid = sio_readw(res, 0x20); revid = sio_read(res, 0x22); } else if (methods_table[m]->vendor == SUPERIO_VENDOR_NUVOTON) { devid = sio_read(res, 0x20); revid = sio_read(res, 0x21); devid = (devid << 8) | revid; } else if (methods_table[m]->vendor == SUPERIO_VENDOR_FINTEK) { devid = sio_read(res, 0x20); revid = sio_read(res, 0x21); devid = (devid << 8) | revid; } else { continue; } methods_table[m]->exit(res, port); for (i = 0; superio_table[i].vendor != 0; i++) { uint16_t mask; mask = superio_table[i].mask; if (superio_table[i].vendor != methods_table[m]->vendor) continue; if ((superio_table[i].devid & ~mask) != (devid & ~mask)) continue; if (prefer > 0 && prefer != superio_table[i].extid) continue; break; } /* Found a matching SuperIO entry. */ if (superio_table[i].vendor != 0) break; } if (methods_table[m] == NULL) error = ENXIO; else error = 0; if (!claim || error != 0) { bus_release_resource(dev, SYS_RES_IOPORT, rid, res); return (error); } sc->methods = methods_table[m]; sc->vendor = sc->methods->vendor; sc->known_devices = superio_table[i].devices; sc->io_res = res; sc->io_rid = rid; sc->io_port = port; sc->devid = devid; sc->revid = revid; sc->extid = superio_table[i].extid; KASSERT(sc->vendor == SUPERIO_VENDOR_ITE || sc->vendor == SUPERIO_VENDOR_NUVOTON || sc->vendor == SUPERIO_VENDOR_FINTEK, ("Only ITE, Nuvoton and Fintek SuperIO-s are supported")); sc->ldn_reg = 0x07; sc->enable_reg = 0x30; /* FIXME enable_reg not used by nctgpio(4). */ sc->current_ldn = 0xff; /* no device should have this */ if (superio_table[i].descr != NULL) { device_set_desc(dev, superio_table[i].descr); } else if (sc->vendor == SUPERIO_VENDOR_ITE) { device_set_descf(dev, "ITE IT%4x SuperIO (revision 0x%02x)", sc->devid, sc->revid); } return (0); } static void superio_identify(driver_t *driver, device_t parent) { device_t child; int i; /* * Don't create child devices if any already exist. * Those could be created via isa hints or if this * driver is loaded, unloaded and then loaded again. */ if (device_find_child(parent, "superio", -1)) { if (bootverbose) printf("superio: device(s) already created\n"); return; } /* * Create a child for each candidate port. * It would be nice if we could somehow clean up those * that this driver fails to probe. */ for (i = 0; ports_table[i] != 0; i++) { child = BUS_ADD_CHILD(parent, ISA_ORDER_SPECULATIVE, "superio", -1); if (child == NULL) { device_printf(parent, "failed to add superio child\n"); continue; } bus_set_resource(child, SYS_RES_IOPORT, 0, ports_table[i], 2); if (superio_detect(child, false, NULL) != 0) device_delete_child(parent, child); } } static int superio_probe(device_t dev) { struct siosc *sc; int error; /* Make sure we do not claim some ISA PNP device. */ if (isa_get_logicalid(dev) != 0) return (ENXIO); /* * XXX We can populate the softc now only because we return * BUS_PROBE_SPECIFIC */ sc = device_get_softc(dev); error = superio_detect(dev, true, sc); if (error != 0) return (error); return (BUS_PROBE_SPECIFIC); } static void superio_add_known_child(device_t dev, superio_dev_type_t type, uint8_t ldn) { struct siosc *sc = device_get_softc(dev); struct superio_devinfo *dinfo; device_t child; child = BUS_ADD_CHILD(dev, 0, NULL, DEVICE_UNIT_ANY); if (child == NULL) { device_printf(dev, "failed to add child for ldn %d, type %s\n", ldn, devtype_to_str(type)); return; } dinfo = device_get_ivars(child); dinfo->ldn = ldn; dinfo->type = type; sio_conf_enter(sc); dinfo->iobase = sio_ldn_readw(sc, ldn, 0x60); dinfo->iobase2 = sio_ldn_readw(sc, ldn, 0x62); dinfo->irq = sio_ldn_readw(sc, ldn, 0x70); dinfo->dma = sio_ldn_readw(sc, ldn, 0x74); sio_conf_exit(sc); STAILQ_INSERT_TAIL(&sc->devlist, dinfo, link); } static int superio_attach(device_t dev) { struct siosc *sc = device_get_softc(dev); int i; mtx_init(&sc->conf_lock, device_get_nameunit(dev), "superio", MTX_DEF); STAILQ_INIT(&sc->devlist); for (i = 0; sc->known_devices[i].type != SUPERIO_DEV_NONE; i++) { superio_add_known_child(dev, sc->known_devices[i].type, sc->known_devices[i].ldn); } bus_identify_children(dev); bus_attach_children(dev); sc->chardev = make_dev(&superio_cdevsw, device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "superio%d", device_get_unit(dev)); if (sc->chardev == NULL) device_printf(dev, "failed to create character device\n"); else sc->chardev->si_drv1 = sc; return (0); } static int superio_detach(device_t dev) { struct siosc *sc = device_get_softc(dev); int error; error = bus_generic_detach(dev); if (error != 0) return (error); if (sc->chardev != NULL) destroy_dev(sc->chardev); - device_delete_children(dev); bus_release_resource(dev, SYS_RES_IOPORT, sc->io_rid, sc->io_res); mtx_destroy(&sc->conf_lock); return (0); } static device_t superio_add_child(device_t dev, u_int order, const char *name, int unit) { struct superio_devinfo *dinfo; device_t child; child = device_add_child_ordered(dev, order, name, unit); if (child == NULL) return (NULL); dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (dinfo == NULL) { device_delete_child(dev, child); return (NULL); } dinfo->ldn = 0xff; dinfo->type = SUPERIO_DEV_NONE; dinfo->dev = child; resource_list_init(&dinfo->resources); device_set_ivars(child, dinfo); return (child); } static void superio_child_deleted(device_t dev, device_t child) { struct superio_devinfo *dinfo; dinfo = device_get_ivars(child); if (dinfo == NULL) return; resource_list_free(&dinfo->resources); free(dinfo, M_DEVBUF); } static int superio_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct superio_devinfo *dinfo; dinfo = device_get_ivars(child); switch (which) { case SUPERIO_IVAR_LDN: *result = dinfo->ldn; break; case SUPERIO_IVAR_TYPE: *result = dinfo->type; break; case SUPERIO_IVAR_IOBASE: *result = dinfo->iobase; break; case SUPERIO_IVAR_IOBASE2: *result = dinfo->iobase2; break; case SUPERIO_IVAR_IRQ: *result = dinfo->irq; break; case SUPERIO_IVAR_DMA: *result = dinfo->dma; break; default: return (ENOENT); } return (0); } static int superio_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { switch (which) { case SUPERIO_IVAR_LDN: case SUPERIO_IVAR_TYPE: case SUPERIO_IVAR_IOBASE: case SUPERIO_IVAR_IOBASE2: case SUPERIO_IVAR_IRQ: case SUPERIO_IVAR_DMA: return (EINVAL); default: return (ENOENT); } } static struct resource_list * superio_get_resource_list(device_t dev, device_t child) { struct superio_devinfo *dinfo = device_get_ivars(child); return (&dinfo->resources); } static int superio_printf(struct superio_devinfo *dinfo, const char *fmt, ...) { va_list ap; int retval; retval = printf("superio:%s@ldn%0x2x: ", devtype_to_str(dinfo->type), dinfo->ldn); va_start(ap, fmt); retval += vprintf(fmt, ap); va_end(ap); return (retval); } static void superio_child_detached(device_t dev, device_t child) { struct superio_devinfo *dinfo; struct resource_list *rl; dinfo = device_get_ivars(child); rl = &dinfo->resources; if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0) superio_printf(dinfo, "Device leaked IRQ resources\n"); if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0) superio_printf(dinfo, "Device leaked memory resources\n"); if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0) superio_printf(dinfo, "Device leaked I/O resources\n"); } static int superio_child_location(device_t parent, device_t child, struct sbuf *sb) { uint8_t ldn; ldn = superio_get_ldn(child); sbuf_printf(sb, "ldn=0x%02x", ldn); return (0); } static int superio_child_pnp(device_t parent, device_t child, struct sbuf *sb) { superio_dev_type_t type; type = superio_get_type(child); sbuf_printf(sb, "type=%s", devtype_to_str(type)); return (0); } static int superio_print_child(device_t parent, device_t child) { superio_dev_type_t type; uint8_t ldn; int retval; ldn = superio_get_ldn(child); type = superio_get_type(child); retval = bus_print_child_header(parent, child); retval += printf(" at %s ldn 0x%02x", devtype_to_str(type), ldn); retval += bus_print_child_footer(parent, child); return (retval); } superio_vendor_t superio_vendor(device_t dev) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); return (sc->vendor); } uint16_t superio_devid(device_t dev) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); return (sc->devid); } uint8_t superio_revid(device_t dev) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); return (sc->revid); } int superio_extid(device_t dev) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); return (sc->extid); } uint8_t superio_ldn_read(device_t dev, uint8_t ldn, uint8_t reg) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); uint8_t v; sio_conf_enter(sc); v = sio_ldn_read(sc, ldn, reg); sio_conf_exit(sc); return (v); } uint8_t superio_read(device_t dev, uint8_t reg) { struct superio_devinfo *dinfo = device_get_ivars(dev); return (superio_ldn_read(dev, dinfo->ldn, reg)); } void superio_ldn_write(device_t dev, uint8_t ldn, uint8_t reg, uint8_t val) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); sio_conf_enter(sc); sio_ldn_write(sc, ldn, reg, val); sio_conf_exit(sc); } void superio_write(device_t dev, uint8_t reg, uint8_t val) { struct superio_devinfo *dinfo = device_get_ivars(dev); return (superio_ldn_write(dev, dinfo->ldn, reg, val)); } bool superio_dev_enabled(device_t dev, uint8_t mask) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); struct superio_devinfo *dinfo = device_get_ivars(dev); uint8_t v; /* GPIO device is always active in ITE chips. */ if (sc->vendor == SUPERIO_VENDOR_ITE && dinfo->ldn == 7) return (true); v = superio_read(dev, sc->enable_reg); /* FIXME enable_reg not used by nctgpio(4). */ return ((v & mask) != 0); } void superio_dev_enable(device_t dev, uint8_t mask) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); struct superio_devinfo *dinfo = device_get_ivars(dev); uint8_t v; /* GPIO device is always active in ITE chips. */ if (sc->vendor == SUPERIO_VENDOR_ITE && dinfo->ldn == 7) return; sio_conf_enter(sc); v = sio_ldn_read(sc, dinfo->ldn, sc->enable_reg); v |= mask; sio_ldn_write(sc, dinfo->ldn, sc->enable_reg, v); sio_conf_exit(sc); } void superio_dev_disable(device_t dev, uint8_t mask) { device_t sio_dev = device_get_parent(dev); struct siosc *sc = device_get_softc(sio_dev); struct superio_devinfo *dinfo = device_get_ivars(dev); uint8_t v; /* GPIO device is always active in ITE chips. */ if (sc->vendor == SUPERIO_VENDOR_ITE && dinfo->ldn == 7) return; sio_conf_enter(sc); v = sio_ldn_read(sc, dinfo->ldn, sc->enable_reg); v &= ~mask; sio_ldn_write(sc, dinfo->ldn, sc->enable_reg, v); sio_conf_exit(sc); } device_t superio_find_dev(device_t superio, superio_dev_type_t type, int ldn) { struct siosc *sc = device_get_softc(superio); struct superio_devinfo *dinfo; if (ldn < -1 || ldn > UINT8_MAX) return (NULL); /* ERANGE */ if (type == SUPERIO_DEV_NONE && ldn == -1) return (NULL); /* EINVAL */ STAILQ_FOREACH(dinfo, &sc->devlist, link) { if (ldn != -1 && dinfo->ldn != ldn) continue; if (type != SUPERIO_DEV_NONE && dinfo->type != type) continue; return (dinfo->dev); } return (NULL); } static int superio_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flags, struct thread *td) { struct siosc *sc; struct superiocmd *s; sc = dev->si_drv1; s = (struct superiocmd *)data; switch (cmd) { case SUPERIO_CR_READ: sio_conf_enter(sc); s->val = sio_ldn_read(sc, s->ldn, s->cr); sio_conf_exit(sc); return (0); case SUPERIO_CR_WRITE: sio_conf_enter(sc); sio_ldn_write(sc, s->ldn, s->cr, s->val); sio_conf_exit(sc); return (0); default: return (ENOTTY); } } static device_method_t superio_methods[] = { DEVMETHOD(device_identify, superio_identify), DEVMETHOD(device_probe, superio_probe), DEVMETHOD(device_attach, superio_attach), DEVMETHOD(device_detach, superio_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), DEVMETHOD(bus_add_child, superio_add_child), DEVMETHOD(bus_child_deleted, superio_child_deleted), DEVMETHOD(bus_child_detached, superio_child_detached), DEVMETHOD(bus_child_location, superio_child_location), DEVMETHOD(bus_child_pnpinfo, superio_child_pnp), DEVMETHOD(bus_print_child, superio_print_child), DEVMETHOD(bus_read_ivar, superio_read_ivar), DEVMETHOD(bus_write_ivar, superio_write_ivar), DEVMETHOD(bus_get_resource_list, superio_get_resource_list), DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_delete_resource, bus_generic_rl_delete_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD_END }; static driver_t superio_driver = { "superio", superio_methods, sizeof(struct siosc) }; DRIVER_MODULE(superio, isa, superio_driver, 0, 0); MODULE_VERSION(superio, 1); diff --git a/sys/dev/usb/misc/cp2112.c b/sys/dev/usb/misc/cp2112.c index e3587241f412..d4776ca342cb 100644 --- a/sys/dev/usb/misc/cp2112.c +++ b/sys/dev/usb/misc/cp2112.c @@ -1,1434 +1,1433 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) Andriy Gapon * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Hardware information links: * - CP2112 Datasheet * https://www.silabs.com/documents/public/data-sheets/cp2112-datasheet.pdf * - AN495: CP2112 Interface Specification * https://www.silabs.com/documents/public/application-notes/an495-cp2112-interface-specification.pdf * - CP2112 Errata * https://www.silabs.com/documents/public/errata/cp2112-errata.pdf */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" #include #include #include #include #include "usbdevs.h" #define USB_DEBUG_VAR usb_debug #include #define SIZEOF_FIELD(_s, _f) sizeof(((struct _s *)NULL)->_f) #define CP2112GPIO_LOCK(sc) sx_xlock(&sc->gpio_lock) #define CP2112GPIO_UNLOCK(sc) sx_xunlock(&sc->gpio_lock) #define CP2112GPIO_LOCKED(sc) sx_assert(&sc->gpio_lock, SX_XLOCKED) #define CP2112_PART_NUM 0x0c #define CP2112_GPIO_COUNT 8 #define CP2112_REPORT_SIZE 64 #define CP2112_REQ_RESET 0x1 #define CP2112_REQ_GPIO_CFG 0x2 #define CP2112_REQ_GPIO_GET 0x3 #define CP2112_REQ_GPIO_SET 0x4 #define CP2112_REQ_VERSION 0x5 #define CP2112_REQ_SMB_CFG 0x6 #define CP2112_REQ_SMB_READ 0x10 #define CP2112_REQ_SMB_WRITE_READ 0x11 #define CP2112_REQ_SMB_READ_FORCE_SEND 0x12 #define CP2112_REQ_SMB_READ_RESPONSE 0x13 #define CP2112_REQ_SMB_WRITE 0x14 #define CP2112_REQ_SMB_XFER_STATUS_REQ 0x15 #define CP2112_REQ_SMB_XFER_STATUS_RESP 0x16 #define CP2112_REQ_SMB_CANCEL 0x17 #define CP2112_REQ_LOCK 0x20 #define CP2112_REQ_USB_CFG 0x21 #define CP2112_IIC_MAX_READ_LEN 512 #define CP2112_IIC_REPSTART_VER 2 /* Erratum CP2112_E10. */ #define CP2112_GPIO_SPEC_CLK7 1 /* Pin 7 is clock output. */ #define CP2112_GPIO_SPEC_TX0 2 /* Pin 0 pulses on USB TX. */ #define CP2112_GPIO_SPEC_RX1 4 /* Pin 1 pulses on USB RX. */ #define CP2112_IIC_STATUS0_IDLE 0 #define CP2112_IIC_STATUS0_BUSY 1 #define CP2112_IIC_STATUS0_CMP 2 #define CP2112_IIC_STATUS0_ERROR 3 #define CP2112_IIC_STATUS1_TIMEOUT_NACK 0 #define CP2112_IIC_STATUS1_TIMEOUT_BUS 1 #define CP2112_IIC_STATUS1_ARB_LOST 2 /* CP2112_REQ_VERSION */ struct version_request { uint8_t id; uint8_t part_num; uint8_t version; } __packed; /* CP2112_REQ_GPIO_GET */ struct gpio_get_req { uint8_t id; uint8_t state; } __packed; /* CP2112_REQ_GPIO_SET */ struct gpio_set_req { uint8_t id; uint8_t state; uint8_t mask; } __packed; /* CP2112_REQ_GPIO_CFG */ struct gpio_config_req { uint8_t id; uint8_t output; uint8_t pushpull; uint8_t special; uint8_t divider; } __packed; /* CP2112_REQ_SMB_XFER_STATUS_REQ */ struct i2c_xfer_status_req { uint8_t id; uint8_t request; } __packed; /* CP2112_REQ_SMB_XFER_STATUS_RESP */ struct i2c_xfer_status_resp { uint8_t id; uint8_t status0; uint8_t status1; uint16_t status2; uint16_t status3; } __packed; /* CP2112_REQ_SMB_READ_FORCE_SEND */ struct i2c_data_read_force_send_req { uint8_t id; uint16_t len; } __packed; /* CP2112_REQ_SMB_READ_RESPONSE */ struct i2c_data_read_resp { uint8_t id; uint8_t status; uint8_t len; uint8_t data[61]; } __packed; /* CP2112_REQ_SMB_READ */ struct i2c_write_read_req { uint8_t id; uint8_t slave; uint16_t rlen; uint8_t wlen; uint8_t wdata[16]; } __packed; /* CP2112_REQ_SMB_WRITE */ struct i2c_read_req { uint8_t id; uint8_t slave; uint16_t len; } __packed; /* CP2112_REQ_SMB_WRITE_READ */ struct i2c_write_req { uint8_t id; uint8_t slave; uint8_t len; uint8_t data[61]; } __packed; /* CP2112_REQ_SMB_CFG */ struct i2c_cfg_req { uint8_t id; uint32_t speed; /* Hz */ uint8_t slave_addr; /* ACK only */ uint8_t auto_send_read; /* boolean */ uint16_t write_timeout; /* 0-1000 ms, 0 ~ no timeout */ uint16_t read_timeout; /* 0-1000 ms, 0 ~ no timeout */ uint8_t scl_low_timeout;/* boolean */ uint16_t retry_count; /* 1-1000, 0 ~ forever */ } __packed; enum cp2112_out_mode { OUT_OD, OUT_PP, OUT_KEEP }; enum { CP2112_INTR_OUT = 0, CP2112_INTR_IN, CP2112_N_TRANSFER, }; struct cp2112_softc { device_t sc_gpio_dev; device_t sc_iic_dev; struct usb_device *sc_udev; uint8_t sc_iface_index; uint8_t sc_version; }; struct cp2112gpio_softc { struct sx gpio_lock; device_t busdev; int gpio_caps; struct gpio_pin pins[CP2112_GPIO_COUNT]; }; struct cp2112iic_softc { device_t dev; device_t iicbus_dev; struct usb_xfer *xfers[CP2112_N_TRANSFER]; u_char own_addr; struct { struct mtx lock; struct cv cv; struct { uint8_t *data; int len; int done; int error; } in; struct { const uint8_t *data; int len; int done; int error; } out; } io; }; static int cp2112gpio_detach(device_t dev); static int cp2112iic_detach(device_t dev); static const STRUCT_USB_HOST_ID cp2112_devs[] = { { USB_VP(USB_VENDOR_SILABS, USB_PRODUCT_SILABS_CP2112) }, { USB_VP(0x1009, USB_PRODUCT_SILABS_CP2112) }, /* XXX */ }; static int cp2112_get_report(device_t dev, uint8_t id, void *data, uint16_t len) { struct cp2112_softc *sc; int err; sc = device_get_softc(dev); err = usbd_req_get_report(sc->sc_udev, NULL, data, len, sc->sc_iface_index, UHID_FEATURE_REPORT, id); return (err); } static int cp2112_set_report(device_t dev, uint8_t id, void *data, uint16_t len) { struct cp2112_softc *sc; int err; sc = device_get_softc(dev); *(uint8_t *)data = id; err = usbd_req_set_report(sc->sc_udev, NULL, data, len, sc->sc_iface_index, UHID_FEATURE_REPORT, id); return (err); } static int cp2112_probe(device_t dev) { struct usb_attach_arg *uaa; uaa = device_get_ivars(dev); if (uaa->usb_mode != USB_MODE_HOST) return (ENXIO); if (uaa->info.bInterfaceClass != UICLASS_HID) return (ENXIO); if (usbd_lookup_id_by_uaa(cp2112_devs, sizeof(cp2112_devs), uaa) == 0) return (BUS_PROBE_DEFAULT); return (ENXIO); } static int cp2112_attach(device_t dev) { struct version_request vdata; struct usb_attach_arg *uaa; struct cp2112_softc *sc; int err; uaa = device_get_ivars(dev); sc = device_get_softc(dev); device_set_usb_desc(dev); sc->sc_udev = uaa->device; sc->sc_iface_index = uaa->info.bIfaceIndex; err = cp2112_get_report(dev, CP2112_REQ_VERSION, &vdata, sizeof(vdata)); if (err != 0) goto detach; device_printf(dev, "part number 0x%02x, version 0x%02x\n", vdata.part_num, vdata.version); if (vdata.part_num != CP2112_PART_NUM) { device_printf(dev, "unsupported part number\n"); goto detach; } sc->sc_version = vdata.version; sc->sc_gpio_dev = device_add_child(dev, "gpio", DEVICE_UNIT_ANY); if (sc->sc_gpio_dev != NULL) { err = device_probe_and_attach(sc->sc_gpio_dev); if (err != 0) { device_printf(dev, "failed to attach gpio child\n"); } } else { device_printf(dev, "failed to create gpio child\n"); } sc->sc_iic_dev = device_add_child(dev, "iichb", DEVICE_UNIT_ANY); if (sc->sc_iic_dev != NULL) { err = device_probe_and_attach(sc->sc_iic_dev); if (err != 0) { device_printf(dev, "failed to attach iic child\n"); } } else { device_printf(dev, "failed to create iic child\n"); } return (0); detach: bus_generic_detach(dev); return (ENXIO); } static int cp2112_gpio_read_pin(device_t dev, uint32_t pin_num, bool *on) { struct gpio_get_req data; struct cp2112gpio_softc *sc __diagused; int err; sc = device_get_softc(dev); CP2112GPIO_LOCKED(sc); err = cp2112_get_report(device_get_parent(dev), CP2112_REQ_GPIO_GET, &data, sizeof(data)); if (err != 0) return (err); *on = (data.state & ((uint8_t)1 << pin_num)) != 0; return (0); } static int cp2112_gpio_write_pin(device_t dev, uint32_t pin_num, bool on) { struct gpio_set_req data; struct cp2112gpio_softc *sc __diagused; int err; bool actual; sc = device_get_softc(dev); CP2112GPIO_LOCKED(sc); data.state = (uint8_t)on << pin_num; data.mask = (uint8_t)1 << pin_num; err = cp2112_set_report(device_get_parent(dev), CP2112_REQ_GPIO_SET, &data, sizeof(data)); if (err != 0) return (err); err = cp2112_gpio_read_pin(dev, pin_num, &actual); if (err != 0) return (err); if (actual != on) return (EIO); return (0); } static int cp2112_gpio_configure_write_pin(device_t dev, uint32_t pin_num, bool output, enum cp2112_out_mode *mode) { struct gpio_config_req data; struct cp2112gpio_softc *sc __diagused; int err; uint8_t mask; sc = device_get_softc(dev); CP2112GPIO_LOCKED(sc); err = cp2112_get_report(device_get_parent(dev), CP2112_REQ_GPIO_CFG, &data, sizeof(data)); if (err != 0) return (err); mask = (uint8_t)1 << pin_num; if (output) { data.output |= mask; switch (*mode) { case OUT_PP: data.pushpull |= mask; break; case OUT_OD: data.pushpull &= ~mask; break; default: break; } } else { data.output &= ~mask; } err = cp2112_set_report(device_get_parent(dev), CP2112_REQ_GPIO_CFG, &data, sizeof(data)); if (err != 0) return (err); /* Read back and verify. */ err = cp2112_get_report(device_get_parent(dev), CP2112_REQ_GPIO_CFG, &data, sizeof(data)); if (err != 0) return (err); if (((data.output & mask) != 0) != output) return (EIO); if (output) { switch (*mode) { case OUT_PP: if ((data.pushpull & mask) == 0) return (EIO); break; case OUT_OD: if ((data.pushpull & mask) != 0) return (EIO); break; default: *mode = (data.pushpull & mask) != 0 ? OUT_PP : OUT_OD; break; } } return (0); } static device_t cp2112_gpio_get_bus(device_t dev) { struct cp2112gpio_softc *sc; sc = device_get_softc(dev); return (sc->busdev); } static int cp2112_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = CP2112_GPIO_COUNT - 1; return (0); } static int cp2112_gpio_pin_set(device_t dev, uint32_t pin_num, uint32_t pin_value) { struct cp2112gpio_softc *sc; int err; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); err = cp2112_gpio_write_pin(dev, pin_num, pin_value != 0); CP2112GPIO_UNLOCK(sc); return (err); } static int cp2112_gpio_pin_get(device_t dev, uint32_t pin_num, uint32_t *pin_value) { struct cp2112gpio_softc *sc; int err; bool on; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); err = cp2112_gpio_read_pin(dev, pin_num, &on); CP2112GPIO_UNLOCK(sc); if (err == 0) *pin_value = on; return (err); } static int cp2112_gpio_pin_toggle(device_t dev, uint32_t pin_num) { struct cp2112gpio_softc *sc; int err; bool on; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); err = cp2112_gpio_read_pin(dev, pin_num, &on); if (err == 0) err = cp2112_gpio_write_pin(dev, pin_num, !on); CP2112GPIO_UNLOCK(sc); return (err); } static int cp2112_gpio_pin_getcaps(device_t dev, uint32_t pin_num, uint32_t *caps) { struct cp2112gpio_softc *sc; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); *caps = sc->gpio_caps; CP2112GPIO_UNLOCK(sc); return (0); } static int cp2112_gpio_pin_getflags(device_t dev, uint32_t pin_num, uint32_t *flags) { struct cp2112gpio_softc *sc; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); *flags = sc->pins[pin_num].gp_flags; CP2112GPIO_UNLOCK(sc); return (0); } static int cp2112_gpio_pin_getname(device_t dev, uint32_t pin_num, char *name) { struct cp2112gpio_softc *sc; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); CP2112GPIO_LOCK(sc); memcpy(name, sc->pins[pin_num].gp_name, GPIOMAXNAME); CP2112GPIO_UNLOCK(sc); return (0); } static int cp2112_gpio_pin_setflags(device_t dev, uint32_t pin_num, uint32_t flags) { struct cp2112gpio_softc *sc; struct gpio_pin *pin; enum cp2112_out_mode out_mode; int err; if (pin_num >= CP2112_GPIO_COUNT) return (EINVAL); sc = device_get_softc(dev); if ((flags & sc->gpio_caps) != flags) return (EINVAL); if ((flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) == 0) return (EINVAL); if ((flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) == (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) { return (EINVAL); } if ((flags & GPIO_PIN_INPUT) != 0) { if ((flags & (GPIO_PIN_OPENDRAIN | GPIO_PIN_PUSHPULL)) != 0) return (EINVAL); } else { if ((flags & (GPIO_PIN_OPENDRAIN | GPIO_PIN_PUSHPULL)) == (GPIO_PIN_OPENDRAIN | GPIO_PIN_PUSHPULL)) return (EINVAL); } /* * If neither push-pull or open-drain is explicitly requested, then * preserve the current state. */ out_mode = OUT_KEEP; if ((flags & GPIO_PIN_OUTPUT) != 0) { if ((flags & GPIO_PIN_OPENDRAIN) != 0) out_mode = OUT_OD; if ((flags & GPIO_PIN_PUSHPULL) != 0) out_mode = OUT_PP; } CP2112GPIO_LOCK(sc); pin = &sc->pins[pin_num]; err = cp2112_gpio_configure_write_pin(dev, pin_num, (flags & GPIO_PIN_OUTPUT) != 0, &out_mode); if (err == 0) { /* * If neither open-drain or push-pull was requested, then see * what hardware actually had. Otherwise, it has been * reconfigured as requested. */ if ((flags & GPIO_PIN_OUTPUT) != 0 && (flags & (GPIO_PIN_OPENDRAIN | GPIO_PIN_PUSHPULL)) == 0) { KASSERT(out_mode != OUT_KEEP, ("impossible current output mode")); if (out_mode == OUT_OD) flags |= GPIO_PIN_OPENDRAIN; else flags |= GPIO_PIN_PUSHPULL; } pin->gp_flags = flags; } CP2112GPIO_UNLOCK(sc); return (err); } static int cp2112gpio_probe(device_t dev) { device_set_desc(dev, "CP2112 GPIO interface"); return (BUS_PROBE_SPECIFIC); } static int cp2112gpio_attach(device_t dev) { struct gpio_config_req data; struct cp2112gpio_softc *sc; device_t cp2112; int err; int i; uint8_t mask; cp2112 = device_get_parent(dev); sc = device_get_softc(dev); sx_init(&sc->gpio_lock, "cp2112 lock"); sc->gpio_caps = GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | GPIO_PIN_OPENDRAIN | GPIO_PIN_PUSHPULL; err = cp2112_get_report(cp2112, CP2112_REQ_GPIO_CFG, &data, sizeof(data)); if (err != 0) goto detach; for (i = 0; i < CP2112_GPIO_COUNT; i++) { struct gpio_pin *pin; mask = (uint8_t)1 << i; pin = &sc->pins[i]; pin->gp_flags = 0; snprintf(pin->gp_name, GPIOMAXNAME, "GPIO%u", i); pin->gp_name[GPIOMAXNAME - 1] = '\0'; if ((i == 0 && (data.special & CP2112_GPIO_SPEC_TX0) != 0) || (i == 1 && (data.special & CP2112_GPIO_SPEC_RX1) != 0) || (i == 7 && (data.special & CP2112_GPIO_SPEC_CLK7) != 0)) { /* Special mode means that a pin is not for GPIO. */ } else if ((data.output & mask) != 0) { pin->gp_flags |= GPIO_PIN_OUTPUT; if ((data.pushpull & mask) != 0) pin->gp_flags |= GPIO_PIN_PUSHPULL; else pin->gp_flags |= GPIO_PIN_OPENDRAIN; } else { pin->gp_flags |= GPIO_PIN_INPUT; } } sc->busdev = gpiobus_attach_bus(dev); if (sc->busdev == NULL) { device_printf(dev, "gpiobus_attach_bus failed\n"); goto detach; } return (0); detach: cp2112gpio_detach(dev); return (ENXIO); } static int cp2112gpio_detach(device_t dev) { struct cp2112gpio_softc *sc; sc = device_get_softc(dev); if (sc->busdev != NULL) gpiobus_detach_bus(dev); sx_destroy(&sc->gpio_lock); return (0); } static void cp2112iic_intr_write_callback(struct usb_xfer *xfer, usb_error_t error) { struct cp2112iic_softc *sc; struct usb_page_cache *pc; sc = usbd_xfer_softc(xfer); mtx_assert(&sc->io.lock, MA_OWNED); switch (USB_GET_STATE(xfer)) { case USB_ST_SETUP: pc = usbd_xfer_get_frame(xfer, 0); usbd_copy_in(pc, 0, sc->io.out.data, sc->io.out.len); usbd_xfer_set_frame_len(xfer, 0, sc->io.out.len); usbd_xfer_set_frames(xfer, 1); usbd_transfer_submit(xfer); break; case USB_ST_TRANSFERRED: sc->io.out.error = 0; sc->io.out.done = 1; cv_signal(&sc->io.cv); break; default: /* Error */ device_printf(sc->dev, "write intr state %d error %d\n", USB_GET_STATE(xfer), error); sc->io.out.error = IIC_EBUSERR; cv_signal(&sc->io.cv); if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); } break; } } static void cp2112iic_intr_read_callback(struct usb_xfer *xfer, usb_error_t error) { struct cp2112iic_softc *sc = usbd_xfer_softc(xfer); struct usb_page_cache *pc; int act_len, len; mtx_assert(&sc->io.lock, MA_OWNED); usbd_xfer_status(xfer, &act_len, NULL, NULL, NULL); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: if (sc->io.in.done) { device_printf(sc->dev, "interrupt while previous is pending, ignored\n"); } else if (sc->io.in.len == 0) { uint8_t buf[8]; /* * There is a spurious Transfer Status Response and * zero-length Read Response during hardware * configuration. Possibly they carry some information * about the initial bus state. */ if (device_is_attached(sc->dev)) { device_printf(sc->dev, "unsolicited interrupt, ignored\n"); if (bootverbose) { pc = usbd_xfer_get_frame(xfer, 0); len = MIN(sizeof(buf), act_len); usbd_copy_out(pc, 0, buf, len); device_printf(sc->dev, "data: %*D\n", len, buf, " "); } } else { pc = usbd_xfer_get_frame(xfer, 0); len = MIN(sizeof(buf), act_len); usbd_copy_out(pc, 0, buf, len); if (buf[0] == CP2112_REQ_SMB_XFER_STATUS_RESP) { device_printf(sc->dev, "initial bus status0 = 0x%02x, " "status1 = 0x%02x\n", buf[1], buf[2]); } } } else if (act_len == CP2112_REPORT_SIZE) { pc = usbd_xfer_get_frame(xfer, 0); usbd_copy_out(pc, 0, sc->io.in.data, sc->io.in.len); sc->io.in.error = 0; sc->io.in.done = 1; } else { device_printf(sc->dev, "unexpected input report length %u\n", act_len); sc->io.in.error = IIC_EBUSERR; sc->io.in.done = 1; } cv_signal(&sc->io.cv); case USB_ST_SETUP: tr_setup: usbd_xfer_set_frame_len(xfer, 0, usbd_xfer_max_len(xfer)); usbd_transfer_submit(xfer); break; default: /* Error */ device_printf(sc->dev, "read intr state %d error %d\n", USB_GET_STATE(xfer), error); sc->io.in.error = IIC_EBUSERR; sc->io.in.done = 1; cv_signal(&sc->io.cv); if (error != USB_ERR_CANCELLED) { /* try to clear stall first */ usbd_xfer_set_stall(xfer); goto tr_setup; } break; } } static const struct usb_config cp2112iic_config[CP2112_N_TRANSFER] = { [CP2112_INTR_OUT] = { .type = UE_INTERRUPT, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_OUT, .flags = { .pipe_bof = 1, .no_pipe_ok = 1, }, .bufsize = 0, /* use wMaxPacketSize */ .callback = &cp2112iic_intr_write_callback, }, [CP2112_INTR_IN] = { .type = UE_INTERRUPT, .endpoint = UE_ADDR_ANY, .direction = UE_DIR_IN, .flags = { .pipe_bof = 1, .short_xfer_ok = 1, }, .bufsize = 0, /* use wMaxPacketSize */ .callback = &cp2112iic_intr_read_callback, }, }; static int cp2112iic_send_req(struct cp2112iic_softc *sc, const void *data, uint16_t len) { int err; mtx_assert(&sc->io.lock, MA_OWNED); KASSERT(sc->io.out.done == 0, ("%s: conflicting request", __func__)); sc->io.out.data = data; sc->io.out.len = len; DTRACE_PROBE1(send__req, uint8_t, *(const uint8_t *)data); usbd_transfer_start(sc->xfers[CP2112_INTR_OUT]); while (!sc->io.out.done) cv_wait(&sc->io.cv, &sc->io.lock); usbd_transfer_stop(sc->xfers[CP2112_INTR_OUT]); sc->io.out.done = 0; sc->io.out.data = NULL; sc->io.out.len = 0; err = sc->io.out.error; if (err != 0) { device_printf(sc->dev, "output report 0x%02x failed: %d\n", *(const uint8_t*)data, err); } return (err); } static int cp2112iic_req_resp(struct cp2112iic_softc *sc, const void *req_data, uint16_t req_len, void *resp_data, uint16_t resp_len) { int err; mtx_assert(&sc->io.lock, MA_OWNED); /* * Prepare to receive a response interrupt even before the * request transfer is confirmed (USB_ST_TRANSFERED). */ KASSERT(sc->io.in.done == 0, ("%s: conflicting request", __func__)); sc->io.in.len = resp_len; sc->io.in.data = resp_data; err = cp2112iic_send_req(sc, req_data, req_len); if (err != 0) { sc->io.in.len = 0; sc->io.in.data = NULL; return (err); } while (!sc->io.in.done) cv_wait(&sc->io.cv, &sc->io.lock); err = sc->io.in.error; sc->io.in.done = 0; sc->io.in.error = 0; sc->io.in.len = 0; sc->io.in.data = NULL; return (err); } static int cp2112iic_check_req_status(struct cp2112iic_softc *sc) { struct i2c_xfer_status_req xfer_status_req; struct i2c_xfer_status_resp xfer_status_resp; int err; mtx_assert(&sc->io.lock, MA_OWNED); do { xfer_status_req.id = CP2112_REQ_SMB_XFER_STATUS_REQ; xfer_status_req.request = 1; err = cp2112iic_req_resp(sc, &xfer_status_req, sizeof(xfer_status_req), &xfer_status_resp, sizeof(xfer_status_resp)); if (xfer_status_resp.id != CP2112_REQ_SMB_XFER_STATUS_RESP) { device_printf(sc->dev, "unexpected response 0x%02x to status request\n", xfer_status_resp.id); err = IIC_EBUSERR; goto out; } DTRACE_PROBE4(xfer__status, uint8_t, xfer_status_resp.status0, uint8_t, xfer_status_resp.status1, uint16_t, be16toh(xfer_status_resp.status2), uint16_t, be16toh(xfer_status_resp.status3)); switch (xfer_status_resp.status0) { case CP2112_IIC_STATUS0_IDLE: err = IIC_ESTATUS; break; case CP2112_IIC_STATUS0_BUSY: err = ERESTART; /* non-I2C, special handling */ break; case CP2112_IIC_STATUS0_CMP: err = IIC_NOERR; break; case CP2112_IIC_STATUS0_ERROR: switch (xfer_status_resp.status1) { case CP2112_IIC_STATUS1_TIMEOUT_NACK: err = IIC_ENOACK; break; case CP2112_IIC_STATUS1_TIMEOUT_BUS: err = IIC_ETIMEOUT; break; case CP2112_IIC_STATUS1_ARB_LOST: err = IIC_EBUSBSY; break; default: device_printf(sc->dev, "i2c error, status = 0x%02x\n", xfer_status_resp.status1); err = IIC_ESTATUS; break; } break; default: device_printf(sc->dev, "unknown i2c xfer status0 0x%02x\n", xfer_status_resp.status0); err = IIC_EBUSERR; break; } } while (err == ERESTART); out: return (err); } static int cp2112iic_read_data(struct cp2112iic_softc *sc, void *data, uint16_t in_len, uint16_t *out_len) { struct i2c_data_read_force_send_req data_read_force_send; struct i2c_data_read_resp data_read_resp; int err; mtx_assert(&sc->io.lock, MA_OWNED); /* * Prepare to receive a response interrupt even before the request * transfer is confirmed (USB_ST_TRANSFERED). */ if (in_len > sizeof(data_read_resp.data)) in_len = sizeof(data_read_resp.data); data_read_force_send.id = CP2112_REQ_SMB_READ_FORCE_SEND; data_read_force_send.len = htobe16(in_len); err = cp2112iic_req_resp(sc, &data_read_force_send, sizeof(data_read_force_send), &data_read_resp, sizeof(data_read_resp)); if (err != 0) goto out; if (data_read_resp.id != CP2112_REQ_SMB_READ_RESPONSE) { device_printf(sc->dev, "unexpected response 0x%02x to data read request\n", data_read_resp.id); err = IIC_EBUSERR; goto out; } DTRACE_PROBE2(read__response, uint8_t, data_read_resp.status, uint8_t, data_read_resp.len); /* * We expect either the request completed status or, more typical for * this driver, the bus idle status because of the preceding * Force Read Status command (which is not an I2C request). */ if (data_read_resp.status != CP2112_IIC_STATUS0_CMP && data_read_resp.status != CP2112_IIC_STATUS0_IDLE) { err = IIC_EBUSERR; goto out; } if (data_read_resp.len > in_len) { device_printf(sc->dev, "device returns more data than asked\n"); err = IIC_EOVERFLOW; goto out; } *out_len = data_read_resp.len; if (*out_len > 0) memcpy(data, data_read_resp.data, *out_len); out: return (err); } static int cp2112iic_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) { struct cp2112iic_softc *sc = device_get_softc(dev); struct cp2112_softc *psc = device_get_softc(device_get_parent(dev)); const char *reason = NULL; uint32_t i; uint16_t read_off, to_read; int err; /* * The hardware interface imposes limits on allowed I2C messages. * It is not possible to explicitly send a start or stop. * It is not possible to do a zero length transfer. * For this reason it's impossible to send a message with no data * at all (like an SMBus quick message). * Each read or write transfer beginning with the start condition * and ends with the stop condition. The only exception is that * it is possible to have a write transfer followed by a read * transfer to the same slave with the repeated start condition * between them. */ for (i = 0; i < nmsgs; i++) { if (i == 0 && (msgs[i].flags & IIC_M_NOSTART) != 0) { reason = "first message without start"; break; } if (i == nmsgs - 1 && (msgs[i].flags & IIC_M_NOSTOP) != 0) { reason = "last message without stop"; break; } if (msgs[i].len == 0) { reason = "message with no data"; break; } if ((msgs[i].flags & IIC_M_RD) != 0 && msgs[i].len > CP2112_IIC_MAX_READ_LEN) { reason = "too long read"; break; } if ((msgs[i].flags & IIC_M_RD) == 0 && msgs[i].len > SIZEOF_FIELD(i2c_write_req, data)) { reason = "too long write"; break; } if ((msgs[i].flags & IIC_M_NOSTART) != 0) { reason = "message without start or repeated start"; break; } if ((msgs[i].flags & IIC_M_NOSTOP) != 0 && (msgs[i].flags & IIC_M_RD) != 0) { reason = "read without stop"; break; } if ((msgs[i].flags & IIC_M_NOSTOP) != 0 && psc->sc_version < CP2112_IIC_REPSTART_VER) { reason = "write without stop"; break; } if ((msgs[i].flags & IIC_M_NOSTOP) != 0 && msgs[i].len > SIZEOF_FIELD(i2c_write_read_req, wdata)) { reason = "too long write without stop"; break; } if (i > 0) { if ((msgs[i - 1].flags & IIC_M_NOSTOP) != 0 && msgs[i].slave != msgs[i - 1].slave) { reason = "change of slave without stop"; break; } if ((msgs[i - 1].flags & IIC_M_NOSTOP) != 0 && (msgs[i].flags & IIC_M_RD) == 0) { reason = "write after repeated start"; break; } } } if (reason != NULL) { if (bootverbose) device_printf(dev, "unsupported i2c message: %s\n", reason); return (IIC_ENOTSUPP); } mtx_lock(&sc->io.lock); for (i = 0; i < nmsgs; i++) { if (i + 1 < nmsgs && (msgs[i].flags & IIC_M_NOSTOP) != 0) { /* * Combine into a single * CP2112 operation. */ struct i2c_write_read_req req; KASSERT((msgs[i].flags & IIC_M_RD) == 0, ("read without stop")); KASSERT((msgs[i + 1].flags & IIC_M_RD) != 0, ("write after write without stop")); req.id = CP2112_REQ_SMB_WRITE_READ; req.slave = msgs[i].slave & ~LSB; to_read = msgs[i + 1].len; req.rlen = htobe16(to_read); req.wlen = msgs[i].len; memcpy(req.wdata, msgs[i].buf, msgs[i].len); err = cp2112iic_send_req(sc, &req, msgs[i].len + 5); /* * The next message is already handled. * Also needed for read data to go into the right msg. */ i++; } else if ((msgs[i].flags & IIC_M_RD) != 0) { struct i2c_read_req req; req.id = CP2112_REQ_SMB_READ; req.slave = msgs[i].slave & ~LSB; to_read = msgs[i].len; req.len = htobe16(to_read); err = cp2112iic_send_req(sc, &req, sizeof(req)); } else { struct i2c_write_req req; req.id = CP2112_REQ_SMB_WRITE; req.slave = msgs[i].slave & ~LSB; req.len = msgs[i].len; memcpy(req.data, msgs[i].buf, msgs[i].len); to_read = 0; err = cp2112iic_send_req(sc, &req, msgs[i].len + 3); } if (err != 0) break; err = cp2112iic_check_req_status(sc); if (err != 0) break; read_off = 0; while (to_read > 0) { uint16_t act_read; err = cp2112iic_read_data(sc, msgs[i].buf + read_off, to_read, &act_read); if (err != 0) break; KASSERT(act_read <= to_read, ("cp2112iic_read_data " "returned more data than asked")); read_off += act_read; to_read -= act_read; } if (err != 0) break; } mtx_unlock(&sc->io.lock); return (err); } static int cp2112iic_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr) { struct i2c_cfg_req i2c_cfg; struct cp2112iic_softc *sc; device_t cp2112; u_int busfreq; int err; sc = device_get_softc(dev); cp2112 = device_get_parent(dev); if (sc->iicbus_dev == NULL) busfreq = 100000; else busfreq = IICBUS_GET_FREQUENCY(sc->iicbus_dev, speed); err = cp2112_get_report(cp2112, CP2112_REQ_SMB_CFG, &i2c_cfg, sizeof(i2c_cfg)); if (err != 0) { device_printf(dev, "failed to get CP2112_REQ_SMB_CFG report\n"); return (err); } if (oldaddr != NULL) *oldaddr = i2c_cfg.slave_addr; /* * For simplicity we do not enable Auto Send Read * because of erratum CP2112_E101 (fixed in version 3). * * TODO: set I2C parameters based on configuration preferences: * - read and write timeouts (no timeout by default), * - SCL low timeout (disabled by default), * etc. * * TODO: should the device reset request (0x01) be sent? * If the device disconnects as a result, then no. */ i2c_cfg.speed = htobe32(busfreq); if (addr != 0) i2c_cfg.slave_addr = addr; i2c_cfg.auto_send_read = 0; i2c_cfg.retry_count = htobe16(1); i2c_cfg.scl_low_timeout = 0; if (bootverbose) { device_printf(dev, "speed %d Hz\n", be32toh(i2c_cfg.speed)); device_printf(dev, "slave addr 0x%02x\n", i2c_cfg.slave_addr); device_printf(dev, "auto send read %s\n", i2c_cfg.auto_send_read ? "on" : "off"); device_printf(dev, "write timeout %d ms (0 - disabled)\n", be16toh(i2c_cfg.write_timeout)); device_printf(dev, "read timeout %d ms (0 - disabled)\n", be16toh(i2c_cfg.read_timeout)); device_printf(dev, "scl low timeout %s\n", i2c_cfg.scl_low_timeout ? "on" : "off"); device_printf(dev, "retry count %d (0 - no limit)\n", be16toh(i2c_cfg.retry_count)); } err = cp2112_set_report(cp2112, CP2112_REQ_SMB_CFG, &i2c_cfg, sizeof(i2c_cfg)); if (err != 0) { device_printf(dev, "failed to set CP2112_REQ_SMB_CFG report\n"); return (err); } return (0); } static int cp2112iic_probe(device_t dev) { device_set_desc(dev, "CP2112 I2C interface"); return (BUS_PROBE_SPECIFIC); } static int cp2112iic_attach(device_t dev) { struct cp2112iic_softc *sc; struct cp2112_softc *psc; device_t cp2112; int err; sc = device_get_softc(dev); sc->dev = dev; cp2112 = device_get_parent(dev); psc = device_get_softc(cp2112); mtx_init(&sc->io.lock, "cp2112iic lock", NULL, MTX_DEF | MTX_RECURSE); cv_init(&sc->io.cv, "cp2112iic cv"); err = usbd_transfer_setup(psc->sc_udev, &psc->sc_iface_index, sc->xfers, cp2112iic_config, nitems(cp2112iic_config), sc, &sc->io.lock); if (err != 0) { device_printf(dev, "usbd_transfer_setup failed %d\n", err); goto detach; } /* Prepare to receive interrupts. */ mtx_lock(&sc->io.lock); usbd_transfer_start(sc->xfers[CP2112_INTR_IN]); mtx_unlock(&sc->io.lock); sc->iicbus_dev = device_add_child(dev, "iicbus", DEVICE_UNIT_ANY); if (sc->iicbus_dev == NULL) { device_printf(dev, "iicbus creation failed\n"); err = ENXIO; goto detach; } bus_attach_children(dev); return (0); detach: cp2112iic_detach(dev); return (err); } static int cp2112iic_detach(device_t dev) { struct cp2112iic_softc *sc; int err; sc = device_get_softc(dev); err = bus_generic_detach(dev); if (err != 0) return (err); - device_delete_children(dev); mtx_lock(&sc->io.lock); usbd_transfer_stop(sc->xfers[CP2112_INTR_IN]); mtx_unlock(&sc->io.lock); usbd_transfer_unsetup(sc->xfers, nitems(cp2112iic_config)); cv_destroy(&sc->io.cv); mtx_destroy(&sc->io.lock); return (0); } static device_method_t cp2112hid_methods[] = { DEVMETHOD(device_probe, cp2112_probe), DEVMETHOD(device_attach, cp2112_attach), DEVMETHOD(device_detach, bus_generic_detach), DEVMETHOD_END }; static driver_t cp2112hid_driver = { .name = "cp2112hid", .methods = cp2112hid_methods, .size = sizeof(struct cp2112_softc), }; DRIVER_MODULE(cp2112hid, uhub, cp2112hid_driver, NULL, NULL); MODULE_DEPEND(cp2112hid, usb, 1, 1, 1); MODULE_VERSION(cp2112hid, 1); USB_PNP_HOST_INFO(cp2112_devs); static device_method_t cp2112gpio_methods[] = { /* Device */ DEVMETHOD(device_probe, cp2112gpio_probe), DEVMETHOD(device_attach, cp2112gpio_attach), DEVMETHOD(device_detach, cp2112gpio_detach), /* GPIO */ DEVMETHOD(gpio_get_bus, cp2112_gpio_get_bus), DEVMETHOD(gpio_pin_max, cp2112_gpio_pin_max), DEVMETHOD(gpio_pin_get, cp2112_gpio_pin_get), DEVMETHOD(gpio_pin_set, cp2112_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, cp2112_gpio_pin_toggle), DEVMETHOD(gpio_pin_getname, cp2112_gpio_pin_getname), DEVMETHOD(gpio_pin_getcaps, cp2112_gpio_pin_getcaps), DEVMETHOD(gpio_pin_getflags, cp2112_gpio_pin_getflags), DEVMETHOD(gpio_pin_setflags, cp2112_gpio_pin_setflags), DEVMETHOD_END }; static driver_t cp2112gpio_driver = { .name = "gpio", .methods = cp2112gpio_methods, .size = sizeof(struct cp2112gpio_softc), }; DRIVER_MODULE(cp2112gpio, cp2112hid, cp2112gpio_driver, NULL, NULL); MODULE_DEPEND(cp2112gpio, cp2112hid, 1, 1, 1); MODULE_DEPEND(cp2112gpio, gpiobus, 1, 1, 1); MODULE_VERSION(cp2112gpio, 1); static device_method_t cp2112iic_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cp2112iic_probe), DEVMETHOD(device_attach, cp2112iic_attach), DEVMETHOD(device_detach, cp2112iic_detach), /* I2C methods */ DEVMETHOD(iicbus_transfer, cp2112iic_transfer), DEVMETHOD(iicbus_reset, cp2112iic_reset), DEVMETHOD(iicbus_callback, iicbus_null_callback), DEVMETHOD_END }; static driver_t cp2112iic_driver = { "iichb", cp2112iic_methods, sizeof(struct cp2112iic_softc) }; DRIVER_MODULE(cp2112iic, cp2112hid, cp2112iic_driver, NULL, NULL); MODULE_DEPEND(cp2112iic, cp2112hid, 1, 1, 1); MODULE_DEPEND(cp2112iic, iicbus, IICBUS_MINVER, IICBUS_PREFVER, IICBUS_MAXVER); MODULE_VERSION(cp2112iic, 1); diff --git a/sys/dev/usb/misc/i2ctinyusb.c b/sys/dev/usb/misc/i2ctinyusb.c index 47f27d9383a6..ca40fd5baf5c 100644 --- a/sys/dev/usb/misc/i2ctinyusb.c +++ b/sys/dev/usb/misc/i2ctinyusb.c @@ -1,302 +1,301 @@ /*- * Copyright (c) 2024 Denis Bodor * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * i2c-tiny-usb, DIY USB to IIC bridge (using AVR or RP2040) from * Till Harbaum & Nicolai Electronics * See : * https://github.com/harbaum/I2C-Tiny-USB * and * https://github.com/Nicolai-Electronics/rp2040-i2c-interface */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" // commands via USB, must match command ids in the firmware #define CMD_ECHO 0 #define CMD_GET_FUNC 1 #define CMD_SET_DELAY 2 #define CMD_GET_STATUS 3 #define CMD_I2C_IO 4 #define CMD_SET_LED 8 #define CMD_I2C_IO_BEGIN (1 << 0) #define CMD_I2C_IO_END (1 << 1) #define STATUS_IDLE 0 #define STATUS_ADDRESS_ACK 1 #define STATUS_ADDRESS_NAK 2 struct i2ctinyusb_softc { struct usb_device *sc_udev; device_t sc_iic_dev; device_t iicbus_dev; struct mtx sc_mtx; }; #define USB_VENDOR_EZPROTOTYPES 0x1c40 #define USB_VENDOR_FTDI 0x0403 static const STRUCT_USB_HOST_ID i2ctinyusb_devs[] = { { USB_VPI(USB_VENDOR_EZPROTOTYPES, 0x0534, 0) }, { USB_VPI(USB_VENDOR_FTDI, 0xc631, 0) }, }; /* Prototypes. */ static int i2ctinyusb_probe(device_t dev); static int i2ctinyusb_attach(device_t dev); static int i2ctinyusb_detach(device_t dev); static int i2ctinyusb_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs); static int i2ctinyusb_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr); static int usb_read(struct i2ctinyusb_softc *sc, int cmd, int value, int index, void *data, int len) { int error; struct usb_device_request req; uint16_t actlen; req.bmRequestType = UT_READ_VENDOR_INTERFACE; req.bRequest = cmd; USETW(req.wValue, value); USETW(req.wIndex, (index >> 1)); USETW(req.wLength, len); error = usbd_do_request_flags(sc->sc_udev, &sc->sc_mtx, &req, data, 0, &actlen, 2000); if (error) actlen = -1; return (actlen); } static int usb_write(struct i2ctinyusb_softc *sc, int cmd, int value, int index, void *data, int len) { int error; struct usb_device_request req; uint16_t actlen; req.bmRequestType = UT_WRITE_VENDOR_INTERFACE; req.bRequest = cmd; USETW(req.wValue, value); USETW(req.wIndex, (index >> 1)); USETW(req.wLength, len); error = usbd_do_request_flags(sc->sc_udev, &sc->sc_mtx, &req, data, 0, &actlen, 2000); if (error) { actlen = -1; } return (actlen); } static int i2ctinyusb_probe(device_t dev) { struct usb_attach_arg *uaa; uaa = device_get_ivars(dev); if (uaa->usb_mode != USB_MODE_HOST) return (ENXIO); if (usbd_lookup_id_by_uaa(i2ctinyusb_devs, sizeof(i2ctinyusb_devs), uaa) == 0) { device_set_desc(dev, "I2C-Tiny-USB I2C interface"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int i2ctinyusb_attach(device_t dev) { struct i2ctinyusb_softc *sc; struct usb_attach_arg *uaa; int err; sc = device_get_softc(dev); uaa = device_get_ivars(dev); device_set_usb_desc(dev); sc->sc_udev = uaa->device; mtx_init(&sc->sc_mtx, "i2ctinyusb lock", NULL, MTX_DEF | MTX_RECURSE); sc->iicbus_dev = device_add_child(dev, "iicbus", -1); if (sc->iicbus_dev == NULL) { device_printf(dev, "iicbus creation failed\n"); err = ENXIO; goto detach; } bus_attach_children(dev); return (0); detach: i2ctinyusb_detach(dev); return (err); } static int i2ctinyusb_detach(device_t dev) { struct i2ctinyusb_softc *sc; int err; sc = device_get_softc(dev); err = bus_generic_detach(dev); if (err != 0) return (err); - device_delete_children(dev); mtx_destroy(&sc->sc_mtx); return (0); } static int i2ctinyusb_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) { struct i2ctinyusb_softc *sc; uint32_t i; int ret = 0; int cmd = CMD_I2C_IO; struct iic_msg *pmsg; unsigned char pstatus; sc = device_get_softc(dev); mtx_lock(&sc->sc_mtx); for (i = 0; i < nmsgs; i++) { pmsg = &msgs[i]; if (i == 0) cmd |= CMD_I2C_IO_BEGIN; if (i == nmsgs - 1) cmd |= CMD_I2C_IO_END; if ((msgs[i].flags & IIC_M_RD) != 0) { if ((ret = usb_read(sc, cmd, pmsg->flags, pmsg->slave, pmsg->buf, pmsg->len)) != pmsg->len) { printf("Read error: got %u\n", ret); ret = EIO; goto out; } } else { if ((ret = usb_write(sc, cmd, pmsg->flags, pmsg->slave, pmsg->buf, pmsg->len)) != pmsg->len) { printf("Write error: got %u\n", ret); ret = EIO; goto out; } } // check status if ((ret = usb_read(sc, CMD_GET_STATUS, 0, 0, &pstatus, 1)) != 1) { ret = EIO; goto out; } if (pstatus == STATUS_ADDRESS_NAK) { ret = EIO; goto out; } } ret = 0; out: mtx_unlock(&sc->sc_mtx); return (ret); } static int i2ctinyusb_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr) { struct i2ctinyusb_softc *sc; int ret; sc = device_get_softc(dev); mtx_lock(&sc->sc_mtx); ret = usb_write(sc, CMD_SET_DELAY, 10, 0, NULL, 0); mtx_unlock(&sc->sc_mtx); if (ret < 0) printf("i2ctinyusb_reset error!\n"); return (0); } static device_method_t i2ctinyusb_methods[] = { /* Device interface */ DEVMETHOD(device_probe, i2ctinyusb_probe), DEVMETHOD(device_attach, i2ctinyusb_attach), DEVMETHOD(device_detach, i2ctinyusb_detach), /* I2C methods */ DEVMETHOD(iicbus_transfer, i2ctinyusb_transfer), DEVMETHOD(iicbus_reset, i2ctinyusb_reset), DEVMETHOD(iicbus_callback, iicbus_null_callback), DEVMETHOD_END }; static driver_t i2ctinyusb_driver = { .name = "iichb", .methods = i2ctinyusb_methods, .size = sizeof(struct i2ctinyusb_softc), }; DRIVER_MODULE(i2ctinyusb, uhub, i2ctinyusb_driver, NULL, NULL); MODULE_DEPEND(i2ctinyusb, usb, 1, 1, 1); MODULE_DEPEND(i2ctinyusb, iicbus, IICBUS_MINVER, IICBUS_PREFVER, IICBUS_MAXVER); MODULE_VERSION(i2ctinyusb, 1); /* vi: set ts=8 sw=8: */ diff --git a/sys/dev/vmd/vmd.c b/sys/dev/vmd/vmd.c index 40565eab6f3e..0595a6c5be16 100644 --- a/sys/dev/vmd/vmd.c +++ b/sys/dev/vmd/vmd.c @@ -1,754 +1,751 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Alexander Motin * Copyright 2019 Cisco Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" struct vmd_type { u_int16_t vmd_vid; u_int16_t vmd_did; char *vmd_name; int flags; #define BUS_RESTRICT 1 #define VECTOR_OFFSET 2 #define CAN_BYPASS_MSI 4 }; #define VMD_CAP 0x40 #define VMD_BUS_RESTRICT 0x1 #define VMD_CONFIG 0x44 #define VMD_BYPASS_MSI 0x2 #define VMD_BUS_START(x) ((x >> 8) & 0x3) #define VMD_LOCK 0x70 SYSCTL_NODE(_hw, OID_AUTO, vmd, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Intel Volume Management Device tuning parameters"); /* * By default all VMD devices remap children MSI/MSI-X interrupts into their * own. It creates additional isolation, but also complicates things due to * sharing, etc. Fortunately some VMD devices can bypass the remapping. */ static int vmd_bypass_msi = 1; SYSCTL_INT(_hw_vmd, OID_AUTO, bypass_msi, CTLFLAG_RWTUN, &vmd_bypass_msi, 0, "Bypass MSI remapping on capable hardware"); /* * All MSIs within a group share address, so VMD can't distinguish them. * It makes no sense to use more than one per device, only if required by * some specific device drivers. */ static int vmd_max_msi = 1; SYSCTL_INT(_hw_vmd, OID_AUTO, max_msi, CTLFLAG_RWTUN, &vmd_max_msi, 0, "Maximum number of MSI vectors per device"); /* * MSI-X can use different addresses, but we have limited number of MSI-X * we can route to, so use conservative default to try to avoid sharing. */ static int vmd_max_msix = 3; SYSCTL_INT(_hw_vmd, OID_AUTO, max_msix, CTLFLAG_RWTUN, &vmd_max_msix, 0, "Maximum number of MSI-X vectors per device"); static struct vmd_type vmd_devs[] = { { 0x8086, 0x201d, "Intel Volume Management Device", 0 }, { 0x8086, 0x28c0, "Intel Volume Management Device", BUS_RESTRICT | CAN_BYPASS_MSI }, { 0x8086, 0x467f, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0x8086, 0x4c3d, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0x8086, 0x7d0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0x8086, 0x9a0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0x8086, 0xa77f, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0x8086, 0xad0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET }, { 0, 0, NULL, 0 } }; static int vmd_probe(device_t dev) { struct vmd_type *t; uint16_t vid, did; vid = pci_get_vendor(dev); did = pci_get_device(dev); for (t = vmd_devs; t->vmd_name != NULL; t++) { if (vid == t->vmd_vid && did == t->vmd_did) { device_set_desc(dev, t->vmd_name); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static void vmd_free(struct vmd_softc *sc) { struct vmd_irq *vi; struct vmd_irq_user *u; int i; if (sc->psc.bus.rman.rm_end != 0) rman_fini(&sc->psc.bus.rman); if (sc->psc.mem.rman.rm_end != 0) rman_fini(&sc->psc.mem.rman); while ((u = LIST_FIRST(&sc->vmd_users)) != NULL) { LIST_REMOVE(u, viu_link); free(u, M_DEVBUF); } if (sc->vmd_irq != NULL) { for (i = 0; i < sc->vmd_msix_count; i++) { vi = &sc->vmd_irq[i]; if (vi->vi_res == NULL) continue; bus_teardown_intr(sc->psc.dev, vi->vi_res, vi->vi_handle); bus_release_resource(sc->psc.dev, SYS_RES_IRQ, vi->vi_rid, vi->vi_res); } } free(sc->vmd_irq, M_DEVBUF); sc->vmd_irq = NULL; pci_release_msi(sc->psc.dev); for (i = 0; i < VMD_MAX_BAR; i++) { if (sc->vmd_regs_res[i] != NULL) bus_release_resource(sc->psc.dev, SYS_RES_MEMORY, sc->vmd_regs_rid[i], sc->vmd_regs_res[i]); } } /* Hidden PCI Roots are hidden in BAR(0). */ static uint32_t vmd_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width) { struct vmd_softc *sc; bus_addr_t offset; sc = device_get_softc(dev); if (b < sc->vmd_bus_start || b > sc->vmd_bus_end) return (0xffffffff); offset = ((b - sc->vmd_bus_start) << 20) + (s << 15) + (f << 12) + reg; switch (width) { case 4: return (bus_read_4(sc->vmd_regs_res[0], offset)); case 2: return (bus_read_2(sc->vmd_regs_res[0], offset)); case 1: return (bus_read_1(sc->vmd_regs_res[0], offset)); default: __assert_unreachable(); return (0xffffffff); } } static void vmd_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width) { struct vmd_softc *sc; bus_addr_t offset; sc = device_get_softc(dev); if (b < sc->vmd_bus_start || b > sc->vmd_bus_end) return; offset = ((b - sc->vmd_bus_start) << 20) + (s << 15) + (f << 12) + reg; switch (width) { case 4: return (bus_write_4(sc->vmd_regs_res[0], offset, val)); case 2: return (bus_write_2(sc->vmd_regs_res[0], offset, val)); case 1: return (bus_write_1(sc->vmd_regs_res[0], offset, val)); default: __assert_unreachable(); } } static void vmd_set_msi_bypass(device_t dev, bool enable) { uint16_t val; val = pci_read_config(dev, VMD_CONFIG, 2); if (enable) val |= VMD_BYPASS_MSI; else val &= ~VMD_BYPASS_MSI; pci_write_config(dev, VMD_CONFIG, val, 2); } static int vmd_intr(void *arg) { /* * We have nothing to do here, but we have to register some interrupt * handler to make PCI code setup and enable the MSI-X vector. */ return (FILTER_STRAY); } static int vmd_attach(device_t dev) { struct vmd_softc *sc; struct pcib_secbus *bus; struct pcib_window *w; struct vmd_type *t; struct vmd_irq *vi; uint16_t vid, did; uint32_t bar; int i, j, error; char buf[64]; sc = device_get_softc(dev); bzero(sc, sizeof(*sc)); sc->psc.dev = dev; sc->psc.domain = PCI_DOMAINMAX - device_get_unit(dev); pci_enable_busmaster(dev); for (i = 0, j = 0; i < VMD_MAX_BAR; i++, j++) { sc->vmd_regs_rid[i] = PCIR_BAR(j); bar = pci_read_config(dev, PCIR_BAR(0), 4); if (PCI_BAR_MEM(bar) && (bar & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) j++; if ((sc->vmd_regs_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->vmd_regs_rid[i], RF_ACTIVE)) == NULL) { device_printf(dev, "Cannot allocate resources\n"); goto fail; } } vid = pci_get_vendor(dev); did = pci_get_device(dev); for (t = vmd_devs; t->vmd_name != NULL; t++) { if (vid == t->vmd_vid && did == t->vmd_did) break; } sc->vmd_bus_start = 0; if ((t->flags & BUS_RESTRICT) && (pci_read_config(dev, VMD_CAP, 2) & VMD_BUS_RESTRICT)) { switch (VMD_BUS_START(pci_read_config(dev, VMD_CONFIG, 2))) { case 0: sc->vmd_bus_start = 0; break; case 1: sc->vmd_bus_start = 128; break; case 2: sc->vmd_bus_start = 224; break; default: device_printf(dev, "Unknown bus offset\n"); goto fail; } } sc->vmd_bus_end = MIN(PCI_BUSMAX, sc->vmd_bus_start + (rman_get_size(sc->vmd_regs_res[0]) >> 20) - 1); bus = &sc->psc.bus; bus->sec = sc->vmd_bus_start; bus->sub = sc->vmd_bus_end; bus->dev = dev; bus->rman.rm_start = 0; bus->rman.rm_end = PCI_BUSMAX; bus->rman.rm_type = RMAN_ARRAY; snprintf(buf, sizeof(buf), "%s bus numbers", device_get_nameunit(dev)); bus->rman.rm_descr = strdup(buf, M_DEVBUF); error = rman_init(&bus->rman); if (error) { device_printf(dev, "Failed to initialize bus rman\n"); bus->rman.rm_end = 0; goto fail; } error = rman_manage_region(&bus->rman, sc->vmd_bus_start, sc->vmd_bus_end); if (error) { device_printf(dev, "Failed to add resource to bus rman\n"); goto fail; } w = &sc->psc.mem; w->rman.rm_type = RMAN_ARRAY; snprintf(buf, sizeof(buf), "%s memory window", device_get_nameunit(dev)); w->rman.rm_descr = strdup(buf, M_DEVBUF); error = rman_init(&w->rman); if (error) { device_printf(dev, "Failed to initialize memory rman\n"); w->rman.rm_end = 0; goto fail; } error = rman_manage_region(&w->rman, rman_get_start(sc->vmd_regs_res[1]), rman_get_end(sc->vmd_regs_res[1])); if (error) { device_printf(dev, "Failed to add resource to memory rman\n"); goto fail; } error = rman_manage_region(&w->rman, rman_get_start(sc->vmd_regs_res[2]) + 0x2000, rman_get_end(sc->vmd_regs_res[2])); if (error) { device_printf(dev, "Failed to add resource to memory rman\n"); goto fail; } LIST_INIT(&sc->vmd_users); sc->vmd_fist_vector = (t->flags & VECTOR_OFFSET) ? 1 : 0; sc->vmd_msix_count = pci_msix_count(dev); if (vmd_bypass_msi && (t->flags & CAN_BYPASS_MSI)) { sc->vmd_msix_count = 0; vmd_set_msi_bypass(dev, true); } else if (pci_alloc_msix(dev, &sc->vmd_msix_count) == 0) { sc->vmd_irq = malloc(sizeof(struct vmd_irq) * sc->vmd_msix_count, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < sc->vmd_msix_count; i++) { vi = &sc->vmd_irq[i]; vi->vi_rid = i + 1; vi->vi_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &vi->vi_rid, RF_ACTIVE | RF_SHAREABLE); if (vi->vi_res == NULL) { device_printf(dev, "Failed to allocate irq\n"); goto fail; } vi->vi_irq = rman_get_start(vi->vi_res); if (bus_setup_intr(dev, vi->vi_res, INTR_TYPE_MISC | INTR_MPSAFE, vmd_intr, NULL, vi, &vi->vi_handle)) { device_printf(dev, "Can't set up interrupt\n"); bus_release_resource(dev, SYS_RES_IRQ, vi->vi_rid, vi->vi_res); vi->vi_res = NULL; goto fail; } } vmd_set_msi_bypass(dev, false); } sc->vmd_dma_tag = bus_get_dma_tag(dev); sc->psc.child = device_add_child(dev, "pci", DEVICE_UNIT_ANY); bus_attach_children(dev); return (0); fail: vmd_free(sc); return (ENXIO); } static int vmd_detach(device_t dev) { struct vmd_softc *sc = device_get_softc(dev); int error; error = bus_generic_detach(dev); - if (error) - return (error); - error = device_delete_children(dev); if (error) return (error); if (sc->vmd_msix_count == 0) vmd_set_msi_bypass(dev, false); vmd_free(sc); return (0); } static bus_dma_tag_t vmd_get_dma_tag(device_t dev, device_t child) { struct vmd_softc *sc = device_get_softc(dev); return (sc->vmd_dma_tag); } static struct rman * vmd_get_rman(device_t dev, int type, u_int flags) { struct vmd_softc *sc = device_get_softc(dev); switch (type) { case SYS_RES_MEMORY: return (&sc->psc.mem.rman); case PCI_RES_BUS: return (&sc->psc.bus.rman); default: /* VMD hardware does not support I/O ports. */ return (NULL); } } static struct resource * vmd_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; if (type == SYS_RES_IRQ) { /* VMD hardware does not support legacy interrupts. */ if (*rid == 0) return (NULL); return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags | RF_SHAREABLE)); } res = bus_generic_rman_alloc_resource(dev, child, type, rid, start, end, count, flags); if (bootverbose && res != NULL) { switch (type) { case SYS_RES_MEMORY: device_printf(dev, "allocated memory range (%#jx-%#jx) for rid %d of %s\n", rman_get_start(res), rman_get_end(res), *rid, pcib_child_name(child)); break; case PCI_RES_BUS: device_printf(dev, "allocated bus range (%ju-%ju) for rid %d of %s\n", rman_get_start(res), rman_get_end(res), *rid, pcib_child_name(child)); break; } } return (res); } static int vmd_adjust_resource(device_t dev, device_t child, struct resource *r, rman_res_t start, rman_res_t end) { if (rman_get_type(r) == SYS_RES_IRQ) { return (bus_generic_adjust_resource(dev, child, r, start, end)); } return (bus_generic_rman_adjust_resource(dev, child, r, start, end)); } static int vmd_release_resource(device_t dev, device_t child, struct resource *r) { if (rman_get_type(r) == SYS_RES_IRQ) { return (bus_generic_release_resource(dev, child, r)); } return (bus_generic_rman_release_resource(dev, child, r)); } static int vmd_activate_resource(device_t dev, device_t child, struct resource *r) { if (rman_get_type(r) == SYS_RES_IRQ) { return (bus_generic_activate_resource(dev, child, r)); } return (bus_generic_rman_activate_resource(dev, child, r)); } static int vmd_deactivate_resource(device_t dev, device_t child, struct resource *r) { if (rman_get_type(r) == SYS_RES_IRQ) { return (bus_generic_deactivate_resource(dev, child, r)); } return (bus_generic_rman_deactivate_resource(dev, child, r)); } static struct resource * vmd_find_parent_resource(struct vmd_softc *sc, struct resource *r) { for (int i = 1; i < 3; i++) { if (rman_get_start(sc->vmd_regs_res[i]) <= rman_get_start(r) && rman_get_end(sc->vmd_regs_res[i]) >= rman_get_end(r)) return (sc->vmd_regs_res[i]); } return (NULL); } static int vmd_map_resource(device_t dev, device_t child, struct resource *r, struct resource_map_request *argsp, struct resource_map *map) { struct vmd_softc *sc = device_get_softc(dev); struct resource_map_request args; struct resource *pres; rman_res_t length, start; int error; /* Resources must be active to be mapped. */ if (!(rman_get_flags(r) & RF_ACTIVE)) return (ENXIO); resource_init_map_request(&args); error = resource_validate_map_request(r, argsp, &args, &start, &length); if (error) return (error); pres = vmd_find_parent_resource(sc, r); if (pres == NULL) return (ENOENT); args.offset = start - rman_get_start(pres); args.length = length; return (bus_map_resource(dev, pres, &args, map)); } static int vmd_unmap_resource(device_t dev, device_t child, struct resource *r, struct resource_map *map) { struct vmd_softc *sc = device_get_softc(dev); struct resource *pres; pres = vmd_find_parent_resource(sc, r); if (pres == NULL) return (ENOENT); return (bus_unmap_resource(dev, pres, map)); } static int vmd_route_interrupt(device_t dev, device_t child, int pin) { /* VMD hardware does not support legacy interrupts. */ return (PCI_INVALID_IRQ); } static int vmd_alloc_msi(device_t dev, device_t child, int count, int maxcount, int *irqs) { struct vmd_softc *sc = device_get_softc(dev); struct vmd_irq_user *u; int i, ibest = 0, best = INT_MAX; if (sc->vmd_msix_count == 0) { return (PCIB_ALLOC_MSI(device_get_parent(device_get_parent(dev)), child, count, maxcount, irqs)); } if (count > vmd_max_msi) return (ENOSPC); LIST_FOREACH(u, &sc->vmd_users, viu_link) { if (u->viu_child == child) return (EBUSY); } for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) { if (best > sc->vmd_irq[i].vi_nusers) { best = sc->vmd_irq[i].vi_nusers; ibest = i; } } u = malloc(sizeof(*u), M_DEVBUF, M_WAITOK | M_ZERO); u->viu_child = child; u->viu_vector = ibest; LIST_INSERT_HEAD(&sc->vmd_users, u, viu_link); sc->vmd_irq[ibest].vi_nusers += count; for (i = 0; i < count; i++) irqs[i] = sc->vmd_irq[ibest].vi_irq; return (0); } static int vmd_release_msi(device_t dev, device_t child, int count, int *irqs) { struct vmd_softc *sc = device_get_softc(dev); struct vmd_irq_user *u; if (sc->vmd_msix_count == 0) { return (PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)), child, count, irqs)); } LIST_FOREACH(u, &sc->vmd_users, viu_link) { if (u->viu_child == child) { sc->vmd_irq[u->viu_vector].vi_nusers -= count; LIST_REMOVE(u, viu_link); free(u, M_DEVBUF); return (0); } } return (EINVAL); } static int vmd_alloc_msix(device_t dev, device_t child, int *irq) { struct vmd_softc *sc = device_get_softc(dev); struct vmd_irq_user *u; int i, ibest = 0, best = INT_MAX; if (sc->vmd_msix_count == 0) { return (PCIB_ALLOC_MSIX(device_get_parent(device_get_parent(dev)), child, irq)); } i = 0; LIST_FOREACH(u, &sc->vmd_users, viu_link) { if (u->viu_child == child) i++; } if (i >= vmd_max_msix) return (ENOSPC); for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) { if (best > sc->vmd_irq[i].vi_nusers) { best = sc->vmd_irq[i].vi_nusers; ibest = i; } } u = malloc(sizeof(*u), M_DEVBUF, M_WAITOK | M_ZERO); u->viu_child = child; u->viu_vector = ibest; LIST_INSERT_HEAD(&sc->vmd_users, u, viu_link); sc->vmd_irq[ibest].vi_nusers++; *irq = sc->vmd_irq[ibest].vi_irq; return (0); } static int vmd_release_msix(device_t dev, device_t child, int irq) { struct vmd_softc *sc = device_get_softc(dev); struct vmd_irq_user *u; if (sc->vmd_msix_count == 0) { return (PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)), child, irq)); } LIST_FOREACH(u, &sc->vmd_users, viu_link) { if (u->viu_child == child && sc->vmd_irq[u->viu_vector].vi_irq == irq) { sc->vmd_irq[u->viu_vector].vi_nusers--; LIST_REMOVE(u, viu_link); free(u, M_DEVBUF); return (0); } } return (EINVAL); } static int vmd_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, uint32_t *data) { struct vmd_softc *sc = device_get_softc(dev); int i; if (sc->vmd_msix_count == 0) { return (PCIB_MAP_MSI(device_get_parent(device_get_parent(dev)), child, irq, addr, data)); } for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) { if (sc->vmd_irq[i].vi_irq == irq) break; } if (i >= sc->vmd_msix_count) return (EINVAL); *addr = MSI_INTEL_ADDR_BASE | (i << 12); *data = 0; return (0); } static device_method_t vmd_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, vmd_probe), DEVMETHOD(device_attach, vmd_attach), DEVMETHOD(device_detach, vmd_detach), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), DEVMETHOD(device_shutdown, bus_generic_shutdown), /* Bus interface */ DEVMETHOD(bus_get_dma_tag, vmd_get_dma_tag), DEVMETHOD(bus_get_rman, vmd_get_rman), DEVMETHOD(bus_read_ivar, pcib_read_ivar), DEVMETHOD(bus_write_ivar, pcib_write_ivar), DEVMETHOD(bus_alloc_resource, vmd_alloc_resource), DEVMETHOD(bus_adjust_resource, vmd_adjust_resource), DEVMETHOD(bus_release_resource, vmd_release_resource), DEVMETHOD(bus_activate_resource, vmd_activate_resource), DEVMETHOD(bus_deactivate_resource, vmd_deactivate_resource), DEVMETHOD(bus_map_resource, vmd_map_resource), DEVMETHOD(bus_unmap_resource, vmd_unmap_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, pcib_maxslots), DEVMETHOD(pcib_read_config, vmd_read_config), DEVMETHOD(pcib_write_config, vmd_write_config), DEVMETHOD(pcib_route_interrupt, vmd_route_interrupt), DEVMETHOD(pcib_alloc_msi, vmd_alloc_msi), DEVMETHOD(pcib_release_msi, vmd_release_msi), DEVMETHOD(pcib_alloc_msix, vmd_alloc_msix), DEVMETHOD(pcib_release_msix, vmd_release_msix), DEVMETHOD(pcib_map_msi, vmd_map_msi), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), DEVMETHOD_END }; DEFINE_CLASS_0(pcib, vmd_pci_driver, vmd_pci_methods, sizeof(struct vmd_softc)); DRIVER_MODULE(vmd, pci, vmd_pci_driver, NULL, NULL); MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmd, vmd_devs, nitems(vmd_devs) - 1);