Index: head/sys/arm/xscale/ixp425/if_npe.c =================================================================== --- head/sys/arm/xscale/ixp425/if_npe.c (revision 328016) +++ head/sys/arm/xscale/ixp425/if_npe.c (revision 328017) @@ -1,1781 +1,1782 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006-2008 Sam Leffler. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Intel XScale NPE Ethernet driver. * * This driver handles the two ports present on the IXP425. * Packet processing is done by the Network Processing Engines * (NPE's) that work together with a MAC and PHY. The MAC * is also mapped to the XScale cpu; the PHY is accessed via * the MAC. NPE-XScale communication happens through h/w * queues managed by the Q Manager block. * * The code here replaces the ethAcc, ethMii, and ethDB classes * in the Intel Access Library (IAL) and the OS-specific driver. * * XXX add vlan support */ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #include #include #include #endif #include #include #include #include #include #include #include #include #include #include #include "miibus_if.h" /* * XXX: For the main bus dma tag. Can go away if the new method to get the * dma tag from the parent got MFC'd into RELENG_6. */ extern struct ixp425_softc *ixp425_softc; struct npebuf { struct npebuf *ix_next; /* chain to next buffer */ void *ix_m; /* backpointer to mbuf */ bus_dmamap_t ix_map; /* bus dma map for associated data */ struct npehwbuf *ix_hw; /* associated h/w block */ uint32_t ix_neaddr; /* phys address of ix_hw */ }; struct npedma { const char* name; int nbuf; /* # npebuf's allocated */ bus_dma_tag_t mtag; /* bus dma tag for mbuf data */ struct npehwbuf *hwbuf; /* NPE h/w buffers */ bus_dma_tag_t buf_tag; /* tag+map for NPE buffers */ bus_dmamap_t buf_map; bus_addr_t buf_phys; /* phys addr of buffers */ struct npebuf *buf; /* s/w buffers (1-1 w/ h/w) */ }; struct npe_softc { /* XXX mii requires this be first; do not move! */ struct ifnet *sc_ifp; /* ifnet pointer */ struct mtx sc_mtx; /* basically a perimeter lock */ device_t sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; /* MAC register window */ device_t sc_mii; /* child miibus */ bus_space_handle_t sc_miih; /* MII register window */ int sc_npeid; struct ixpnpe_softc *sc_npe; /* NPE support */ int sc_debug; /* DPRINTF* control */ int sc_tickinterval; struct callout tick_ch; /* Tick callout */ int npe_watchdog_timer; struct npedma txdma; struct npebuf *tx_free; /* list of free tx buffers */ struct npedma rxdma; bus_addr_t buf_phys; /* XXX for returning a value */ int rx_qid; /* rx qid */ int rx_freeqid; /* rx free buffers qid */ int tx_qid; /* tx qid */ int tx_doneqid; /* tx completed qid */ struct ifmib_iso_8802_3 mibdata; bus_dma_tag_t sc_stats_tag; /* bus dma tag for stats block */ struct npestats *sc_stats; bus_dmamap_t sc_stats_map; bus_addr_t sc_stats_phys; /* phys addr of sc_stats */ struct npestats sc_totals; /* accumulated sc_stats */ }; /* * Static configuration for IXP425. The tx and * rx free Q id's are fixed by the NPE microcode. The * rx Q id's are programmed to be separate to simplify * multi-port processing. It may be better to handle * all traffic through one Q (as done by the Intel drivers). * * Note that the PHY's are accessible only from MAC B on the * IXP425 and from MAC C on other devices. This and other * platform-specific assumptions are handled with hints. */ static const struct { uint32_t macbase; uint32_t miibase; int phy; /* phy id */ uint8_t rx_qid; uint8_t rx_freeqid; uint8_t tx_qid; uint8_t tx_doneqid; } npeconfig[NPE_MAX] = { [NPE_A] = { .macbase = IXP435_MAC_A_HWBASE, .miibase = IXP425_MAC_C_HWBASE, .phy = 2, .rx_qid = 4, .rx_freeqid = 26, .tx_qid = 23, .tx_doneqid = 31 }, [NPE_B] = { .macbase = IXP425_MAC_B_HWBASE, .miibase = IXP425_MAC_B_HWBASE, .phy = 0, .rx_qid = 4, .rx_freeqid = 27, .tx_qid = 24, .tx_doneqid = 31 }, [NPE_C] = { .macbase = IXP425_MAC_C_HWBASE, .miibase = IXP425_MAC_B_HWBASE, .phy = 1, .rx_qid = 12, .rx_freeqid = 28, .tx_qid = 25, .tx_doneqid = 31 }, }; static struct npe_softc *npes[NPE_MAX]; /* NB: indexed by npeid */ static __inline uint32_t RD4(struct npe_softc *sc, bus_size_t off) { return bus_space_read_4(sc->sc_iot, sc->sc_ioh, off); } static __inline void WR4(struct npe_softc *sc, bus_size_t off, uint32_t val) { bus_space_write_4(sc->sc_iot, sc->sc_ioh, off, val); } #define NPE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define NPE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define NPE_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define NPE_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define NPE_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define NPE_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static devclass_t npe_devclass; static int override_npeid(device_t, const char *resname, int *val); static int npe_activate(device_t dev); static void npe_deactivate(device_t dev); static int npe_ifmedia_update(struct ifnet *ifp); static void npe_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr); static void npe_setmac(struct npe_softc *sc, u_char *eaddr); static void npe_getmac(struct npe_softc *sc, u_char *eaddr); static void npe_txdone(int qid, void *arg); static int npe_rxbuf_init(struct npe_softc *, struct npebuf *, struct mbuf *); static int npe_rxdone(int qid, void *arg); static void npeinit(void *); static void npestart_locked(struct ifnet *); static void npestart(struct ifnet *); static void npestop(struct npe_softc *); static void npewatchdog(struct npe_softc *); static int npeioctl(struct ifnet * ifp, u_long, caddr_t); static int npe_setrxqosentry(struct npe_softc *, int classix, int trafclass, int qid); static int npe_setportaddress(struct npe_softc *, const uint8_t mac[]); static int npe_setfirewallmode(struct npe_softc *, int onoff); static int npe_updatestats(struct npe_softc *); #if 0 static int npe_getstats(struct npe_softc *); static uint32_t npe_getimageid(struct npe_softc *); static int npe_setloopback(struct npe_softc *, int ena); #endif /* NB: all tx done processing goes through one queue */ static int tx_doneqid = -1; static SYSCTL_NODE(_hw, OID_AUTO, npe, CTLFLAG_RD, 0, "IXP4XX NPE driver parameters"); static int npe_debug = 0; SYSCTL_INT(_hw_npe, OID_AUTO, debug, CTLFLAG_RWTUN, &npe_debug, 0, "IXP4XX NPE network interface debug msgs"); #define DPRINTF(sc, fmt, ...) do { \ if (sc->sc_debug) device_printf(sc->sc_dev, fmt, __VA_ARGS__); \ } while (0) #define DPRINTFn(n, sc, fmt, ...) do { \ if (sc->sc_debug >= n) device_printf(sc->sc_dev, fmt, __VA_ARGS__);\ } while (0) static int npe_tickinterval = 3; /* npe_tick frequency (secs) */ SYSCTL_INT(_hw_npe, OID_AUTO, tickinterval, CTLFLAG_RDTUN, &npe_tickinterval, 0, "periodic work interval (secs)"); static int npe_rxbuf = 64; /* # rx buffers to allocate */ SYSCTL_INT(_hw_npe, OID_AUTO, rxbuf, CTLFLAG_RDTUN, &npe_rxbuf, 0, "rx buffers allocated"); static int npe_txbuf = 128; /* # tx buffers to allocate */ SYSCTL_INT(_hw_npe, OID_AUTO, txbuf, CTLFLAG_RDTUN, &npe_txbuf, 0, "tx buffers allocated"); static int unit2npeid(int unit) { static const int npeidmap[2][3] = { /* on 425 A is for HSS, B & C are for Ethernet */ { NPE_B, NPE_C, -1 }, /* IXP425 */ /* 435 only has A & C, order C then A */ { NPE_C, NPE_A, -1 }, /* IXP435 */ }; /* XXX check feature register instead */ return (unit < 3 ? npeidmap[ (cpu_ident() & CPU_ID_CPU_MASK) == CPU_ID_IXP435][unit] : -1); } static int npe_probe(device_t dev) { static const char *desc[NPE_MAX] = { [NPE_A] = "IXP NPE-A", [NPE_B] = "IXP NPE-B", [NPE_C] = "IXP NPE-C" }; int unit = device_get_unit(dev); int npeid; if (unit > 2 || (ixp4xx_read_feature_bits() & (unit == 0 ? EXP_FCTRL_ETH0 : EXP_FCTRL_ETH1)) == 0) return EINVAL; npeid = -1; if (!override_npeid(dev, "npeid", &npeid)) npeid = unit2npeid(unit); if (npeid == -1) { device_printf(dev, "unit %d not supported\n", unit); return EINVAL; } device_set_desc(dev, desc[npeid]); return 0; } static int npe_attach(device_t dev) { struct npe_softc *sc = device_get_softc(dev); struct ixp425_softc *sa = device_get_softc(device_get_parent(dev)); struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid *tree = device_get_sysctl_tree(dev); struct ifnet *ifp; int error; u_char eaddr[6]; sc->sc_dev = dev; sc->sc_iot = sa->sc_iot; NPE_LOCK_INIT(sc); callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0); sc->sc_debug = npe_debug; sc->sc_tickinterval = npe_tickinterval; ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "cannot allocate ifnet\n"); error = EIO; /* XXX */ goto out; } /* NB: must be setup prior to invoking mii code */ sc->sc_ifp = ifp; error = npe_activate(dev); if (error) { device_printf(dev, "cannot activate npe\n"); goto out; } npe_getmac(sc, eaddr); ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_start = npestart; ifp->if_ioctl = npeioctl; ifp->if_init = npeinit; IFQ_SET_MAXLEN(&ifp->if_snd, sc->txdma.nbuf - 1); ifp->if_snd.ifq_drv_maxlen = ifqmaxlen; IFQ_SET_READY(&ifp->if_snd); ifp->if_linkmib = &sc->mibdata; ifp->if_linkmiblen = sizeof(sc->mibdata); sc->mibdata.dot3Compliance = DOT3COMPLIANCE_STATS; /* device supports oversided vlan frames */ ifp->if_capabilities |= IFCAP_VLAN_MTU; ifp->if_capenable = ifp->if_capabilities; #ifdef DEVICE_POLLING ifp->if_capabilities |= IFCAP_POLLING; #endif SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "debug", CTLFLAG_RW, &sc->sc_debug, 0, "control debugging printfs"); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tickinterval", CTLFLAG_RW, &sc->sc_tickinterval, 0, "periodic work frequency"); SYSCTL_ADD_STRUCT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "stats", CTLFLAG_RD, &sc->sc_totals, npestats, "onboard stats"); ether_ifattach(ifp, eaddr); return 0; out: if (ifp != NULL) if_free(ifp); NPE_LOCK_DESTROY(sc); npe_deactivate(dev); return error; } static int npe_detach(device_t dev) { struct npe_softc *sc = device_get_softc(dev); struct ifnet *ifp = sc->sc_ifp; #ifdef DEVICE_POLLING if (ifp->if_capenable & IFCAP_POLLING) ether_poll_deregister(ifp); #endif npestop(sc); if (ifp != NULL) { ether_ifdetach(ifp); if_free(ifp); } NPE_LOCK_DESTROY(sc); npe_deactivate(dev); return 0; } /* * Compute and install the multicast filter. */ static void npe_setmcast(struct npe_softc *sc) { struct ifnet *ifp = sc->sc_ifp; uint8_t mask[ETHER_ADDR_LEN], addr[ETHER_ADDR_LEN]; int i; if (ifp->if_flags & IFF_PROMISC) { memset(mask, 0, ETHER_ADDR_LEN); memset(addr, 0, ETHER_ADDR_LEN); } else if (ifp->if_flags & IFF_ALLMULTI) { static const uint8_t allmulti[ETHER_ADDR_LEN] = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 }; memcpy(mask, allmulti, ETHER_ADDR_LEN); memcpy(addr, allmulti, ETHER_ADDR_LEN); } else { uint8_t clr[ETHER_ADDR_LEN], set[ETHER_ADDR_LEN]; struct ifmultiaddr *ifma; const uint8_t *mac; memset(clr, 0, ETHER_ADDR_LEN); memset(set, 0xff, ETHER_ADDR_LEN); if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; mac = LLADDR((struct sockaddr_dl *) ifma->ifma_addr); for (i = 0; i < ETHER_ADDR_LEN; i++) { clr[i] |= mac[i]; set[i] &= mac[i]; } } if_maddr_runlock(ifp); for (i = 0; i < ETHER_ADDR_LEN; i++) { mask[i] = set[i] | ~clr[i]; addr[i] = set[i]; } } /* * Write the mask and address registers. */ for (i = 0; i < ETHER_ADDR_LEN; i++) { WR4(sc, NPE_MAC_ADDR_MASK(i), mask[i]); WR4(sc, NPE_MAC_ADDR(i), addr[i]); } } static void npe_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct npe_softc *sc; if (error != 0) return; sc = (struct npe_softc *)arg; sc->buf_phys = segs[0].ds_addr; } static int npe_dma_setup(struct npe_softc *sc, struct npedma *dma, const char *name, int nbuf, int maxseg) { int error, i; memset(dma, 0, sizeof(*dma)); dma->name = name; dma->nbuf = nbuf; /* DMA tag for mapped mbufs */ error = bus_dma_tag_create(ixp425_softc->sc_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, maxseg, MCLBYTES, 0, busdma_lock_mutex, &sc->sc_mtx, &dma->mtag); if (error != 0) { device_printf(sc->sc_dev, "unable to create %s mbuf dma tag, " "error %u\n", dma->name, error); return error; } /* DMA tag and map for the NPE buffers */ error = bus_dma_tag_create(ixp425_softc->sc_dmat, sizeof(uint32_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, nbuf * sizeof(struct npehwbuf), 1, nbuf * sizeof(struct npehwbuf), 0, busdma_lock_mutex, &sc->sc_mtx, &dma->buf_tag); if (error != 0) { device_printf(sc->sc_dev, "unable to create %s npebuf dma tag, error %u\n", dma->name, error); return error; } if (bus_dmamem_alloc(dma->buf_tag, (void **)&dma->hwbuf, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &dma->buf_map) != 0) { device_printf(sc->sc_dev, "unable to allocate memory for %s h/w buffers, error %u\n", dma->name, error); return error; } /* XXX M_TEMP */ - dma->buf = malloc(nbuf * sizeof(struct npebuf), M_TEMP, M_NOWAIT | M_ZERO); + dma->buf = mallocarray(nbuf, sizeof(struct npebuf), M_TEMP, + M_NOWAIT | M_ZERO); if (dma->buf == NULL) { device_printf(sc->sc_dev, "unable to allocate memory for %s s/w buffers\n", dma->name); return error; } if (bus_dmamap_load(dma->buf_tag, dma->buf_map, dma->hwbuf, nbuf*sizeof(struct npehwbuf), npe_getaddr, sc, 0) != 0) { device_printf(sc->sc_dev, "unable to map memory for %s h/w buffers, error %u\n", dma->name, error); return error; } dma->buf_phys = sc->buf_phys; for (i = 0; i < dma->nbuf; i++) { struct npebuf *npe = &dma->buf[i]; struct npehwbuf *hw = &dma->hwbuf[i]; /* calculate offset to shared area */ npe->ix_neaddr = dma->buf_phys + ((uintptr_t)hw - (uintptr_t)dma->hwbuf); KASSERT((npe->ix_neaddr & 0x1f) == 0, ("ixpbuf misaligned, PA 0x%x", npe->ix_neaddr)); error = bus_dmamap_create(dma->mtag, BUS_DMA_NOWAIT, &npe->ix_map); if (error != 0) { device_printf(sc->sc_dev, "unable to create dmamap for %s buffer %u, " "error %u\n", dma->name, i, error); return error; } npe->ix_hw = hw; } bus_dmamap_sync(dma->buf_tag, dma->buf_map, BUS_DMASYNC_PREWRITE); return 0; } static void npe_dma_destroy(struct npe_softc *sc, struct npedma *dma) { int i; if (dma->hwbuf != NULL) { for (i = 0; i < dma->nbuf; i++) { struct npebuf *npe = &dma->buf[i]; bus_dmamap_destroy(dma->mtag, npe->ix_map); } bus_dmamap_unload(dma->buf_tag, dma->buf_map); bus_dmamem_free(dma->buf_tag, dma->hwbuf, dma->buf_map); } if (dma->buf != NULL) free(dma->buf, M_TEMP); if (dma->buf_tag) bus_dma_tag_destroy(dma->buf_tag); if (dma->mtag) bus_dma_tag_destroy(dma->mtag); memset(dma, 0, sizeof(*dma)); } static int override_addr(device_t dev, const char *resname, int *base) { int unit = device_get_unit(dev); const char *resval; /* XXX warn for wrong hint type */ if (resource_string_value("npe", unit, resname, &resval) != 0) return 0; switch (resval[0]) { case 'A': *base = IXP435_MAC_A_HWBASE; break; case 'B': *base = IXP425_MAC_B_HWBASE; break; case 'C': *base = IXP425_MAC_C_HWBASE; break; default: device_printf(dev, "Warning, bad value %s for " "npe.%d.%s ignored\n", resval, unit, resname); return 0; } if (bootverbose) device_printf(dev, "using npe.%d.%s=%s override\n", unit, resname, resval); return 1; } static int override_npeid(device_t dev, const char *resname, int *npeid) { int unit = device_get_unit(dev); const char *resval; /* XXX warn for wrong hint type */ if (resource_string_value("npe", unit, resname, &resval) != 0) return 0; switch (resval[0]) { case 'A': *npeid = NPE_A; break; case 'B': *npeid = NPE_B; break; case 'C': *npeid = NPE_C; break; default: device_printf(dev, "Warning, bad value %s for " "npe.%d.%s ignored\n", resval, unit, resname); return 0; } if (bootverbose) device_printf(dev, "using npe.%d.%s=%s override\n", unit, resname, resval); return 1; } static int override_unit(device_t dev, const char *resname, int *val, int min, int max) { int unit = device_get_unit(dev); int resval; if (resource_int_value("npe", unit, resname, &resval) != 0) return 0; if (!(min <= resval && resval <= max)) { device_printf(dev, "Warning, bad value %d for npe.%d.%s " "ignored (value must be [%d-%d])\n", resval, unit, resname, min, max); return 0; } if (bootverbose) device_printf(dev, "using npe.%d.%s=%d override\n", unit, resname, resval); *val = resval; return 1; } static void npe_mac_reset(struct npe_softc *sc) { /* * Reset MAC core. */ WR4(sc, NPE_MAC_CORE_CNTRL, NPE_CORE_RESET); DELAY(NPE_MAC_RESET_DELAY); /* configure MAC to generate MDC clock */ WR4(sc, NPE_MAC_CORE_CNTRL, NPE_CORE_MDC_EN); } static int npe_activate(device_t dev) { struct npe_softc *sc = device_get_softc(dev); int error, i, macbase, miibase, phy; /* * Setup NEP ID, MAC, and MII bindings. We allow override * via hints to handle unexpected board configs. */ if (!override_npeid(dev, "npeid", &sc->sc_npeid)) sc->sc_npeid = unit2npeid(device_get_unit(dev)); sc->sc_npe = ixpnpe_attach(dev, sc->sc_npeid); if (sc->sc_npe == NULL) { device_printf(dev, "cannot attach ixpnpe\n"); return EIO; /* XXX */ } /* MAC */ if (!override_addr(dev, "mac", &macbase)) macbase = npeconfig[sc->sc_npeid].macbase; if (bootverbose) device_printf(sc->sc_dev, "MAC at 0x%x\n", macbase); if (bus_space_map(sc->sc_iot, macbase, IXP425_REG_SIZE, 0, &sc->sc_ioh)) { device_printf(dev, "cannot map mac registers 0x%x:0x%x\n", macbase, IXP425_REG_SIZE); return ENOMEM; } /* PHY */ if (!override_unit(dev, "phy", &phy, 0, MII_NPHY - 1)) phy = npeconfig[sc->sc_npeid].phy; if (!override_addr(dev, "mii", &miibase)) miibase = npeconfig[sc->sc_npeid].miibase; if (bootverbose) device_printf(sc->sc_dev, "MII at 0x%x\n", miibase); if (miibase != macbase) { /* * PHY is mapped through a different MAC, setup an * additional mapping for frobbing the PHY registers. */ if (bus_space_map(sc->sc_iot, miibase, IXP425_REG_SIZE, 0, &sc->sc_miih)) { device_printf(dev, "cannot map MII registers 0x%x:0x%x\n", miibase, IXP425_REG_SIZE); return ENOMEM; } } else sc->sc_miih = sc->sc_ioh; /* * Load NPE firmware and start it running. */ error = ixpnpe_init(sc->sc_npe); if (error != 0) { device_printf(dev, "cannot init NPE (error %d)\n", error); return error; } /* attach PHY */ error = mii_attach(dev, &sc->sc_mii, sc->sc_ifp, npe_ifmedia_update, npe_ifmedia_status, BMSR_DEFCAPMASK, phy, MII_OFFSET_ANY, 0); if (error != 0) { device_printf(dev, "attaching PHYs failed\n"); return error; } error = npe_dma_setup(sc, &sc->txdma, "tx", npe_txbuf, NPE_MAXSEG); if (error != 0) return error; error = npe_dma_setup(sc, &sc->rxdma, "rx", npe_rxbuf, 1); if (error != 0) return error; /* setup statistics block */ error = bus_dma_tag_create(ixp425_softc->sc_dmat, sizeof(uint32_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, sizeof(struct npestats), 1, sizeof(struct npestats), 0, busdma_lock_mutex, &sc->sc_mtx, &sc->sc_stats_tag); if (error != 0) { device_printf(sc->sc_dev, "unable to create stats tag, " "error %u\n", error); return error; } if (bus_dmamem_alloc(sc->sc_stats_tag, (void **)&sc->sc_stats, BUS_DMA_NOWAIT, &sc->sc_stats_map) != 0) { device_printf(sc->sc_dev, "unable to allocate memory for stats block, error %u\n", error); return error; } if (bus_dmamap_load(sc->sc_stats_tag, sc->sc_stats_map, sc->sc_stats, sizeof(struct npestats), npe_getaddr, sc, 0) != 0) { device_printf(sc->sc_dev, "unable to load memory for stats block, error %u\n", error); return error; } sc->sc_stats_phys = sc->buf_phys; /* * Setup h/w rx/tx queues. There are four q's: * rx inbound q of rx'd frames * rx_free pool of ixpbuf's for receiving frames * tx outbound q of frames to send * tx_done q of tx frames that have been processed * * The NPE handles the actual tx/rx process and the q manager * handles the queues. The driver just writes entries to the * q manager mailbox's and gets callbacks when there are rx'd * frames to process or tx'd frames to reap. These callbacks * are controlled by the q configurations; e.g. we get a * callback when tx_done has 2 or more frames to process and * when the rx q has at least one frame. These setings can * changed at the time the q is configured. */ sc->rx_qid = npeconfig[sc->sc_npeid].rx_qid; ixpqmgr_qconfig(sc->rx_qid, npe_rxbuf, 0, 1, IX_QMGR_Q_SOURCE_ID_NOT_E, (qconfig_hand_t *)npe_rxdone, sc); sc->rx_freeqid = npeconfig[sc->sc_npeid].rx_freeqid; ixpqmgr_qconfig(sc->rx_freeqid, npe_rxbuf, 0, npe_rxbuf/2, 0, NULL, sc); /* * Setup the NPE to direct all traffic to rx_qid. * When QoS is enabled in the firmware there are * 8 traffic classes; otherwise just 4. */ for (i = 0; i < 8; i++) npe_setrxqosentry(sc, i, 0, sc->rx_qid); /* disable firewall mode just in case (should be off) */ npe_setfirewallmode(sc, 0); sc->tx_qid = npeconfig[sc->sc_npeid].tx_qid; sc->tx_doneqid = npeconfig[sc->sc_npeid].tx_doneqid; ixpqmgr_qconfig(sc->tx_qid, npe_txbuf, 0, npe_txbuf, 0, NULL, sc); if (tx_doneqid == -1) { ixpqmgr_qconfig(sc->tx_doneqid, npe_txbuf, 0, 2, IX_QMGR_Q_SOURCE_ID_NOT_E, npe_txdone, sc); tx_doneqid = sc->tx_doneqid; } KASSERT(npes[sc->sc_npeid] == NULL, ("npe %u already setup", sc->sc_npeid)); npes[sc->sc_npeid] = sc; return 0; } static void npe_deactivate(device_t dev) { struct npe_softc *sc = device_get_softc(dev); npes[sc->sc_npeid] = NULL; /* XXX disable q's */ if (sc->sc_npe != NULL) { ixpnpe_stop(sc->sc_npe); ixpnpe_detach(sc->sc_npe); } if (sc->sc_stats != NULL) { bus_dmamap_unload(sc->sc_stats_tag, sc->sc_stats_map); bus_dmamem_free(sc->sc_stats_tag, sc->sc_stats, sc->sc_stats_map); } if (sc->sc_stats_tag != NULL) bus_dma_tag_destroy(sc->sc_stats_tag); npe_dma_destroy(sc, &sc->txdma); npe_dma_destroy(sc, &sc->rxdma); bus_generic_detach(sc->sc_dev); if (sc->sc_mii != NULL) device_delete_child(sc->sc_dev, sc->sc_mii); } /* * Change media according to request. */ static int npe_ifmedia_update(struct ifnet *ifp) { struct npe_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->sc_mii); NPE_LOCK(sc); mii_mediachg(mii); /* XXX push state ourself? */ NPE_UNLOCK(sc); return (0); } /* * Notify the world which media we're using. */ static void npe_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct npe_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->sc_mii); NPE_LOCK(sc); mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; NPE_UNLOCK(sc); } static void npe_addstats(struct npe_softc *sc) { #define NPEADD(x) sc->sc_totals.x += be32toh(ns->x) #define MIBADD(x) do { sc->mibdata.x += be32toh(ns->x); NPEADD(x); } while (0) struct ifnet *ifp = sc->sc_ifp; struct npestats *ns = sc->sc_stats; MIBADD(dot3StatsAlignmentErrors); MIBADD(dot3StatsFCSErrors); MIBADD(dot3StatsInternalMacReceiveErrors); NPEADD(RxOverrunDiscards); NPEADD(RxLearnedEntryDiscards); NPEADD(RxLargeFramesDiscards); NPEADD(RxSTPBlockedDiscards); NPEADD(RxVLANTypeFilterDiscards); NPEADD(RxVLANIdFilterDiscards); NPEADD(RxInvalidSourceDiscards); NPEADD(RxBlackListDiscards); NPEADD(RxWhiteListDiscards); NPEADD(RxUnderflowEntryDiscards); MIBADD(dot3StatsSingleCollisionFrames); MIBADD(dot3StatsMultipleCollisionFrames); MIBADD(dot3StatsDeferredTransmissions); MIBADD(dot3StatsLateCollisions); MIBADD(dot3StatsExcessiveCollisions); MIBADD(dot3StatsInternalMacTransmitErrors); MIBADD(dot3StatsCarrierSenseErrors); NPEADD(TxLargeFrameDiscards); NPEADD(TxVLANIdFilterDiscards); sc->mibdata.dot3StatsFrameTooLongs += be32toh(ns->RxLargeFramesDiscards) + be32toh(ns->TxLargeFrameDiscards); sc->mibdata.dot3StatsMissedFrames += be32toh(ns->RxOverrunDiscards) + be32toh(ns->RxUnderflowEntryDiscards); if_inc_counter(ifp, IFCOUNTER_OERRORS, be32toh(ns->dot3StatsInternalMacTransmitErrors) + be32toh(ns->dot3StatsCarrierSenseErrors) + be32toh(ns->TxVLANIdFilterDiscards)); if_inc_counter(ifp, IFCOUNTER_IERRORS, be32toh(ns->dot3StatsFCSErrors) + be32toh(ns->dot3StatsInternalMacReceiveErrors) + be32toh(ns->RxOverrunDiscards) + be32toh(ns->RxUnderflowEntryDiscards)); if_inc_counter(ifp, IFCOUNTER_COLLISIONS, be32toh(ns->dot3StatsSingleCollisionFrames) + be32toh(ns->dot3StatsMultipleCollisionFrames)); #undef NPEADD #undef MIBADD } static void npe_tick(void *xsc) { #define ACK (NPE_RESETSTATS << NPE_MAC_MSGID_SHL) struct npe_softc *sc = xsc; struct mii_data *mii = device_get_softc(sc->sc_mii); uint32_t msg[2]; NPE_ASSERT_LOCKED(sc); /* * NB: to avoid sleeping with the softc lock held we * split the NPE msg processing into two parts. The * request for statistics is sent w/o waiting for a * reply and then on the next tick we retrieve the * results. This works because npe_tick is the only * code that talks via the mailbox's (except at setup). * This likely can be handled better. */ if (ixpnpe_recvmsg_async(sc->sc_npe, msg) == 0 && msg[0] == ACK) { bus_dmamap_sync(sc->sc_stats_tag, sc->sc_stats_map, BUS_DMASYNC_POSTREAD); npe_addstats(sc); } npe_updatestats(sc); mii_tick(mii); npewatchdog(sc); /* schedule next poll */ callout_reset(&sc->tick_ch, sc->sc_tickinterval * hz, npe_tick, sc); #undef ACK } static void npe_setmac(struct npe_softc *sc, u_char *eaddr) { WR4(sc, NPE_MAC_UNI_ADDR_1, eaddr[0]); WR4(sc, NPE_MAC_UNI_ADDR_2, eaddr[1]); WR4(sc, NPE_MAC_UNI_ADDR_3, eaddr[2]); WR4(sc, NPE_MAC_UNI_ADDR_4, eaddr[3]); WR4(sc, NPE_MAC_UNI_ADDR_5, eaddr[4]); WR4(sc, NPE_MAC_UNI_ADDR_6, eaddr[5]); } static void npe_getmac(struct npe_softc *sc, u_char *eaddr) { /* NB: the unicast address appears to be loaded from EEPROM on reset */ eaddr[0] = RD4(sc, NPE_MAC_UNI_ADDR_1) & 0xff; eaddr[1] = RD4(sc, NPE_MAC_UNI_ADDR_2) & 0xff; eaddr[2] = RD4(sc, NPE_MAC_UNI_ADDR_3) & 0xff; eaddr[3] = RD4(sc, NPE_MAC_UNI_ADDR_4) & 0xff; eaddr[4] = RD4(sc, NPE_MAC_UNI_ADDR_5) & 0xff; eaddr[5] = RD4(sc, NPE_MAC_UNI_ADDR_6) & 0xff; } struct txdone { struct npebuf *head; struct npebuf **tail; int count; }; static __inline void npe_txdone_finish(struct npe_softc *sc, const struct txdone *td) { struct ifnet *ifp = sc->sc_ifp; NPE_LOCK(sc); *td->tail = sc->tx_free; sc->tx_free = td->head; /* * We're no longer busy, so clear the busy flag and call the * start routine to xmit more packets. */ if_inc_counter(ifp, IFCOUNTER_OPACKETS, td->count); ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sc->npe_watchdog_timer = 0; npestart_locked(ifp); NPE_UNLOCK(sc); } /* * Q manager callback on tx done queue. Reap mbufs * and return tx buffers to the free list. Finally * restart output. Note the microcode has only one * txdone q wired into it so we must use the NPE ID * returned with each npehwbuf to decide where to * send buffers. */ static void npe_txdone(int qid, void *arg) { #define P2V(a, dma) \ &(dma)->buf[((a) - (dma)->buf_phys) / sizeof(struct npehwbuf)] struct npe_softc *sc0 = arg; struct npe_softc *sc; struct npebuf *npe; struct txdone *td, q[NPE_MAX]; uint32_t entry; q[NPE_A].tail = &q[NPE_A].head; q[NPE_A].count = 0; q[NPE_B].tail = &q[NPE_B].head; q[NPE_B].count = 0; q[NPE_C].tail = &q[NPE_C].head; q[NPE_C].count = 0; /* XXX max # at a time? */ while (ixpqmgr_qread(qid, &entry) == 0) { DPRINTF(sc0, "%s: entry 0x%x NPE %u port %u\n", __func__, entry, NPE_QM_Q_NPE(entry), NPE_QM_Q_PORT(entry)); sc = npes[NPE_QM_Q_NPE(entry)]; npe = P2V(NPE_QM_Q_ADDR(entry), &sc->txdma); m_freem(npe->ix_m); npe->ix_m = NULL; td = &q[NPE_QM_Q_NPE(entry)]; *td->tail = npe; td->tail = &npe->ix_next; td->count++; } if (q[NPE_A].count) npe_txdone_finish(npes[NPE_A], &q[NPE_A]); if (q[NPE_B].count) npe_txdone_finish(npes[NPE_B], &q[NPE_B]); if (q[NPE_C].count) npe_txdone_finish(npes[NPE_C], &q[NPE_C]); #undef P2V } static int npe_rxbuf_init(struct npe_softc *sc, struct npebuf *npe, struct mbuf *m) { bus_dma_segment_t segs[1]; struct npedma *dma = &sc->rxdma; struct npehwbuf *hw; int error, nseg; if (m == NULL) { m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (m == NULL) return ENOBUFS; } KASSERT(m->m_ext.ext_size >= 1536 + ETHER_ALIGN, ("ext_size %d", m->m_ext.ext_size)); m->m_pkthdr.len = m->m_len = 1536; /* backload payload and align ip hdr */ m->m_data = m->m_ext.ext_buf + (m->m_ext.ext_size - (1536+ETHER_ALIGN)); bus_dmamap_unload(dma->mtag, npe->ix_map); error = bus_dmamap_load_mbuf_sg(dma->mtag, npe->ix_map, m, segs, &nseg, 0); if (error != 0) { m_freem(m); return error; } hw = npe->ix_hw; hw->ix_ne[0].data = htobe32(segs[0].ds_addr); /* NB: NPE requires length be a multiple of 64 */ /* NB: buffer length is shifted in word */ hw->ix_ne[0].len = htobe32(segs[0].ds_len << 16); hw->ix_ne[0].next = 0; bus_dmamap_sync(dma->buf_tag, dma->buf_map, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); npe->ix_m = m; /* Flush the memory in the mbuf */ bus_dmamap_sync(dma->mtag, npe->ix_map, BUS_DMASYNC_PREREAD); return 0; } /* * RX q processing for a specific NPE. Claim entries * from the hardware queue and pass the frames up the * stack. Pass the rx buffers to the free list. */ static int npe_rxdone(int qid, void *arg) { #define P2V(a, dma) \ &(dma)->buf[((a) - (dma)->buf_phys) / sizeof(struct npehwbuf)] struct npe_softc *sc = arg; struct npedma *dma = &sc->rxdma; uint32_t entry; int rx_npkts = 0; while (ixpqmgr_qread(qid, &entry) == 0) { struct npebuf *npe = P2V(NPE_QM_Q_ADDR(entry), dma); struct mbuf *m; bus_dmamap_sync(dma->buf_tag, dma->buf_map, BUS_DMASYNC_POSTREAD); DPRINTF(sc, "%s: entry 0x%x neaddr 0x%x ne_len 0x%x\n", __func__, entry, npe->ix_neaddr, npe->ix_hw->ix_ne[0].len); /* * Allocate a new mbuf to replenish the rx buffer. * If doing so fails we drop the rx'd frame so we * can reuse the previous mbuf. When we're able to * allocate a new mbuf dispatch the mbuf w/ rx'd * data up the stack and replace it with the newly * allocated one. */ m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (m != NULL) { struct mbuf *mrx = npe->ix_m; struct npehwbuf *hw = npe->ix_hw; struct ifnet *ifp = sc->sc_ifp; /* Flush mbuf memory for rx'd data */ bus_dmamap_sync(dma->mtag, npe->ix_map, BUS_DMASYNC_POSTREAD); /* set m_len etc. per rx frame size */ mrx->m_len = be32toh(hw->ix_ne[0].len) & 0xffff; mrx->m_pkthdr.len = mrx->m_len; mrx->m_pkthdr.rcvif = ifp; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); ifp->if_input(ifp, mrx); rx_npkts++; } else { /* discard frame and re-use mbuf */ m = npe->ix_m; } if (npe_rxbuf_init(sc, npe, m) == 0) { /* return npe buf to rx free list */ ixpqmgr_qwrite(sc->rx_freeqid, npe->ix_neaddr); } else { /* XXX should not happen */ } } return rx_npkts; #undef P2V } #ifdef DEVICE_POLLING static int npe_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { struct npe_softc *sc = ifp->if_softc; int rx_npkts = 0; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { rx_npkts = npe_rxdone(sc->rx_qid, sc); npe_txdone(sc->tx_doneqid, sc); /* XXX polls both NPE's */ } return rx_npkts; } #endif /* DEVICE_POLLING */ static void npe_startxmit(struct npe_softc *sc) { struct npedma *dma = &sc->txdma; int i; NPE_ASSERT_LOCKED(sc); sc->tx_free = NULL; for (i = 0; i < dma->nbuf; i++) { struct npebuf *npe = &dma->buf[i]; if (npe->ix_m != NULL) { /* NB: should not happen */ device_printf(sc->sc_dev, "%s: free mbuf at entry %u\n", __func__, i); m_freem(npe->ix_m); } npe->ix_m = NULL; npe->ix_next = sc->tx_free; sc->tx_free = npe; } } static void npe_startrecv(struct npe_softc *sc) { struct npedma *dma = &sc->rxdma; struct npebuf *npe; int i; NPE_ASSERT_LOCKED(sc); for (i = 0; i < dma->nbuf; i++) { npe = &dma->buf[i]; npe_rxbuf_init(sc, npe, npe->ix_m); /* set npe buf on rx free list */ ixpqmgr_qwrite(sc->rx_freeqid, npe->ix_neaddr); } } /* * Reset and initialize the chip */ static void npeinit_locked(void *xsc) { struct npe_softc *sc = xsc; struct ifnet *ifp = sc->sc_ifp; NPE_ASSERT_LOCKED(sc); if (ifp->if_drv_flags & IFF_DRV_RUNNING) return;/*XXX*/ /* * Reset MAC core. */ npe_mac_reset(sc); /* disable transmitter and reciver in the MAC */ WR4(sc, NPE_MAC_RX_CNTRL1, RD4(sc, NPE_MAC_RX_CNTRL1) &~ NPE_RX_CNTRL1_RX_EN); WR4(sc, NPE_MAC_TX_CNTRL1, RD4(sc, NPE_MAC_TX_CNTRL1) &~ NPE_TX_CNTRL1_TX_EN); /* * Set the MAC core registers. */ WR4(sc, NPE_MAC_INT_CLK_THRESH, 0x1); /* clock ratio: for ipx4xx */ WR4(sc, NPE_MAC_TX_CNTRL2, 0xf); /* max retries */ WR4(sc, NPE_MAC_RANDOM_SEED, 0x8); /* LFSR back-off seed */ /* thresholds determined by NPE firmware FS */ WR4(sc, NPE_MAC_THRESH_P_EMPTY, 0x12); WR4(sc, NPE_MAC_THRESH_P_FULL, 0x30); WR4(sc, NPE_MAC_BUF_SIZE_TX, 0x8); /* tx fifo threshold (bytes) */ WR4(sc, NPE_MAC_TX_DEFER, 0x15); /* for single deferral */ WR4(sc, NPE_MAC_RX_DEFER, 0x16); /* deferral on inter-frame gap*/ WR4(sc, NPE_MAC_TX_TWO_DEFER_1, 0x8); /* for 2-part deferral */ WR4(sc, NPE_MAC_TX_TWO_DEFER_2, 0x7); /* for 2-part deferral */ WR4(sc, NPE_MAC_SLOT_TIME, 0x80); /* assumes MII mode */ WR4(sc, NPE_MAC_TX_CNTRL1, NPE_TX_CNTRL1_RETRY /* retry failed xmits */ | NPE_TX_CNTRL1_FCS_EN /* append FCS */ | NPE_TX_CNTRL1_2DEFER /* 2-part deferal */ | NPE_TX_CNTRL1_PAD_EN); /* pad runt frames */ /* XXX pad strip? */ /* ena pause frame handling */ WR4(sc, NPE_MAC_RX_CNTRL1, NPE_RX_CNTRL1_PAUSE_EN); WR4(sc, NPE_MAC_RX_CNTRL2, 0); npe_setmac(sc, IF_LLADDR(ifp)); npe_setportaddress(sc, IF_LLADDR(ifp)); npe_setmcast(sc); npe_startxmit(sc); npe_startrecv(sc); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sc->npe_watchdog_timer = 0; /* just in case */ /* enable transmitter and reciver in the MAC */ WR4(sc, NPE_MAC_RX_CNTRL1, RD4(sc, NPE_MAC_RX_CNTRL1) | NPE_RX_CNTRL1_RX_EN); WR4(sc, NPE_MAC_TX_CNTRL1, RD4(sc, NPE_MAC_TX_CNTRL1) | NPE_TX_CNTRL1_TX_EN); callout_reset(&sc->tick_ch, sc->sc_tickinterval * hz, npe_tick, sc); } static void npeinit(void *xsc) { struct npe_softc *sc = xsc; NPE_LOCK(sc); npeinit_locked(sc); NPE_UNLOCK(sc); } /* * Dequeue packets and place on the h/w transmit queue. */ static void npestart_locked(struct ifnet *ifp) { struct npe_softc *sc = ifp->if_softc; struct npebuf *npe; struct npehwbuf *hw; struct mbuf *m, *n; struct npedma *dma = &sc->txdma; bus_dma_segment_t segs[NPE_MAXSEG]; int nseg, len, error, i; uint32_t next; NPE_ASSERT_LOCKED(sc); /* XXX can this happen? */ if (ifp->if_drv_flags & IFF_DRV_OACTIVE) return; while (sc->tx_free != NULL) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) { /* XXX? */ ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; return; } npe = sc->tx_free; bus_dmamap_unload(dma->mtag, npe->ix_map); error = bus_dmamap_load_mbuf_sg(dma->mtag, npe->ix_map, m, segs, &nseg, 0); if (error == EFBIG) { n = m_collapse(m, M_NOWAIT, NPE_MAXSEG); if (n == NULL) { if_printf(ifp, "%s: too many fragments %u\n", __func__, nseg); m_freem(m); return; /* XXX? */ } m = n; error = bus_dmamap_load_mbuf_sg(dma->mtag, npe->ix_map, m, segs, &nseg, 0); } if (error != 0 || nseg == 0) { if_printf(ifp, "%s: error %u nseg %u\n", __func__, error, nseg); m_freem(m); return; /* XXX? */ } sc->tx_free = npe->ix_next; bus_dmamap_sync(dma->mtag, npe->ix_map, BUS_DMASYNC_PREWRITE); /* * Tap off here if there is a bpf listener. */ BPF_MTAP(ifp, m); npe->ix_m = m; hw = npe->ix_hw; len = m->m_pkthdr.len; next = npe->ix_neaddr + sizeof(hw->ix_ne[0]); for (i = 0; i < nseg; i++) { hw->ix_ne[i].data = htobe32(segs[i].ds_addr); hw->ix_ne[i].len = htobe32((segs[i].ds_len<<16) | len); hw->ix_ne[i].next = htobe32(next); len = 0; /* zero for segments > 1 */ next += sizeof(hw->ix_ne[0]); } hw->ix_ne[i-1].next = 0; /* zero last in chain */ bus_dmamap_sync(dma->buf_tag, dma->buf_map, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); DPRINTF(sc, "%s: qwrite(%u, 0x%x) ne_data %x ne_len 0x%x\n", __func__, sc->tx_qid, npe->ix_neaddr, hw->ix_ne[0].data, hw->ix_ne[0].len); /* stick it on the tx q */ /* XXX add vlan priority */ ixpqmgr_qwrite(sc->tx_qid, npe->ix_neaddr); sc->npe_watchdog_timer = 5; } if (sc->tx_free == NULL) ifp->if_drv_flags |= IFF_DRV_OACTIVE; } void npestart(struct ifnet *ifp) { struct npe_softc *sc = ifp->if_softc; NPE_LOCK(sc); npestart_locked(ifp); NPE_UNLOCK(sc); } static void npe_stopxmit(struct npe_softc *sc) { struct npedma *dma = &sc->txdma; int i; NPE_ASSERT_LOCKED(sc); /* XXX qmgr */ for (i = 0; i < dma->nbuf; i++) { struct npebuf *npe = &dma->buf[i]; if (npe->ix_m != NULL) { bus_dmamap_unload(dma->mtag, npe->ix_map); m_freem(npe->ix_m); npe->ix_m = NULL; } } } static void npe_stoprecv(struct npe_softc *sc) { struct npedma *dma = &sc->rxdma; int i; NPE_ASSERT_LOCKED(sc); /* XXX qmgr */ for (i = 0; i < dma->nbuf; i++) { struct npebuf *npe = &dma->buf[i]; if (npe->ix_m != NULL) { bus_dmamap_unload(dma->mtag, npe->ix_map); m_freem(npe->ix_m); npe->ix_m = NULL; } } } /* * Turn off interrupts, and stop the nic. */ void npestop(struct npe_softc *sc) { struct ifnet *ifp = sc->sc_ifp; /* disable transmitter and reciver in the MAC */ WR4(sc, NPE_MAC_RX_CNTRL1, RD4(sc, NPE_MAC_RX_CNTRL1) &~ NPE_RX_CNTRL1_RX_EN); WR4(sc, NPE_MAC_TX_CNTRL1, RD4(sc, NPE_MAC_TX_CNTRL1) &~ NPE_TX_CNTRL1_TX_EN); sc->npe_watchdog_timer = 0; ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); callout_stop(&sc->tick_ch); npe_stopxmit(sc); npe_stoprecv(sc); /* XXX go into loopback & drain q's? */ /* XXX but beware of disabling tx above */ /* * The MAC core rx/tx disable may leave the MAC hardware in an * unpredictable state. A hw reset is executed before resetting * all the MAC parameters to a known value. */ WR4(sc, NPE_MAC_CORE_CNTRL, NPE_CORE_RESET); DELAY(NPE_MAC_RESET_DELAY); WR4(sc, NPE_MAC_INT_CLK_THRESH, NPE_MAC_INT_CLK_THRESH_DEFAULT); WR4(sc, NPE_MAC_CORE_CNTRL, NPE_CORE_MDC_EN); } void npewatchdog(struct npe_softc *sc) { NPE_ASSERT_LOCKED(sc); if (sc->npe_watchdog_timer == 0 || --sc->npe_watchdog_timer != 0) return; device_printf(sc->sc_dev, "watchdog timeout\n"); if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1); npeinit_locked(sc); } static int npeioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct npe_softc *sc = ifp->if_softc; struct mii_data *mii; struct ifreq *ifr = (struct ifreq *)data; int error = 0; #ifdef DEVICE_POLLING int mask; #endif switch (cmd) { case SIOCSIFFLAGS: NPE_LOCK(sc); if ((ifp->if_flags & IFF_UP) == 0 && ifp->if_drv_flags & IFF_DRV_RUNNING) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; npestop(sc); } else { /* reinitialize card on any parameter change */ npeinit_locked(sc); } NPE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: /* update multicast filter list. */ NPE_LOCK(sc); npe_setmcast(sc); NPE_UNLOCK(sc); error = 0; break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: mii = device_get_softc(sc->sc_mii); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; #ifdef DEVICE_POLLING case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_POLLING) { if (ifr->ifr_reqcap & IFCAP_POLLING) { error = ether_poll_register(npe_poll, ifp); if (error) return error; NPE_LOCK(sc); /* disable callbacks XXX txdone is shared */ ixpqmgr_notify_disable(sc->rx_qid); ixpqmgr_notify_disable(sc->tx_doneqid); ifp->if_capenable |= IFCAP_POLLING; NPE_UNLOCK(sc); } else { error = ether_poll_deregister(ifp); /* NB: always enable qmgr callbacks */ NPE_LOCK(sc); /* enable qmgr callbacks */ ixpqmgr_notify_enable(sc->rx_qid, IX_QMGR_Q_SOURCE_ID_NOT_E); ixpqmgr_notify_enable(sc->tx_doneqid, IX_QMGR_Q_SOURCE_ID_NOT_E); ifp->if_capenable &= ~IFCAP_POLLING; NPE_UNLOCK(sc); } } break; #endif default: error = ether_ioctl(ifp, cmd, data); break; } return error; } /* * Setup a traffic class -> rx queue mapping. */ static int npe_setrxqosentry(struct npe_softc *sc, int classix, int trafclass, int qid) { uint32_t msg[2]; msg[0] = (NPE_SETRXQOSENTRY << 24) | (sc->sc_npeid << 20) | classix; msg[1] = (trafclass << 24) | (1 << 23) | (qid << 16) | (qid << 4); return ixpnpe_sendandrecvmsg_sync(sc->sc_npe, msg, msg); } static int npe_setportaddress(struct npe_softc *sc, const uint8_t mac[ETHER_ADDR_LEN]) { uint32_t msg[2]; msg[0] = (NPE_SETPORTADDRESS << 24) | (sc->sc_npeid << 20) | (mac[0] << 8) | (mac[1] << 0); msg[1] = (mac[2] << 24) | (mac[3] << 16) | (mac[4] << 8) | (mac[5] << 0); return ixpnpe_sendandrecvmsg_sync(sc->sc_npe, msg, msg); } static int npe_setfirewallmode(struct npe_softc *sc, int onoff) { uint32_t msg[2]; /* XXX honor onoff */ msg[0] = (NPE_SETFIREWALLMODE << 24) | (sc->sc_npeid << 20); msg[1] = 0; return ixpnpe_sendandrecvmsg_sync(sc->sc_npe, msg, msg); } /* * Update and reset the statistics in the NPE. */ static int npe_updatestats(struct npe_softc *sc) { uint32_t msg[2]; msg[0] = NPE_RESETSTATS << NPE_MAC_MSGID_SHL; msg[1] = sc->sc_stats_phys; /* physical address of stat block */ return ixpnpe_sendmsg_async(sc->sc_npe, msg); } #if 0 /* * Get the current statistics block. */ static int npe_getstats(struct npe_softc *sc) { uint32_t msg[2]; msg[0] = NPE_GETSTATS << NPE_MAC_MSGID_SHL; msg[1] = sc->sc_stats_phys; /* physical address of stat block */ return ixpnpe_sendandrecvmsg(sc->sc_npe, msg, msg); } /* * Query the image id of the loaded firmware. */ static uint32_t npe_getimageid(struct npe_softc *sc) { uint32_t msg[2]; msg[0] = NPE_GETSTATUS << NPE_MAC_MSGID_SHL; msg[1] = 0; return ixpnpe_sendandrecvmsg_sync(sc->sc_npe, msg, msg) == 0 ? msg[1] : 0; } /* * Enable/disable loopback. */ static int npe_setloopback(struct npe_softc *sc, int ena) { uint32_t msg[2]; msg[0] = (NPE_SETLOOPBACK << NPE_MAC_MSGID_SHL) | (ena != 0); msg[1] = 0; return ixpnpe_sendandrecvmsg_sync(sc->sc_npe, msg, msg); } #endif static void npe_child_detached(device_t dev, device_t child) { struct npe_softc *sc; sc = device_get_softc(dev); if (child == sc->sc_mii) sc->sc_mii = NULL; } /* * MII bus support routines. */ #define MII_RD4(sc, reg) bus_space_read_4(sc->sc_iot, sc->sc_miih, reg) #define MII_WR4(sc, reg, v) \ bus_space_write_4(sc->sc_iot, sc->sc_miih, reg, v) static uint32_t npe_mii_mdio_read(struct npe_softc *sc, int reg) { uint32_t v; /* NB: registers are known to be sequential */ v = (MII_RD4(sc, reg+0) & 0xff) << 0; v |= (MII_RD4(sc, reg+4) & 0xff) << 8; v |= (MII_RD4(sc, reg+8) & 0xff) << 16; v |= (MII_RD4(sc, reg+12) & 0xff) << 24; return v; } static void npe_mii_mdio_write(struct npe_softc *sc, int reg, uint32_t cmd) { /* NB: registers are known to be sequential */ MII_WR4(sc, reg+0, cmd & 0xff); MII_WR4(sc, reg+4, (cmd >> 8) & 0xff); MII_WR4(sc, reg+8, (cmd >> 16) & 0xff); MII_WR4(sc, reg+12, (cmd >> 24) & 0xff); } static int npe_mii_mdio_wait(struct npe_softc *sc) { uint32_t v; int i; /* NB: typically this takes 25-30 trips */ for (i = 0; i < 1000; i++) { v = npe_mii_mdio_read(sc, NPE_MAC_MDIO_CMD); if ((v & NPE_MII_GO) == 0) return 1; DELAY(1); } device_printf(sc->sc_dev, "%s: timeout after ~1ms, cmd 0x%x\n", __func__, v); return 0; /* NB: timeout */ } static int npe_miibus_readreg(device_t dev, int phy, int reg) { struct npe_softc *sc = device_get_softc(dev); uint32_t v; v = (phy << NPE_MII_ADDR_SHL) | (reg << NPE_MII_REG_SHL) | NPE_MII_GO; npe_mii_mdio_write(sc, NPE_MAC_MDIO_CMD, v); if (npe_mii_mdio_wait(sc)) v = npe_mii_mdio_read(sc, NPE_MAC_MDIO_STS); else v = 0xffff | NPE_MII_READ_FAIL; return (v & NPE_MII_READ_FAIL) ? 0xffff : (v & 0xffff); } static int npe_miibus_writereg(device_t dev, int phy, int reg, int data) { struct npe_softc *sc = device_get_softc(dev); uint32_t v; v = (phy << NPE_MII_ADDR_SHL) | (reg << NPE_MII_REG_SHL) | data | NPE_MII_WRITE | NPE_MII_GO; npe_mii_mdio_write(sc, NPE_MAC_MDIO_CMD, v); /* XXX complain about timeout */ (void) npe_mii_mdio_wait(sc); return (0); } static void npe_miibus_statchg(device_t dev) { struct npe_softc *sc = device_get_softc(dev); struct mii_data *mii = device_get_softc(sc->sc_mii); uint32_t tx1, rx1; /* sync MAC duplex state */ tx1 = RD4(sc, NPE_MAC_TX_CNTRL1); rx1 = RD4(sc, NPE_MAC_RX_CNTRL1); if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) { tx1 &= ~NPE_TX_CNTRL1_DUPLEX; rx1 |= NPE_RX_CNTRL1_PAUSE_EN; } else { tx1 |= NPE_TX_CNTRL1_DUPLEX; rx1 &= ~NPE_RX_CNTRL1_PAUSE_EN; } WR4(sc, NPE_MAC_RX_CNTRL1, rx1); WR4(sc, NPE_MAC_TX_CNTRL1, tx1); } static device_method_t npe_methods[] = { /* Device interface */ DEVMETHOD(device_probe, npe_probe), DEVMETHOD(device_attach, npe_attach), DEVMETHOD(device_detach, npe_detach), /* Bus interface */ DEVMETHOD(bus_child_detached, npe_child_detached), /* MII interface */ DEVMETHOD(miibus_readreg, npe_miibus_readreg), DEVMETHOD(miibus_writereg, npe_miibus_writereg), DEVMETHOD(miibus_statchg, npe_miibus_statchg), { 0, 0 } }; static driver_t npe_driver = { "npe", npe_methods, sizeof(struct npe_softc), }; DRIVER_MODULE(npe, ixp, npe_driver, npe_devclass, 0, 0); DRIVER_MODULE(miibus, npe, miibus_driver, miibus_devclass, 0, 0); MODULE_DEPEND(npe, ixpqmgr, 1, 1, 1); MODULE_DEPEND(npe, miibus, 1, 1, 1); MODULE_DEPEND(npe, ether, 1, 1, 1); Index: head/sys/arm64/arm64/busdma_bounce.c =================================================================== --- head/sys/arm64/arm64/busdma_bounce.c (revision 328016) +++ head/sys/arm64/arm64/busdma_bounce.c (revision 328017) @@ -1,1333 +1,1333 @@ /*- * Copyright (c) 1997, 1998 Justin T. Gibbs. * Copyright (c) 2015-2016 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Andrew Turner * under sponsorship of the FreeBSD Foundation. * * Portions of this software were developed by Semihalf * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX_BPAGES 4096 enum { BF_COULD_BOUNCE = 0x01, BF_MIN_ALLOC_COMP = 0x02, BF_KMEM_ALLOC = 0x04, BF_COHERENT = 0x10, }; struct bounce_zone; struct bus_dma_tag { struct bus_dma_tag_common common; int map_count; int bounce_flags; bus_dma_segment_t *segments; struct bounce_zone *bounce_zone; }; struct bounce_page { vm_offset_t vaddr; /* kva of bounce buffer */ bus_addr_t busaddr; /* Physical address */ vm_offset_t datavaddr; /* kva of client data */ vm_page_t datapage; /* physical page of client data */ vm_offset_t dataoffs; /* page offset of client data */ bus_size_t datacount; /* client data count */ STAILQ_ENTRY(bounce_page) links; }; int busdma_swi_pending; struct bounce_zone { STAILQ_ENTRY(bounce_zone) links; STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; int total_bpages; int free_bpages; int reserved_bpages; int active_bpages; int total_bounced; int total_deferred; int map_count; bus_size_t alignment; bus_addr_t lowaddr; char zoneid[8]; char lowaddrid[20]; struct sysctl_ctx_list sysctl_tree; struct sysctl_oid *sysctl_tree_top; }; static struct mtx bounce_lock; static int total_bpages; static int busdma_zonecount; static STAILQ_HEAD(, bounce_zone) bounce_zone_list; static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, "Total bounce pages"); struct sync_list { vm_offset_t vaddr; /* kva of client data */ bus_addr_t paddr; /* physical address */ vm_page_t pages; /* starting page of client data */ bus_size_t datacount; /* client data count */ }; struct bus_dmamap { struct bp_list bpages; int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; struct memdesc mem; bus_dmamap_callback_t *callback; void *callback_arg; STAILQ_ENTRY(bus_dmamap) links; u_int flags; #define DMAMAP_COULD_BOUNCE (1 << 0) #define DMAMAP_FROM_DMAMEM (1 << 1) int sync_count; struct sync_list slist[]; }; static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; static void init_bounce_pages(void *dummy); static int alloc_bounce_zone(bus_dma_tag_t dmat); static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit); static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags); static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags); static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags); /* * Allocate a device specific dma_tag. */ static int bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { bus_dma_tag_t newtag; int error; *dmat = NULL; error = common_bus_dma_tag_create(parent != NULL ? &parent->common : NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, sizeof (struct bus_dma_tag), (void **)&newtag); if (error != 0) return (error); newtag->common.impl = &bus_dma_bounce_impl; newtag->map_count = 0; newtag->segments = NULL; if ((flags & BUS_DMA_COHERENT) != 0) newtag->bounce_flags |= BF_COHERENT; if (parent != NULL) { if ((newtag->common.filter != NULL || (parent->bounce_flags & BF_COULD_BOUNCE) != 0)) newtag->bounce_flags |= BF_COULD_BOUNCE; /* Copy some flags from the parent */ newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; } if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || newtag->common.alignment > 1) newtag->bounce_flags |= BF_COULD_BOUNCE; if (((newtag->bounce_flags & BF_COULD_BOUNCE) != 0) && (flags & BUS_DMA_ALLOCNOW) != 0) { struct bounce_zone *bz; /* Must bounce */ if ((error = alloc_bounce_zone(newtag)) != 0) { free(newtag, M_DEVBUF); return (error); } bz = newtag->bounce_zone; if (ptoa(bz->total_bpages) < maxsize) { int pages; pages = atop(maxsize) - bz->total_bpages; /* Add pages to our bounce pool */ if (alloc_bounce_pages(newtag, pages) < pages) error = ENOMEM; } /* Performed initial allocation */ newtag->bounce_flags |= BF_MIN_ALLOC_COMP; } else error = 0; if (error != 0) free(newtag, M_DEVBUF); else *dmat = newtag; CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), error); return (error); } static int bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy, parent; int error; error = 0; dmat_copy = dmat; if (dmat != NULL) { if (dmat->map_count != 0) { error = EBUSY; goto out; } while (dmat != NULL) { parent = (bus_dma_tag_t)dmat->common.parent; atomic_subtract_int(&dmat->common.ref_count, 1); if (dmat->common.ref_count == 0) { if (dmat->segments != NULL) free(dmat->segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so * release our reference * count on our parent. */ dmat = parent; } else dmat = NULL; } } out: CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); return (error); } static bus_dmamap_t alloc_dmamap(bus_dma_tag_t dmat, int flags) { u_long mapsize; bus_dmamap_t map; mapsize = sizeof(*map); mapsize += sizeof(struct sync_list) * dmat->common.nsegments; map = malloc(mapsize, M_DEVBUF, flags | M_ZERO); if (map == NULL) return (NULL); /* Initialize the new map */ STAILQ_INIT(&map->bpages); return (map); } /* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. */ static int bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { struct bounce_zone *bz; int error, maxpages, pages; error = 0; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( - sizeof(bus_dma_segment_t) * dmat->common.nsegments, + dmat->segments = (bus_dma_segment_t *)mallocarray( + dmat->common.nsegments, sizeof(bus_dma_segment_t), M_DEVBUF, M_NOWAIT); if (dmat->segments == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (ENOMEM); } } *mapp = alloc_dmamap(dmat, M_NOWAIT); if (*mapp == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); return (ENOMEM); } /* * Bouncing might be required if the driver asks for an active * exclusion region, a data alignment that is stricter than 1, and/or * an active address boundary. */ if (dmat->bounce_flags & BF_COULD_BOUNCE) { /* Must bounce */ if (dmat->bounce_zone == NULL) { if ((error = alloc_bounce_zone(dmat)) != 0) { free(*mapp, M_DEVBUF); return (error); } } bz = dmat->bounce_zone; (*mapp)->flags = DMAMAP_COULD_BOUNCE; /* * Attempt to add pages to our pool on a per-instance * basis up to a sane limit. */ if (dmat->common.alignment > 1) maxpages = MAX_BPAGES; else maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->common.lowaddr)); if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || (bz->map_count > 0 && bz->total_bpages < maxpages)) { pages = MAX(atop(dmat->common.maxsize), 1); pages = MIN(maxpages - bz->total_bpages, pages); pages = MAX(pages, 1); if (alloc_bounce_pages(dmat, pages) < pages) error = ENOMEM; if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { if (error == 0) { dmat->bounce_flags |= BF_MIN_ALLOC_COMP; } } else error = 0; } bz->map_count++; } if (error == 0) dmat->map_count++; else free(*mapp, M_DEVBUF); CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, error); return (error); } /* * Destroy a handle for mapping from kva/uva/physical * address space into bus device space. */ static int bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { /* Check we are destroying the correct map type */ if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); return (EBUSY); } if (dmat->bounce_zone) { KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0, ("%s: Bounce zone when cannot bounce", __func__)); dmat->bounce_zone->map_count--; } free(map, M_DEVBUF); dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); return (0); } /* * Allocate a piece of memory that can be efficiently mapped into * bus device space based on the constraints lited in the dma tag. * A dmamap to for use with dmamap_load is also allocated. */ static int bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp) { /* * XXX ARM64TODO: * This bus_dma implementation requires IO-Coherent architecutre. * If IO-Coherency is not guaranteed, the BUS_DMA_COHERENT flag has * to be implented using non-cacheable memory. */ vm_memattr_t attr; int mflags; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; else mflags = M_WAITOK; if (dmat->segments == NULL) { dmat->segments = (bus_dma_segment_t *)malloc( sizeof(bus_dma_segment_t) * dmat->common.nsegments, M_DEVBUF, mflags); if (dmat->segments == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, ENOMEM); return (ENOMEM); } } if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; if (flags & BUS_DMA_NOCACHE) attr = VM_MEMATTR_UNCACHEABLE; else if ((flags & BUS_DMA_COHERENT) != 0 && (dmat->bounce_flags & BF_COHERENT) == 0) /* * If we have a non-coherent tag, and are trying to allocate * a coherent block of memory it needs to be uncached. */ attr = VM_MEMATTR_UNCACHEABLE; else attr = VM_MEMATTR_DEFAULT; /* * Create the map, but don't set the could bounce flag as * this allocation should never bounce; */ *mapp = alloc_dmamap(dmat, mflags); if (*mapp == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, ENOMEM); return (ENOMEM); } (*mapp)->flags = DMAMAP_FROM_DMAMEM; /* * Allocate the buffer from the malloc(9) allocator if... * - It's small enough to fit into a single power of two sized bucket. * - The alignment is less than or equal to the maximum size * - The low address requirement is fulfilled. * else allocate non-contiguous pages if... * - The page count that could get allocated doesn't exceed * nsegments also when the maximum segment size is less * than PAGE_SIZE. * - The alignment constraint isn't larger than a page boundary. * - There are no boundary-crossing constraints. * else allocate a block of contiguous pages because one or more of the * constraints is something that only the contig allocator can fulfill. * * NOTE: The (dmat->common.alignment <= dmat->maxsize) check * below is just a quick hack. The exact alignment guarantees * of malloc(9) need to be nailed down, and the code below * should be rewritten to take that into account. * * In the meantime warn the user if malloc gets it wrong. */ if ((dmat->common.maxsize <= PAGE_SIZE) && (dmat->common.alignment <= dmat->common.maxsize) && dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && attr == VM_MEMATTR_DEFAULT) { *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags); } else if (dmat->common.nsegments >= howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && dmat->common.alignment <= PAGE_SIZE && (dmat->common.boundary % PAGE_SIZE) == 0) { /* Page-based multi-segment allocations allowed */ *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, attr); dmat->bounce_flags |= BF_KMEM_ALLOC; } else { *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, dmat->common.alignment != 0 ? dmat->common.alignment : 1ul, dmat->common.boundary, attr); dmat->bounce_flags |= BF_KMEM_ALLOC; } if (*vaddr == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, ENOMEM); free(*mapp, M_DEVBUF); return (ENOMEM); } else if (vtophys(*vaddr) & (dmat->common.alignment - 1)) { printf("bus_dmamem_alloc failed to align memory properly.\n"); } dmat->map_count++; CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, 0); return (0); } /* * Free a piece of memory and it's allociated dmamap, that was allocated * via bus_dmamem_alloc. Make the same choice for free/contigfree. */ static void bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { /* * Check the map came from bounce_bus_dmamem_alloc, so the map * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() * was used and set if kmem_alloc_contig() was used. */ if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) panic("bus_dmamem_free: Invalid map freed\n"); if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) free(vaddr, M_DEVBUF); else kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->common.maxsize); free(map, M_DEVBUF); dmat->map_count--; CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->bounce_flags); } static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags) { bus_addr_t curaddr; bus_size_t sgsize; if ((map->flags & DMAMAP_COULD_BOUNCE) != 0 && map->pagesneeded == 0) { /* * Count the number of bounce pages * needed in order to complete this transfer */ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->common.maxsegsz); if (bus_dma_run_filter(&dmat->common, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); map->pagesneeded++; } curaddr += sgsize; buflen -= sgsize; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } } static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { vm_offset_t vaddr; vm_offset_t vendaddr; bus_addr_t paddr; bus_size_t sg_len; if ((map->flags & DMAMAP_COULD_BOUNCE) != 0 && map->pagesneeded == 0) { CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem), dmat->common.boundary, dmat->common.alignment); CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); /* * Count the number of bounce pages * needed in order to complete this transfer */ vaddr = (vm_offset_t)buf; vendaddr = (vm_offset_t)buf + buflen; while (vaddr < vendaddr) { sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); if (pmap == kernel_pmap) paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); if (bus_dma_run_filter(&dmat->common, paddr) != 0) { sg_len = roundup2(sg_len, dmat->common.alignment); map->pagesneeded++; } vaddr += sg_len; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } } static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) { /* Reserve Necessary Bounce Pages */ mtx_lock(&bounce_lock); if (flags & BUS_DMA_NOWAIT) { if (reserve_bounce_pages(dmat, map, 0) != 0) { mtx_unlock(&bounce_lock); return (ENOMEM); } } else { if (reserve_bounce_pages(dmat, map, 1) != 0) { /* Queue us for resources */ STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); mtx_unlock(&bounce_lock); return (EINPROGRESS); } } mtx_unlock(&bounce_lock); return (0); } /* * Add a single contiguous physical range to the segment list. */ static int _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) { bus_addr_t baddr, bmask; int seg; /* * Make sure we don't cross any boundaries. */ bmask = ~(dmat->common.boundary - 1); if (dmat->common.boundary > 0) { baddr = (curaddr + dmat->common.boundary) & bmask; if (sgsize > (baddr - curaddr)) sgsize = (baddr - curaddr); } /* * Insert chunk into a segment, coalescing with * previous segment if possible. */ seg = *segp; if (seg == -1) { seg = 0; segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } else { if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz && (dmat->common.boundary == 0 || (segs[seg].ds_addr & bmask) == (curaddr & bmask))) segs[seg].ds_len += sgsize; else { if (++seg >= dmat->common.nsegments) return (0); segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } } *segp = seg; return (sgsize); } /* * Utility function to load a physical buffer. segp contains * the starting segment on entrace, and the ending segment on exit. */ static int bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { struct sync_list *sl; bus_size_t sgsize; bus_addr_t curaddr, sl_end; int error; if (segs == NULL) segs = dmat->segments; if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) { _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } sl = map->slist + map->sync_count - 1; sl_end = 0; while (buflen > 0) { curaddr = buf; sgsize = MIN(buflen, dmat->common.maxsegsz); if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) && map->pagesneeded != 0 && bus_dma_run_filter(&dmat->common, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); } else if ((dmat->bounce_flags & BF_COHERENT) == 0) { if (map->sync_count > 0) sl_end = sl->paddr + sl->datacount; if (map->sync_count == 0 || curaddr != sl_end) { if (++map->sync_count > dmat->common.nsegments) break; sl++; sl->vaddr = 0; sl->paddr = curaddr; sl->datacount = sgsize; sl->pages = PHYS_TO_VM_PAGE(curaddr); KASSERT(sl->pages != NULL, ("%s: page at PA:0x%08lx is not in " "vm_page_array", __func__, curaddr)); } else sl->datacount += sgsize; } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; buf += sgsize; buflen -= sgsize; } /* * Did we fit? */ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } /* * Utility function to load a linear buffer. segp contains * the starting segment on entrace, and the ending segment on exit. */ static int bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { struct sync_list *sl; bus_size_t sgsize, max_sgsize; bus_addr_t curaddr, sl_pend; vm_offset_t kvaddr, vaddr, sl_vend; int error; if (segs == NULL) segs = dmat->segments; if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) { _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); if (error) return (error); } } sl = map->slist + map->sync_count - 1; vaddr = (vm_offset_t)buf; sl_pend = 0; sl_vend = 0; while (buflen > 0) { /* * Get the physical address for this segment. */ if (pmap == kernel_pmap) { curaddr = pmap_kextract(vaddr); kvaddr = vaddr; } else { curaddr = pmap_extract(pmap, vaddr); kvaddr = 0; } /* * Compute the segment size, and adjust counts. */ max_sgsize = MIN(buflen, dmat->common.maxsegsz); sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) && map->pagesneeded != 0 && bus_dma_run_filter(&dmat->common, curaddr)) { sgsize = roundup2(sgsize, dmat->common.alignment); sgsize = MIN(sgsize, max_sgsize); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, sgsize); } else if ((dmat->bounce_flags & BF_COHERENT) == 0) { sgsize = MIN(sgsize, max_sgsize); if (map->sync_count > 0) { sl_pend = sl->paddr + sl->datacount; sl_vend = sl->vaddr + sl->datacount; } if (map->sync_count == 0 || (kvaddr != 0 && kvaddr != sl_vend) || (curaddr != sl_pend)) { if (++map->sync_count > dmat->common.nsegments) goto cleanup; sl++; sl->vaddr = kvaddr; sl->paddr = curaddr; if (kvaddr != 0) { sl->pages = NULL; } else { sl->pages = PHYS_TO_VM_PAGE(curaddr); KASSERT(sl->pages != NULL, ("%s: page at PA:0x%08lx is not " "in vm_page_array", __func__, curaddr)); } sl->datacount = sgsize; } else sl->datacount += sgsize; } else { sgsize = MIN(sgsize, max_sgsize); } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; vaddr += sgsize; buflen -= sgsize; } cleanup: /* * Did we fit? */ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } static void bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { if ((map->flags & DMAMAP_COULD_BOUNCE) == 0) return; map->mem = *mem; map->dmat = dmat; map->callback = callback; map->callback_arg = callback_arg; } static bus_dma_segment_t * bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error) { if (segs == NULL) segs = dmat->segments; return (segs); } /* * Release the mapping held by map. */ static void bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_page *bpage; while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { STAILQ_REMOVE_HEAD(&map->bpages, links); free_bounce_page(dmat, bpage); } map->sync_count = 0; } static void dma_preread_safe(vm_offset_t va, vm_size_t size) { /* * Write back any partial cachelines immediately before and * after the DMA region. */ if (va & (dcache_line_size - 1)) cpu_dcache_wb_range(va, 1); if ((va + size) & (dcache_line_size - 1)) cpu_dcache_wb_range(va + size, 1); cpu_dcache_inv_range(va, size); } static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) { uint32_t len, offset; vm_page_t m; vm_paddr_t pa; vm_offset_t va, tempva; bus_size_t size; offset = sl->paddr & PAGE_MASK; m = sl->pages; size = sl->datacount; pa = sl->paddr; for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { tempva = 0; if (sl->vaddr == 0) { len = min(PAGE_SIZE - offset, size); tempva = pmap_quick_enter_page(m); va = tempva | offset; KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", VM_PAGE_TO_PHYS(m) | offset, pa)); } else { len = sl->datacount; va = sl->vaddr; } switch (op) { case BUS_DMASYNC_PREWRITE: case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: cpu_dcache_wb_range(va, len); break; case BUS_DMASYNC_PREREAD: /* * An mbuf may start in the middle of a cacheline. There * will be no cpu writes to the beginning of that line * (which contains the mbuf header) while dma is in * progress. Handle that case by doing a writeback of * just the first cacheline before invalidating the * overall buffer. Any mbuf in a chain may have this * misalignment. Buffers which are not mbufs bounce if * they are not aligned to a cacheline. */ dma_preread_safe(va, len); break; case BUS_DMASYNC_POSTREAD: case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: cpu_dcache_inv_range(va, len); break; default: panic("unsupported combination of sync operations: " "0x%08x\n", op); } if (tempva != 0) pmap_quick_remove_page(tempva); } } static void bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; struct sync_list *sl, *end; vm_offset_t datavaddr, tempvaddr; if (op == BUS_DMASYNC_POSTWRITE) return; if ((op & BUS_DMASYNC_POSTREAD) != 0) { /* * Wait for any DMA operations to complete before the bcopy. */ dsb(sy); } if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing bounce", __func__, dmat, dmat->common.flags, op); if ((op & BUS_DMASYNC_PREWRITE) != 0) { while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)datavaddr, (void *)bpage->vaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); if ((dmat->bounce_flags & BF_COHERENT) == 0) cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } else if ((op & BUS_DMASYNC_PREREAD) != 0) { while (bpage != NULL) { if ((dmat->bounce_flags & BF_COHERENT) == 0) cpu_dcache_wbinv_range(bpage->vaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } } if ((op & BUS_DMASYNC_POSTREAD) != 0) { while (bpage != NULL) { if ((dmat->bounce_flags & BF_COHERENT) == 0) cpu_dcache_inv_range(bpage->vaddr, bpage->datacount); tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { tempvaddr = pmap_quick_enter_page( bpage->datapage); datavaddr = tempvaddr | bpage->dataoffs; } bcopy((void *)bpage->vaddr, (void *)datavaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } } /* * Cache maintenance for normal (non-COHERENT non-bounce) buffers. */ if (map->sync_count != 0) { sl = &map->slist[0]; end = &map->slist[map->sync_count]; CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " "performing sync", __func__, dmat, op); for ( ; sl != end; ++sl) dma_dcache_sync(sl, op); } if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { /* * Wait for the bcopy to complete before any DMA operations. */ dsb(sy); } } static void init_bounce_pages(void *dummy __unused) { total_bpages = 0; STAILQ_INIT(&bounce_zone_list); STAILQ_INIT(&bounce_map_waitinglist); STAILQ_INIT(&bounce_map_callbacklist); mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); } SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); static struct sysctl_ctx_list * busdma_sysctl_tree(struct bounce_zone *bz) { return (&bz->sysctl_tree); } static struct sysctl_oid * busdma_sysctl_tree_top(struct bounce_zone *bz) { return (bz->sysctl_tree_top); } static int alloc_bounce_zone(bus_dma_tag_t dmat) { struct bounce_zone *bz; /* Check to see if we already have a suitable zone */ STAILQ_FOREACH(bz, &bounce_zone_list, links) { if ((dmat->common.alignment <= bz->alignment) && (dmat->common.lowaddr >= bz->lowaddr)) { dmat->bounce_zone = bz; return (0); } } if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) return (ENOMEM); STAILQ_INIT(&bz->bounce_page_list); bz->free_bpages = 0; bz->reserved_bpages = 0; bz->active_bpages = 0; bz->lowaddr = dmat->common.lowaddr; bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE); bz->map_count = 0; snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); busdma_zonecount++; snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); dmat->bounce_zone = bz; sysctl_ctx_init(&bz->sysctl_tree); bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, CTLFLAG_RD, 0, ""); if (bz->sysctl_tree_top == NULL) { sysctl_ctx_free(&bz->sysctl_tree); return (0); /* XXX error code? */ } SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, "Total bounce pages"); SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, "Free bounce pages"); SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, "Reserved bounce pages"); SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, "Active bounce pages"); SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, "Total bounce requests"); SYSCTL_ADD_INT(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, "Total bounce requests that were deferred"); SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "alignment", CTLFLAG_RD, &bz->alignment, ""); return (0); } static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) { struct bounce_zone *bz; int count; bz = dmat->bounce_zone; count = 0; while (numpages > 0) { struct bounce_page *bpage; bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, M_NOWAIT | M_ZERO); if (bpage == NULL) break; bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); if (bpage->vaddr == 0) { free(bpage, M_DEVBUF); break; } bpage->busaddr = pmap_kextract(bpage->vaddr); mtx_lock(&bounce_lock); STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); total_bpages++; bz->total_bpages++; bz->free_bpages++; mtx_unlock(&bounce_lock); count++; numpages--; } return (count); } static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) { struct bounce_zone *bz; int pages; mtx_assert(&bounce_lock, MA_OWNED); bz = dmat->bounce_zone; pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) return (map->pagesneeded - (map->pagesreserved + pages)); bz->free_bpages -= pages; bz->reserved_bpages += pages; map->pagesreserved += pages; pages = map->pagesneeded - map->pagesreserved; return (pages); } static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, bus_addr_t addr, bus_size_t size) { struct bounce_zone *bz; struct bounce_page *bpage; KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0, ("add_bounce_page: bad map %p", map)); bz = dmat->bounce_zone; if (map->pagesneeded == 0) panic("add_bounce_page: map doesn't need any pages"); map->pagesneeded--; if (map->pagesreserved == 0) panic("add_bounce_page: map doesn't need any pages"); map->pagesreserved--; mtx_lock(&bounce_lock); bpage = STAILQ_FIRST(&bz->bounce_page_list); if (bpage == NULL) panic("add_bounce_page: free page list is empty"); STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); bz->reserved_bpages--; bz->active_bpages++; mtx_unlock(&bounce_lock); if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { /* Page offset needs to be preserved. */ bpage->vaddr |= addr & PAGE_MASK; bpage->busaddr |= addr & PAGE_MASK; } bpage->datavaddr = vaddr; bpage->datapage = PHYS_TO_VM_PAGE(addr); bpage->dataoffs = addr & PAGE_MASK; bpage->datacount = size; STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); return (bpage->busaddr); } static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) { struct bus_dmamap *map; struct bounce_zone *bz; bz = dmat->bounce_zone; bpage->datavaddr = 0; bpage->datacount = 0; if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { /* * Reset the bounce page to start at offset 0. Other uses * of this bounce page may need to store a full page of * data and/or assume it starts on a page boundary. */ bpage->vaddr &= ~PAGE_MASK; bpage->busaddr &= ~PAGE_MASK; } mtx_lock(&bounce_lock); STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); bz->free_bpages++; bz->active_bpages--; if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { if (reserve_bounce_pages(map->dmat, map, 1) == 0) { STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); busdma_swi_pending = 1; bz->total_deferred++; swi_sched(vm_ih, 0); } } mtx_unlock(&bounce_lock); } void busdma_swi(void) { bus_dma_tag_t dmat; struct bus_dmamap *map; mtx_lock(&bounce_lock); while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); mtx_unlock(&bounce_lock); dmat = map->dmat; (dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_LOCK); bus_dmamap_load_mem(map->dmat, map, &map->mem, map->callback, map->callback_arg, BUS_DMA_WAITOK); (dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_UNLOCK); mtx_lock(&bounce_lock); } mtx_unlock(&bounce_lock); } struct bus_dma_impl bus_dma_bounce_impl = { .tag_create = bounce_bus_dma_tag_create, .tag_destroy = bounce_bus_dma_tag_destroy, .map_create = bounce_bus_dmamap_create, .map_destroy = bounce_bus_dmamap_destroy, .mem_alloc = bounce_bus_dmamem_alloc, .mem_free = bounce_bus_dmamem_free, .load_phys = bounce_bus_dmamap_load_phys, .load_buffer = bounce_bus_dmamap_load_buffer, .load_ma = bus_dmamap_load_ma_triv, .map_waitok = bounce_bus_dmamap_waitok, .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, .map_sync = bounce_bus_dmamap_sync };