Index: projects/physbio/sys/sparc64/sparc64/bus_machdep.c =================================================================== --- projects/physbio/sys/sparc64/sparc64/bus_machdep.c (revision 244034) +++ projects/physbio/sys/sparc64/sparc64/bus_machdep.c (revision 244035) @@ -1,753 +1,753 @@ /*- * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This software was developed by the Computer Systems Engineering group * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and * contributed to Berkeley. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 1997, 1998 Justin T. Gibbs. * All rights reserved. * Copyright 2001 by Thomas Moestl . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)machdep.c 8.6 (Berkeley) 1/14/94 * from: NetBSD: machdep.c,v 1.221 2008/04/28 20:23:37 martin Exp * and * from: FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.24 2001/08/15 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void nexus_bus_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t, bus_size_t, int); /* ASIs for bus access */ const int bus_type_asi[] = { ASI_PHYS_BYPASS_EC_WITH_EBIT, /* nexus */ ASI_PHYS_BYPASS_EC_WITH_EBIT, /* SBus */ ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI configuration space */ ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI memory space */ ASI_PHYS_BYPASS_EC_WITH_EBIT_L, /* PCI I/O space */ 0 }; const int bus_stream_asi[] = { ASI_PHYS_BYPASS_EC_WITH_EBIT, /* nexus */ ASI_PHYS_BYPASS_EC_WITH_EBIT, /* SBus */ ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI configuration space */ ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI memory space */ ASI_PHYS_BYPASS_EC_WITH_EBIT, /* PCI I/O space */ 0 }; /* * Convenience function for manipulating driver locks from busdma (during * busdma_swi, for example). Drivers that don't provide their own locks * should specify &Giant to dmat->lockfuncarg. Drivers that use their own * non-mutex locking scheme don't have to use this at all. */ void busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) { struct mtx *dmtx; dmtx = (struct mtx *)arg; switch (op) { case BUS_DMA_LOCK: mtx_lock(dmtx); break; case BUS_DMA_UNLOCK: mtx_unlock(dmtx); break; default: panic("Unknown operation 0x%x for busdma_lock_mutex!", op); } } /* * dflt_lock should never get called. It gets put into the dma tag when * lockfunc == NULL, which is only valid if the maps that are associated * with the tag are meant to never be defered. * XXX Should have a way to identify which driver is responsible here. */ static void dflt_lock(void *arg, bus_dma_lock_op_t op) { panic("driver error: busdma dflt_lock called"); } /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { bus_dma_tag_t newtag; /* Return a NULL tag on failure */ *dmat = NULL; /* Enforce the usage of BUS_GET_DMA_TAG(). */ if (parent == NULL) panic("%s: parent DMA tag NULL", __func__); newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); if (newtag == NULL) return (ENOMEM); /* * The method table pointer and the cookie need to be taken over from * the parent. */ newtag->dt_cookie = parent->dt_cookie; newtag->dt_mt = parent->dt_mt; newtag->dt_parent = parent; newtag->dt_alignment = alignment; newtag->dt_boundary = boundary; newtag->dt_lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); newtag->dt_highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); newtag->dt_filter = filter; newtag->dt_filterarg = filterarg; newtag->dt_maxsize = maxsize; newtag->dt_nsegments = nsegments; newtag->dt_maxsegsz = maxsegsz; newtag->dt_flags = flags; newtag->dt_ref_count = 1; /* Count ourselves */ newtag->dt_map_count = 0; if (lockfunc != NULL) { newtag->dt_lockfunc = lockfunc; newtag->dt_lockfuncarg = lockfuncarg; } else { newtag->dt_lockfunc = dflt_lock; newtag->dt_lockfuncarg = NULL; } newtag->dt_segments = NULL; /* Take into account any restrictions imposed by our parent tag. */ newtag->dt_lowaddr = ulmin(parent->dt_lowaddr, newtag->dt_lowaddr); newtag->dt_highaddr = ulmax(parent->dt_highaddr, newtag->dt_highaddr); if (newtag->dt_boundary == 0) newtag->dt_boundary = parent->dt_boundary; else if (parent->dt_boundary != 0) newtag->dt_boundary = ulmin(parent->dt_boundary, newtag->dt_boundary); atomic_add_int(&parent->dt_ref_count, 1); if (newtag->dt_boundary > 0) newtag->dt_maxsegsz = ulmin(newtag->dt_maxsegsz, newtag->dt_boundary); *dmat = newtag; return (0); } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t parent; if (dmat != NULL) { if (dmat->dt_map_count != 0) return (EBUSY); while (dmat != NULL) { parent = dmat->dt_parent; atomic_subtract_int(&dmat->dt_ref_count, 1); if (dmat->dt_ref_count == 0) { if (dmat->dt_segments != NULL) free(dmat->dt_segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so * release our reference * count on our parent. */ dmat = parent; } else dmat = NULL; } } return (0); } /* Allocate/free a tag, and do the necessary management work. */ int sparc64_dma_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp) { if (dmat->dt_segments == NULL) { dmat->dt_segments = (bus_dma_segment_t *)malloc( sizeof(bus_dma_segment_t) * dmat->dt_nsegments, M_DEVBUF, M_NOWAIT); if (dmat->dt_segments == NULL) return (ENOMEM); } *mapp = malloc(sizeof(**mapp), M_DEVBUF, M_NOWAIT | M_ZERO); if (*mapp == NULL) return (ENOMEM); SLIST_INIT(&(*mapp)->dm_reslist); dmat->dt_map_count++; return (0); } void sparc64_dma_free_map(bus_dma_tag_t dmat, bus_dmamap_t map) { free(map, M_DEVBUF); dmat->dt_map_count--; } static int nexus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { return (sparc64_dma_alloc_map(dmat, mapp)); } static int nexus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { sparc64_dma_free_map(dmat, map); return (0); } /* * Utility function to load a linear buffer. segp contains * the starting segment on entrace, and the ending segment on exit. */ static int nexus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { bus_size_t sgsize; bus_addr_t curaddr, baddr, bmask; vm_offset_t vaddr = (vm_offset_t)buf; int seg; if (segs == NULL) segs = dmat->dt_segments; bmask = ~(dmat->dt_boundary - 1); for (seg = *segp; buflen > 0 ; ) { /* * Get the physical address for this segment. */ if (pmap == kernel_pmap) curaddr = pmap_kextract(vaddr); else curaddr = pmap_extract(pmap, vaddr); /* * Compute the segment size, and adjust counts. */ sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); if (sgsize > dmat->dt_maxsegsz) sgsize = dmat->dt_maxsegsz; if (buflen < sgsize) sgsize = buflen; /* * Make sure we don't cross any boundaries. */ if (dmat->dt_boundary > 0) { baddr = (curaddr + dmat->dt_boundary) & bmask; if (sgsize > (baddr - curaddr)) sgsize = (baddr - curaddr); } /* * Insert chunk into a segment, coalescing with * previous segment if possible. */ if (seg == -1) { seg = 0; segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } else { if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && (segs[seg].ds_len + sgsize) <= dmat->dt_maxsegsz && (dmat->dt_boundary == 0 || (segs[seg].ds_addr & bmask) == (curaddr & bmask))) segs[seg].ds_len += sgsize; else { if (++seg >= dmat->dt_nsegments) break; segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } } vaddr += sgsize; buflen -= sgsize; } *segp = seg; /* * Did we fit? */ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } static void nexus_dmamap_mayblock(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmamap_callback_t *callback, void *callback_arg, int *flags) { } -void +static void nexus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmamap_callback_t *callback, void *callback_arg, int nsegs, int error) { if (error) { (*callback)(callback_arg, dmat->dt_segments, 0, error); } else { map->dm_flags |= DMF_LOADED; (*callback)(callback_arg, dmat->dt_segments, nsegs, 0); } } static void nexus_dmamap_complete2(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmamap_callback2_t *callback, void *callback_arg, int nsegs, bus_size_t len, int error) { if (error) { (*callback)(callback_arg, dmat->dt_segments, 0, 0, error); } else { map->dm_flags |= DMF_LOADED; (*callback)(callback_arg, dmat->dt_segments, nsegs, len, error); } } static void nexus_dmamap_directseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error) { } /* * Common function for unloading a DMA map. May be called by * bus-specific DMA map unload functions. */ static void nexus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { map->dm_flags &= ~DMF_LOADED; } /* * Common function for DMA map synchronization. May be called * by bus-specific DMA map synchronization functions. */ static void nexus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { /* * We sync out our caches, but the bus must do the same. * * Actually a #Sync is expensive. We should optimize. */ if ((op & BUS_DMASYNC_PREREAD) || (op & BUS_DMASYNC_PREWRITE)) { /* * Don't really need to do anything, but flush any pending * writes anyway. */ membar(Sync); } if (op & BUS_DMASYNC_POSTWRITE) { /* Nothing to do. Handled by the bus controller. */ } } /* * Common function for DMA-safe memory allocation. May be called * by bus-specific DMA memory allocation functions. */ static int nexus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, bus_dmamap_t *mapp) { int mflags; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; else mflags = M_WAITOK; if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; /* * XXX: * (dmat->dt_alignment < dmat->dt_maxsize) is just a quick hack; the * exact alignment guarantees of malloc need to be nailed down, and * the code below should be rewritten to take that into account. * * In the meantime, we'll warn the user if malloc gets it wrong. */ if (dmat->dt_maxsize <= PAGE_SIZE && dmat->dt_alignment < dmat->dt_maxsize) *vaddr = malloc(dmat->dt_maxsize, M_DEVBUF, mflags); else { /* * XXX use contigmalloc until it is merged into this * facility and handles multi-seg allocations. Nobody * is doing multi-seg allocations yet though. */ *vaddr = contigmalloc(dmat->dt_maxsize, M_DEVBUF, mflags, 0ul, dmat->dt_lowaddr, dmat->dt_alignment ? dmat->dt_alignment : 1UL, dmat->dt_boundary); } if (*vaddr == NULL) return (ENOMEM); if (vtophys(*vaddr) % dmat->dt_alignment) printf("%s: failed to align memory properly.\n", __func__); return (0); } /* * Common function for freeing DMA-safe memory. May be called by * bus-specific DMA memory free functions. */ static void nexus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { if (dmat->dt_maxsize <= PAGE_SIZE && dmat->dt_alignment < dmat->dt_maxsize) free(vaddr, M_DEVBUF); else contigfree(vaddr, dmat->dt_maxsize, M_DEVBUF); } static struct bus_dma_methods nexus_dma_methods = { nexus_dmamap_create, nexus_dmamap_destroy, nexus_dmamap_load_buffer, nexus_dmamap_mayblock, nexus_dmamap_complete, nexus_dmamap_complete2, nexus_dmamap_directseg, nexus_dmamap_unload, nexus_dmamap_sync, nexus_dmamem_alloc, nexus_dmamem_free, }; struct bus_dma_tag nexus_dmatag = { NULL, NULL, 1, 0, ~0, ~0, NULL, /* XXX */ NULL, ~0, ~0, ~0, 0, 0, 0, NULL, NULL, NULL, &nexus_dma_methods, }; /* * Helpers to map/unmap bus memory */ int bus_space_map(bus_space_tag_t tag, bus_addr_t address, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (sparc64_bus_mem_map(tag, address, size, flags, 0, handlep)); } int sparc64_bus_mem_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, int flags, vm_offset_t vaddr, bus_space_handle_t *hp) { vm_offset_t sva; vm_offset_t va; vm_paddr_t pa; vm_size_t vsz; u_long pm_flags; /* * Given that we use physical access for bus_space(9) there's no need * need to map anything in unless BUS_SPACE_MAP_LINEAR is requested. */ if ((flags & BUS_SPACE_MAP_LINEAR) == 0) { *hp = addr; return (0); } if (tag->bst_cookie == NULL) { printf("%s: resource cookie not set\n", __func__); return (EINVAL); } size = round_page(size); if (size == 0) { printf("%s: zero size\n", __func__); return (EINVAL); } switch (tag->bst_type) { case PCI_CONFIG_BUS_SPACE: case PCI_IO_BUS_SPACE: case PCI_MEMORY_BUS_SPACE: pm_flags = TD_IE; break; default: pm_flags = 0; break; } if ((flags & BUS_SPACE_MAP_CACHEABLE) == 0) pm_flags |= TD_E; if (vaddr != 0L) sva = trunc_page(vaddr); else { if ((sva = kmem_alloc_nofault(kernel_map, size)) == 0) panic("%s: cannot allocate virtual memory", __func__); } pa = trunc_page(addr); if ((flags & BUS_SPACE_MAP_READONLY) == 0) pm_flags |= TD_W; va = sva; vsz = size; do { pmap_kenter_flags(va, pa, pm_flags); va += PAGE_SIZE; pa += PAGE_SIZE; } while ((vsz -= PAGE_SIZE) > 0); tlb_range_demap(kernel_pmap, sva, sva + size - 1); /* Note: we preserve the page offset. */ rman_set_virtual(tag->bst_cookie, (void *)(sva | (addr & PAGE_MASK))); return (0); } void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t size) { sparc64_bus_mem_unmap(tag, handle, size); } int sparc64_bus_mem_unmap(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t size) { vm_offset_t sva; vm_offset_t va; vm_offset_t endva; if (tag->bst_cookie == NULL || (sva = (vm_offset_t)rman_get_virtual(tag->bst_cookie)) == 0) return (0); sva = trunc_page(sva); endva = sva + round_page(size); for (va = sva; va < endva; va += PAGE_SIZE) pmap_kremove_flags(va); tlb_range_demap(kernel_pmap, sva, sva + size - 1); kmem_free(kernel_map, sva, size); return (0); } /* * Fake up a bus tag, for use by console drivers in early boot when the * regular means to allocate resources are not yet available. * Addr is the physical address of the desired start of the handle. */ bus_space_handle_t sparc64_fake_bustag(int space, bus_addr_t addr, struct bus_space_tag *ptag) { ptag->bst_cookie = NULL; ptag->bst_parent = NULL; ptag->bst_type = space; ptag->bst_bus_barrier = nexus_bus_barrier; return (addr); } /* * Allocate a bus tag. */ bus_space_tag_t sparc64_alloc_bus_tag(void *cookie, struct bus_space_tag *ptag, int type, void *barrier) { bus_space_tag_t bt; bt = malloc(sizeof(struct bus_space_tag), M_DEVBUF, M_NOWAIT); if (bt == NULL) return (NULL); bt->bst_cookie = cookie; bt->bst_parent = ptag; bt->bst_type = type; bt->bst_bus_barrier = barrier; return (bt); } /* * Base bus space handlers. */ static void nexus_bus_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, bus_size_t size, int flags) { /* * We have lots of alternatives depending on whether we're * synchronizing loads with loads, loads with stores, stores * with loads, or stores with stores. The only ones that seem * generic are #Sync and #MemIssue. I'll use #Sync for safety. */ switch(flags) { case BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE: case BUS_SPACE_BARRIER_READ: case BUS_SPACE_BARRIER_WRITE: membar(Sync); break; default: panic("%s: unknown flags", __func__); } return; } struct bus_space_tag nexus_bustag = { NULL, /* cookie */ NULL, /* parent bus tag */ NEXUS_BUS_SPACE, /* type */ nexus_bus_barrier, /* bus_space_barrier */ }; Index: projects/physbio/sys/sparc64/sparc64/iommu.c =================================================================== --- projects/physbio/sys/sparc64/sparc64/iommu.c (revision 244034) +++ projects/physbio/sys/sparc64/sparc64/iommu.c (revision 244035) @@ -1,1138 +1,1136 @@ /*- * Copyright (c) 1999, 2000 Matthew R. Green * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: iommu.c,v 1.82 2008/05/30 02:29:37 mrg Exp */ /*- * Copyright (c) 1999-2002 Eduardo Horvath * Copyright (c) 2001-2003 Thomas Moestl * Copyright (c) 2007, 2009 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: sbus.c,v 1.50 2002/06/20 18:26:24 eeh Exp */ #include __FBSDID("$FreeBSD$"); /* * UltraSPARC IOMMU support; used by both the PCI and SBus code. * * TODO: * - Support sub-page boundaries. * - Fix alignment handling for small allocations (the possible page offset * of malloc()ed memory is not handled at all). Revise interaction of * alignment with the load_mbuf and load_uio functions. * - Handle lowaddr and highaddr in some way, and try to work out a way * for filter callbacks to work. Currently, only lowaddr is honored * in that no addresses above it are considered at all. * - Implement BUS_DMA_ALLOCNOW in bus_dma_tag_create as far as possible. * - Check the possible return values and callback error arguments; * the callback currently gets called in error conditions where it should * not be. * - When running out of DVMA space, return EINPROGRESS in the non- * BUS_DMA_NOWAIT case and delay the callback until sufficient space * becomes available. */ #include "opt_iommu.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Tuning constants */ #define IOMMU_MAX_PRE (32 * 1024) #define IOMMU_MAX_PRE_SEG 3 /* Threshold for using the streaming buffer */ #define IOMMU_STREAM_THRESH 128 static MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers"); static int iommu_strbuf_flush_sync(struct iommu_state *); #ifdef IOMMU_DIAG static void iommu_diag(struct iommu_state *, vm_offset_t va); #endif /* * Helpers */ #define IOMMU_READ8(is, reg, off) \ bus_space_read_8((is)->is_bustag, (is)->is_bushandle, \ (is)->reg + (off)) #define IOMMU_WRITE8(is, reg, off, v) \ bus_space_write_8((is)->is_bustag, (is)->is_bushandle, \ (is)->reg + (off), (v)) #define IOMMU_HAS_SB(is) \ ((is)->is_sb[0] != 0 || (is)->is_sb[1] != 0) /* * Always overallocate one page; this is needed to handle alignment of the * buffer, so it makes sense using a lazy allocation scheme. */ #define IOMMU_SIZE_ROUNDUP(sz) \ (round_io_page(sz) + IO_PAGE_SIZE) #define IOMMU_SET_TTE(is, va, tte) \ ((is)->is_tsb[IOTSBSLOT(va)] = (tte)) #define IOMMU_GET_TTE(is, va) \ (is)->is_tsb[IOTSBSLOT(va)] /* Resource helpers */ #define IOMMU_RES_START(res) \ ((bus_addr_t)rman_get_start(res) << IO_PAGE_SHIFT) #define IOMMU_RES_END(res) \ ((bus_addr_t)(rman_get_end(res) + 1) << IO_PAGE_SHIFT) #define IOMMU_RES_SIZE(res) \ ((bus_size_t)rman_get_size(res) << IO_PAGE_SHIFT) /* Helpers for struct bus_dmamap_res */ #define BDR_START(r) IOMMU_RES_START((r)->dr_res) #define BDR_END(r) IOMMU_RES_END((r)->dr_res) #define BDR_SIZE(r) IOMMU_RES_SIZE((r)->dr_res) /* Locking macros */ #define IS_LOCK(is) mtx_lock(&is->is_mtx) #define IS_LOCK_ASSERT(is) mtx_assert(&is->is_mtx, MA_OWNED) #define IS_UNLOCK(is) mtx_unlock(&is->is_mtx) /* Flush a page from the TLB. No locking required, since this is atomic. */ static __inline void iommu_tlb_flush(struct iommu_state *is, bus_addr_t va) { if ((is->is_flags & IOMMU_FIRE) != 0) /* * Direct page flushing is not supported and also not * necessary due to cache snooping. */ return; IOMMU_WRITE8(is, is_iommu, IMR_FLUSH, va); } /* * Flush a page from the streaming buffer. No locking required, since this * is atomic. */ static __inline void iommu_strbuf_flushpg(struct iommu_state *is, bus_addr_t va) { int i; for (i = 0; i < 2; i++) if (is->is_sb[i] != 0) IOMMU_WRITE8(is, is_sb[i], ISR_PGFLUSH, va); } /* * Flush an address from the streaming buffer(s); this is an asynchronous * operation. To make sure that it has completed, iommu_strbuf_sync() needs * to be called. No locking required. */ static __inline void iommu_strbuf_flush(struct iommu_state *is, bus_addr_t va) { iommu_strbuf_flushpg(is, va); } /* Synchronize all outstanding flush operations. */ static __inline void iommu_strbuf_sync(struct iommu_state *is) { IS_LOCK_ASSERT(is); iommu_strbuf_flush_sync(is); } /* LRU queue handling for lazy resource allocation. */ static __inline void iommu_map_insq(struct iommu_state *is, bus_dmamap_t map) { IS_LOCK_ASSERT(is); if (!SLIST_EMPTY(&map->dm_reslist)) { if (map->dm_onq) TAILQ_REMOVE(&is->is_maplruq, map, dm_maplruq); TAILQ_INSERT_TAIL(&is->is_maplruq, map, dm_maplruq); map->dm_onq = 1; } } static __inline void iommu_map_remq(struct iommu_state *is, bus_dmamap_t map) { IS_LOCK_ASSERT(is); if (map->dm_onq) TAILQ_REMOVE(&is->is_maplruq, map, dm_maplruq); map->dm_onq = 0; } /* * initialise the UltraSPARC IOMMU (PCI or SBus): * - allocate and setup the iotsb. * - enable the IOMMU * - initialise the streaming buffers (if they exist) * - create a private DVMA map. */ void iommu_init(const char *name, struct iommu_state *is, u_int tsbsize, uint32_t iovabase, u_int resvpg) { vm_size_t size; vm_offset_t offs; uint64_t end, obpmap, obpptsb, tte; u_int maxtsbsize, obptsbentries, obptsbsize, slot, tsbentries; int i; /* * Setup the IOMMU. * * The sun4u IOMMU is part of the PCI or SBus controller so we * will deal with it here.. * * The IOMMU address space always ends at 0xffffe000, but the starting * address depends on the size of the map. The map size is 1024 * 2 ^ * is->is_tsbsize entries, where each entry is 8 bytes. The start of * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)). */ if ((is->is_flags & IOMMU_FIRE) != 0) { maxtsbsize = IOMMU_TSB512K; /* * We enable bypass in order to be able to use a physical * address for the event queue base. */ is->is_cr = IOMMUCR_SE | IOMMUCR_CM_C_TLB_TBW | IOMMUCR_BE; } else { maxtsbsize = IOMMU_TSB128K; is->is_cr = (tsbsize << IOMMUCR_TSBSZ_SHIFT) | IOMMUCR_DE; } if (tsbsize > maxtsbsize) panic("%s: unsupported TSB size ", __func__); tsbentries = IOMMU_TSBENTRIES(tsbsize); is->is_cr |= IOMMUCR_EN; is->is_tsbsize = tsbsize; is->is_dvmabase = iovabase; if (iovabase == -1) is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); size = IOTSB_BASESZ << is->is_tsbsize; printf("%s: DVMA map: %#lx to %#lx %d entries%s\n", name, is->is_dvmabase, is->is_dvmabase + (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)) - 1, tsbentries, IOMMU_HAS_SB(is) ? ", streaming buffer" : ""); /* * Set up resource mamangement. */ mtx_init(&is->is_mtx, "iommu", NULL, MTX_DEF); end = is->is_dvmabase + (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)); is->is_dvma_rman.rm_type = RMAN_ARRAY; is->is_dvma_rman.rm_descr = "DVMA Memory"; if (rman_init(&is->is_dvma_rman) != 0 || rman_manage_region(&is->is_dvma_rman, (is->is_dvmabase >> IO_PAGE_SHIFT) + resvpg, (end >> IO_PAGE_SHIFT) - 1) != 0) panic("%s: could not initialize DVMA rman", __func__); TAILQ_INIT(&is->is_maplruq); /* * Allocate memory for I/O page tables. They need to be * physically contiguous. */ is->is_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL, PAGE_SIZE, 0); if (is->is_tsb == NULL) panic("%s: contigmalloc failed", __func__); is->is_ptsb = pmap_kextract((vm_offset_t)is->is_tsb); bzero(is->is_tsb, size); /* * Add the PROM mappings to the kernel IOTSB if desired. * Note that the firmware of certain Darwin boards doesn't set * the TSB size correctly. */ if ((is->is_flags & IOMMU_FIRE) != 0) obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_TSB) & IOMMUTB_TSBSZ_MASK) >> IOMMUTB_TSBSZ_SHIFT; else obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_CTL) & IOMMUCR_TSBSZ_MASK) >> IOMMUCR_TSBSZ_SHIFT; obptsbentries = IOMMU_TSBENTRIES(obptsbsize); if (bootverbose) printf("%s: PROM IOTSB size: %d (%d entries)\n", name, obptsbsize, obptsbentries); if ((is->is_flags & IOMMU_PRESERVE_PROM) != 0 && !(PCPU_GET(impl) == CPU_IMPL_ULTRASPARCIIi && obptsbsize == 7)) { if (obptsbentries > tsbentries) panic("%s: PROM IOTSB entries exceed kernel", __func__); obpptsb = IOMMU_READ8(is, is_iommu, IMR_TSB) & IOMMUTB_TB_MASK; for (i = 0; i < obptsbentries; i++) { tte = ldxa(obpptsb + i * 8, ASI_PHYS_USE_EC); if ((tte & IOTTE_V) == 0) continue; slot = tsbentries - obptsbentries + i; if (bootverbose) printf("%s: adding PROM IOTSB slot %d " "(kernel slot %d) TTE: %#lx\n", name, i, slot, tte); obpmap = (is->is_dvmabase + slot * IO_PAGE_SIZE) >> IO_PAGE_SHIFT; if (rman_reserve_resource(&is->is_dvma_rman, obpmap, obpmap, IO_PAGE_SIZE >> IO_PAGE_SHIFT, RF_ACTIVE, NULL) == NULL) panic("%s: could not reserve PROM IOTSB slot " "%d (kernel slot %d)", __func__, i, slot); is->is_tsb[slot] = tte; } } /* * Initialize streaming buffer, if it is there. */ if (IOMMU_HAS_SB(is)) { /* * Find two 64-byte blocks in is_flush that are aligned on * a 64-byte boundary for flushing. */ offs = roundup2((vm_offset_t)is->is_flush, STRBUF_FLUSHSYNC_NBYTES); for (i = 0; i < 2; i++, offs += STRBUF_FLUSHSYNC_NBYTES) { is->is_flushva[i] = (uint64_t *)offs; is->is_flushpa[i] = pmap_kextract(offs); } } /* * Now actually start up the IOMMU. */ iommu_reset(is); } /* * Streaming buffers don't exist on the UltraSPARC IIi; we should have * detected that already and disabled them. If not, we will notice that * they aren't there when the STRBUF_EN bit does not remain. */ void iommu_reset(struct iommu_state *is) { uint64_t tsb; int i; tsb = is->is_ptsb; if ((is->is_flags & IOMMU_FIRE) != 0) { tsb |= is->is_tsbsize; IOMMU_WRITE8(is, is_iommu, IMR_CACHE_INVAL, ~0ULL); } IOMMU_WRITE8(is, is_iommu, IMR_TSB, tsb); IOMMU_WRITE8(is, is_iommu, IMR_CTL, is->is_cr); for (i = 0; i < 2; i++) { if (is->is_sb[i] != 0) { IOMMU_WRITE8(is, is_sb[i], ISR_CTL, STRBUF_EN | ((is->is_flags & IOMMU_RERUN_DISABLE) != 0 ? STRBUF_RR_DIS : 0)); /* No streaming buffers? Disable them. */ if ((IOMMU_READ8(is, is_sb[i], ISR_CTL) & STRBUF_EN) == 0) is->is_sb[i] = 0; } } (void)IOMMU_READ8(is, is_iommu, IMR_CTL); } /* * Enter a mapping into the TSB. No locking required, since each TSB slot is * uniquely assigned to a single map. */ static void iommu_enter(struct iommu_state *is, vm_offset_t va, vm_paddr_t pa, int stream, int flags) { uint64_t tte; KASSERT(va >= is->is_dvmabase, ("%s: va %#lx not in DVMA space", __func__, va)); KASSERT(pa <= is->is_pmaxaddr, ("%s: XXX: physical address too large (%#lx)", __func__, pa)); tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), !(flags & BUS_DMA_NOCACHE), stream); IOMMU_SET_TTE(is, va, tte); iommu_tlb_flush(is, va); #ifdef IOMMU_DIAG IS_LOCK(is); iommu_diag(is, va); IS_UNLOCK(is); #endif } /* * Remove mappings created by iommu_enter(). Flush the streaming buffer, * but do not synchronize it. Returns whether a streaming buffer flush * was performed. */ static int iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len) { int slot, streamed = 0; #ifdef IOMMU_DIAG iommu_diag(is, va); #endif KASSERT(va >= is->is_dvmabase, ("%s: va 0x%lx not in DVMA space", __func__, (u_long)va)); KASSERT(va + len >= va, ("%s: va 0x%lx + len 0x%lx wraps", __func__, (long)va, (long)len)); va = trunc_io_page(va); while (len > 0) { if ((IOMMU_GET_TTE(is, va) & IOTTE_STREAM) != 0) { streamed = 1; iommu_strbuf_flush(is, va); } len -= ulmin(len, IO_PAGE_SIZE); IOMMU_SET_TTE(is, va, 0); iommu_tlb_flush(is, va); if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) { slot = IOTSBSLOT(va); if (len <= IO_PAGE_SIZE || slot % 8 == 7) IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH, is->is_ptsb + slot * 8); } va += IO_PAGE_SIZE; } return (streamed); } /* Decode an IOMMU fault for host bridge error handlers. */ void iommu_decode_fault(struct iommu_state *is, vm_offset_t phys) { bus_addr_t va; long idx; idx = phys - is->is_ptsb; if (phys < is->is_ptsb || idx > (PAGE_SIZE << is->is_tsbsize)) return; va = is->is_dvmabase + (((bus_addr_t)idx >> IOTTE_SHIFT) << IO_PAGE_SHIFT); printf("IOMMU fault virtual address %#lx\n", (u_long)va); } /* * A barrier operation which makes sure that all previous streaming buffer * flushes complete before it returns. */ static int iommu_strbuf_flush_sync(struct iommu_state *is) { struct timeval cur, end; int i; IS_LOCK_ASSERT(is); if (!IOMMU_HAS_SB(is)) return (0); /* * Streaming buffer flushes: * * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If * we're not on a cache line boundary (64-bits): * 2 Store 0 in flag * 3 Store pointer to flag in flushsync * 4 wait till flushsync becomes 0x1 * * If it takes more than .5 sec, something went wrong. */ *is->is_flushva[0] = 1; *is->is_flushva[1] = 1; membar(StoreStore); for (i = 0; i < 2; i++) { if (is->is_sb[i] != 0) { *is->is_flushva[i] = 0; IOMMU_WRITE8(is, is_sb[i], ISR_FLUSHSYNC, is->is_flushpa[i]); } } microuptime(&cur); end.tv_sec = 0; /* * 0.5s is the recommended timeout from the U2S manual. The actual * time required should be smaller by at least a factor of 1000. * We have no choice but to busy-wait. */ end.tv_usec = 500000; timevaladd(&end, &cur); while ((!*is->is_flushva[0] || !*is->is_flushva[1]) && timevalcmp(&cur, &end, <=)) microuptime(&cur); if (!*is->is_flushva[0] || !*is->is_flushva[1]) { panic("%s: flush timeout %ld, %ld at %#lx", __func__, *is->is_flushva[0], *is->is_flushva[1], is->is_flushpa[0]); } return (1); } /* Determine whether we may enable streaming on a mapping. */ static __inline int iommu_use_streaming(struct iommu_state *is, bus_dmamap_t map, bus_size_t size) { return (size >= IOMMU_STREAM_THRESH && IOMMU_HAS_SB(is) && (map->dm_flags & DMF_COHERENT) == 0); } /* * Allocate DVMA virtual memory for a map. The map may not be on a queue, * so that it can be freely modified. */ static int iommu_dvma_valloc(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map, bus_size_t size) { struct resource *res; struct bus_dmamap_res *bdr; bus_size_t align, sgsize; KASSERT(!map->dm_onq, ("%s: map on queue!", __func__)); if ((bdr = malloc(sizeof(*bdr), M_IOMMU, M_NOWAIT)) == NULL) return (EAGAIN); /* * If a boundary is specified, a map cannot be larger than it; however * we do not clip currently, as that does not play well with the lazy * allocation code. * Alignment to a page boundary is always enforced. */ align = (t->dt_alignment + IO_PAGE_MASK) >> IO_PAGE_SHIFT; sgsize = round_io_page(size) >> IO_PAGE_SHIFT; if (t->dt_boundary > 0 && t->dt_boundary < IO_PAGE_SIZE) panic("%s: illegal boundary specified", __func__); res = rman_reserve_resource_bound(&is->is_dvma_rman, 0L, t->dt_lowaddr >> IO_PAGE_SHIFT, sgsize, t->dt_boundary >> IO_PAGE_SHIFT, RF_ACTIVE | rman_make_alignment_flags(align), NULL); if (res == NULL) { free(bdr, M_IOMMU); return (ENOMEM); } bdr->dr_res = res; bdr->dr_used = 0; SLIST_INSERT_HEAD(&map->dm_reslist, bdr, dr_link); return (0); } /* Unload the map and mark all resources as unused, but do not free them. */ static void iommu_dvmamap_vunload(struct iommu_state *is, bus_dmamap_t map) { struct bus_dmamap_res *r; int streamed = 0; IS_LOCK_ASSERT(is); /* for iommu_strbuf_sync() below */ SLIST_FOREACH(r, &map->dm_reslist, dr_link) { streamed |= iommu_remove(is, BDR_START(r), r->dr_used); r->dr_used = 0; } if (streamed) iommu_strbuf_sync(is); } /* Free a DVMA virtual memory resource. */ static __inline void iommu_dvma_vfree_res(bus_dmamap_t map, struct bus_dmamap_res *r) { KASSERT(r->dr_used == 0, ("%s: resource busy!", __func__)); if (r->dr_res != NULL && rman_release_resource(r->dr_res) != 0) printf("warning: DVMA space lost\n"); SLIST_REMOVE(&map->dm_reslist, r, bus_dmamap_res, dr_link); free(r, M_IOMMU); } /* Free all DVMA virtual memory for a map. */ static void iommu_dvma_vfree(struct iommu_state *is, bus_dmamap_t map) { IS_LOCK(is); iommu_map_remq(is, map); iommu_dvmamap_vunload(is, map); IS_UNLOCK(is); while (!SLIST_EMPTY(&map->dm_reslist)) iommu_dvma_vfree_res(map, SLIST_FIRST(&map->dm_reslist)); } /* Prune a map, freeing all unused DVMA resources. */ static bus_size_t iommu_dvma_vprune(struct iommu_state *is, bus_dmamap_t map) { struct bus_dmamap_res *r, *n; bus_size_t freed = 0; IS_LOCK_ASSERT(is); for (r = SLIST_FIRST(&map->dm_reslist); r != NULL; r = n) { n = SLIST_NEXT(r, dr_link); if (r->dr_used == 0) { freed += BDR_SIZE(r); iommu_dvma_vfree_res(map, r); } } if (SLIST_EMPTY(&map->dm_reslist)) iommu_map_remq(is, map); return (freed); } /* * Try to find a suitably-sized (and if requested, -aligned) slab of DVMA * memory with IO page offset voffs. */ static bus_addr_t iommu_dvma_vfindseg(bus_dmamap_t map, vm_offset_t voffs, bus_size_t size, bus_addr_t amask) { struct bus_dmamap_res *r; bus_addr_t dvmaddr, dvmend; KASSERT(!map->dm_onq, ("%s: map on queue!", __func__)); SLIST_FOREACH(r, &map->dm_reslist, dr_link) { dvmaddr = round_io_page(BDR_START(r) + r->dr_used); /* Alignment can only work with voffs == 0. */ dvmaddr = (dvmaddr + amask) & ~amask; dvmaddr += voffs; dvmend = dvmaddr + size; if (dvmend <= BDR_END(r)) { r->dr_used = dvmend - BDR_START(r); return (dvmaddr); } } return (0); } /* * Try to find or allocate a slab of DVMA space; see above. */ static int iommu_dvma_vallocseg(bus_dma_tag_t dt, struct iommu_state *is, bus_dmamap_t map, vm_offset_t voffs, bus_size_t size, bus_addr_t amask, bus_addr_t *addr) { bus_dmamap_t tm, last; bus_addr_t dvmaddr, freed; int error, complete = 0; dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask); /* Need to allocate. */ if (dvmaddr == 0) { while ((error = iommu_dvma_valloc(dt, is, map, voffs + size)) == ENOMEM && !complete) { /* * Free the allocated DVMA of a few maps until * the required size is reached. This is an * approximation to not have to call the allocation * function too often; most likely one free run * will not suffice if not one map was large enough * itself due to fragmentation. */ IS_LOCK(is); freed = 0; last = TAILQ_LAST(&is->is_maplruq, iommu_maplruq_head); do { tm = TAILQ_FIRST(&is->is_maplruq); complete = tm == last; if (tm == NULL) break; freed += iommu_dvma_vprune(is, tm); /* Move to the end. */ iommu_map_insq(is, tm); } while (freed < size && !complete); IS_UNLOCK(is); } if (error != 0) return (error); dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask); KASSERT(dvmaddr != 0, ("%s: allocation failed unexpectedly!", __func__)); } *addr = dvmaddr; return (0); } static int iommu_dvmamem_alloc(bus_dma_tag_t dt, void **vaddr, int flags, bus_dmamap_t *mapp) { struct iommu_state *is = dt->dt_cookie; int error, mflags; /* * XXX: This will break for 32 bit transfers on machines with more * than is->is_pmaxaddr memory. */ if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0) return (error); if ((flags & BUS_DMA_NOWAIT) != 0) mflags = M_NOWAIT; else mflags = M_WAITOK; if ((flags & BUS_DMA_ZERO) != 0) mflags |= M_ZERO; if ((*vaddr = malloc(dt->dt_maxsize, M_IOMMU, mflags)) == NULL) { error = ENOMEM; sparc64_dma_free_map(dt, *mapp); return (error); } if ((flags & BUS_DMA_COHERENT) != 0) (*mapp)->dm_flags |= DMF_COHERENT; /* * Try to preallocate DVMA space. If this fails, it is retried at * load time. */ iommu_dvma_valloc(dt, is, *mapp, IOMMU_SIZE_ROUNDUP(dt->dt_maxsize)); IS_LOCK(is); iommu_map_insq(is, *mapp); IS_UNLOCK(is); return (0); } static void iommu_dvmamem_free(bus_dma_tag_t dt, void *vaddr, bus_dmamap_t map) { struct iommu_state *is = dt->dt_cookie; iommu_dvma_vfree(is, map); sparc64_dma_free_map(dt, map); free(vaddr, M_IOMMU); } static int iommu_dvmamap_create(bus_dma_tag_t dt, int flags, bus_dmamap_t *mapp) { struct iommu_state *is = dt->dt_cookie; bus_size_t totsz, presz, currsz; int error, i, maxpre; if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0) return (error); if ((flags & BUS_DMA_COHERENT) != 0) (*mapp)->dm_flags |= DMF_COHERENT; /* * Preallocate DVMA space; if this fails now, it is retried at load * time. Through bus_dmamap_load_mbuf() and bus_dmamap_load_uio(), * it is possible to have multiple discontiguous segments in a single * map, which is handled by allocating additional resources, instead * of increasing the size, to avoid fragmentation. * Clamp preallocation to IOMMU_MAX_PRE. In some situations we can * handle more; that case is handled by reallocating at map load time. */ totsz = ulmin(IOMMU_SIZE_ROUNDUP(dt->dt_maxsize), IOMMU_MAX_PRE); error = iommu_dvma_valloc(dt, is, *mapp, totsz); if (error != 0) return (0); /* * Try to be smart about preallocating some additional segments if * needed. */ maxpre = imin(dt->dt_nsegments, IOMMU_MAX_PRE_SEG); presz = dt->dt_maxsize / maxpre; for (i = 1; i < maxpre && totsz < IOMMU_MAX_PRE; i++) { currsz = round_io_page(ulmin(presz, IOMMU_MAX_PRE - totsz)); error = iommu_dvma_valloc(dt, is, *mapp, currsz); if (error != 0) break; totsz += currsz; } IS_LOCK(is); iommu_map_insq(is, *mapp); IS_UNLOCK(is); return (0); } static int iommu_dvmamap_destroy(bus_dma_tag_t dt, bus_dmamap_t map) { struct iommu_state *is = dt->dt_cookie; iommu_dvma_vfree(is, map); sparc64_dma_free_map(dt, map); return (0); } /* * IOMMU DVMA operations, common to PCI and SBus */ static int iommu_dvmamap_load_buffer(bus_dma_tag_t dt, bus_dmamap_t map, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { bus_addr_t amask, dvmaddr, dvmoffs; bus_size_t sgsize, esize; struct iommu_state *is; vm_offset_t vaddr, voffs; vm_paddr_t curaddr; int error, firstpg, sgcnt; - int first; u_int slot; is = dt->dt_cookie; if (*segp == -1) { if ((map->dm_flags & DMF_LOADED) != 0) { #ifdef DIAGNOSTIC printf("%s: map still in use\n", __func__); #endif bus_dmamap_unload(dt, map); } /* * Make sure that the map is not on a queue so that the * resource list may be safely accessed and modified without * needing the lock to cover the whole operation. */ IS_LOCK(is); iommu_map_remq(is, map); IS_UNLOCK(is); amask = dt->dt_alignment - 1; } else amask = 0; KASSERT(buflen != 0, ("%s: buflen == 0!", __func__)); if (buflen > dt->dt_maxsize) return (EINVAL); if (segs == NULL) segs = dt->dt_segments; vaddr = (vm_offset_t)buf; voffs = vaddr & IO_PAGE_MASK; /* Try to find a slab that is large enough. */ error = iommu_dvma_vallocseg(dt, is, map, voffs, buflen, amask, &dvmaddr); if (error != 0) return (error); sgcnt = *segp; firstpg = 1; map->dm_flags &= ~DMF_STREAMED; map->dm_flags |= iommu_use_streaming(is, map, buflen) != 0 ? DMF_STREAMED : 0; for (; buflen > 0; ) { /* * Get the physical address for this page. */ if (pmap == kernel_pmap) curaddr = pmap_kextract(vaddr); else curaddr = pmap_extract(pmap, vaddr); /* * Compute the segment size, and adjust counts. */ sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK); if (buflen < sgsize) sgsize = buflen; buflen -= sgsize; vaddr += sgsize; dvmoffs = trunc_io_page(dvmaddr); iommu_enter(is, dvmoffs, trunc_io_page(curaddr), (map->dm_flags & DMF_STREAMED) != 0, flags); if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) { slot = IOTSBSLOT(dvmoffs); if (buflen <= 0 || slot % 8 == 7) IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH, is->is_ptsb + slot * 8); } /* * Chop the chunk up into segments of at most maxsegsz, but try * to fill each segment as well as possible. */ if (!firstpg) { esize = ulmin(sgsize, dt->dt_maxsegsz - segs[sgcnt].ds_len); segs[sgcnt].ds_len += esize; sgsize -= esize; dvmaddr += esize; } while (sgsize > 0) { sgcnt++; if (sgcnt >= dt->dt_nsegments) return (EFBIG); /* * No extra alignment here - the common practice in * the busdma code seems to be that only the first * segment needs to satisfy the alignment constraints * (and that only for bus_dmamem_alloc()ed maps). * It is assumed that such tags have maxsegsize >= * maxsize. */ esize = ulmin(sgsize, dt->dt_maxsegsz); segs[sgcnt].ds_addr = dvmaddr; segs[sgcnt].ds_len = esize; sgsize -= esize; dvmaddr += esize; } firstpg = 0; } *segp = sgcnt; return (0); } static void iommu_dvmamap_mayblock(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmamap_callback_t *callback, void *callback_arg, int *flags) { } static void -iommu_dvmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_dvmamap_complete(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmamap_callback_t *callback, void *callback_arg, int nsegs, int error) { struct iommu_state *is = dt->dt_cookie; IS_LOCK(is); iommu_map_insq(is, map); if (error != 0) { iommu_dvmamap_vunload(is, map); IS_UNLOCK(is); (*callback)(callback_arg, dt->dt_segments, 0, error); } else { IS_UNLOCK(is); map->dm_flags |= DMF_LOADED; (*callback)(callback_arg, dt->dt_segments, nsegs, 0); } } static void -iommu_dvmamap_complete2(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_dvmamap_complete2(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmamap_callback2_t *callback, void *callback_arg, int nsegs, bus_size_t len, int error) { struct iommu_state *is = dt->dt_cookie; IS_LOCK(is); iommu_map_insq(is, map); if (error != 0) { iommu_dvmamap_vunload(is, map); IS_UNLOCK(is); (*callback)(callback_arg, dt->dt_segments, 0, 0, error); } else { IS_UNLOCK(is); map->dm_flags |= DMF_LOADED; (*callback)(callback_arg, dt->dt_segments, nsegs, len, 0); } } static void -iommu_dvmamap_directseg(bus_dma_tag_t dmat, bus_dmamap_t map, +iommu_dvmamap_directseg(bus_dma_tag_t dt, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error) { struct iommu_state *is = dt->dt_cookie; IS_LOCK(is); iommu_map_insq(is, map); if (error != 0) { iommu_dvmamap_vunload(is, map); IS_UNLOCK(is); } else { IS_UNLOCK(is); map->dm_flags |= DMF_LOADED; } - return (error); } static void iommu_dvmamap_unload(bus_dma_tag_t dt, bus_dmamap_t map) { struct iommu_state *is = dt->dt_cookie; if ((map->dm_flags & DMF_LOADED) == 0) return; IS_LOCK(is); iommu_dvmamap_vunload(is, map); iommu_map_insq(is, map); IS_UNLOCK(is); map->dm_flags &= ~DMF_LOADED; } static void iommu_dvmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op) { struct iommu_state *is = dt->dt_cookie; struct bus_dmamap_res *r; vm_offset_t va; vm_size_t len; int streamed = 0; if ((map->dm_flags & DMF_LOADED) == 0) return; if ((map->dm_flags & DMF_STREAMED) != 0 && ((op & BUS_DMASYNC_POSTREAD) != 0 || (op & BUS_DMASYNC_PREWRITE) != 0)) { IS_LOCK(is); SLIST_FOREACH(r, &map->dm_reslist, dr_link) { va = (vm_offset_t)BDR_START(r); len = r->dr_used; /* * If we have a streaming buffer, flush it here * first. */ while (len > 0) { if ((IOMMU_GET_TTE(is, va) & IOTTE_STREAM) != 0) { streamed = 1; iommu_strbuf_flush(is, va); } len -= ulmin(len, IO_PAGE_SIZE); va += IO_PAGE_SIZE; } } if (streamed) iommu_strbuf_sync(is); IS_UNLOCK(is); } if ((op & BUS_DMASYNC_PREWRITE) != 0) membar(Sync); } #ifdef IOMMU_DIAG /* * Perform an IOMMU diagnostic access and print the tag belonging to va. */ static void iommu_diag(struct iommu_state *is, vm_offset_t va) { int i; uint64_t data, tag; if ((is->is_flags & IOMMU_FIRE) != 0) return; IS_LOCK_ASSERT(is); IOMMU_WRITE8(is, is_dva, 0, trunc_io_page(va)); membar(StoreStore | StoreLoad); printf("%s: tte entry %#lx", __func__, IOMMU_GET_TTE(is, va)); if (is->is_dtcmp != 0) { printf(", tag compare register is %#lx\n", IOMMU_READ8(is, is_dtcmp, 0)); } else printf("\n"); for (i = 0; i < 16; i++) { tag = IOMMU_READ8(is, is_dtag, i * 8); data = IOMMU_READ8(is, is_ddram, i * 8); printf("%s: tag %d: %#lx, vpn %#lx, err %lx; " "data %#lx, pa %#lx, v %d, c %d\n", __func__, i, tag, (tag & IOMMU_DTAG_VPNMASK) << IOMMU_DTAG_VPNSHIFT, (tag & IOMMU_DTAG_ERRMASK) >> IOMMU_DTAG_ERRSHIFT, data, (data & IOMMU_DDATA_PGMASK) << IOMMU_DDATA_PGSHIFT, (data & IOMMU_DDATA_V) != 0, (data & IOMMU_DDATA_C) != 0); } } #endif /* IOMMU_DIAG */ struct bus_dma_methods iommu_dma_methods = { iommu_dvmamap_create, iommu_dvmamap_destroy, iommu_dvmamap_load_buffer, iommu_dvmamap_mayblock, iommu_dvmamap_complete, iommu_dvmamap_complete2, iommu_dvmamap_directseg, iommu_dvmamap_unload, iommu_dvmamap_sync, iommu_dvmamem_alloc, iommu_dvmamem_free, };