Index: sys/amd64/include/bus_dma.h =================================================================== --- sys/amd64/include/bus_dma.h +++ sys/amd64/include/bus_dma.h @@ -30,5 +30,6 @@ #define _AMD64_BUS_DMA_H_ #include +#include #endif /* _AMD64_BUS_DMA_H_ */ Index: sys/arm/arm/busdma_machdep-v4.c =================================================================== --- sys/arm/arm/busdma_machdep-v4.c +++ sys/arm/arm/busdma_machdep-v4.c @@ -1006,7 +1006,7 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); @@ -1127,14 +1127,14 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { @@ -1159,7 +1159,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_page *bpage; struct bounce_zone *bz; @@ -1332,7 +1332,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct sync_list *sl, *end; int bufaligned; Index: sys/arm/arm/busdma_machdep-v6.c =================================================================== --- sys/arm/arm/busdma_machdep-v6.c +++ sys/arm/arm/busdma_machdep-v6.c @@ -1056,7 +1056,7 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); @@ -1181,14 +1181,14 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { @@ -1212,7 +1212,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_page *bpage; struct bounce_zone *bz; @@ -1312,7 +1312,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; struct sync_list *sl, *end; Index: sys/arm/include/bus_dma.h =================================================================== --- sys/arm/include/bus_dma.h +++ sys/arm/include/bus_dma.h @@ -67,6 +67,7 @@ #define _ARM_BUS_DMA_H #include +#include /* Bus Space DMA macros */ Index: sys/arm64/arm64/busdma_machdep.c =================================================================== --- sys/arm64/arm64/busdma_machdep.c +++ sys/arm64/arm64/busdma_machdep.c @@ -314,7 +314,7 @@ } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { struct bus_dma_tag_common *tc; @@ -337,7 +337,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bus_dma_tag_common *tc; @@ -346,7 +346,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bus_dma_tag_common *tc; Index: sys/arm64/include/bus_dma.h =================================================================== --- sys/arm64/include/bus_dma.h +++ sys/arm64/include/bus_dma.h @@ -4,5 +4,6 @@ #define _MACHINE_BUS_DMA_H_ #include +#include #endif /* !_MACHINE_BUS_DMA_H_ */ Index: sys/i386/include/bus_dma.h =================================================================== --- sys/i386/include/bus_dma.h +++ sys/i386/include/bus_dma.h @@ -30,5 +30,6 @@ #define _I386_BUS_DMA_H_ #include +#include #endif /* _I386_BUS_DMA_H_ */ Index: sys/mips/include/bus_dma.h =================================================================== --- sys/mips/include/bus_dma.h +++ sys/mips/include/bus_dma.h @@ -30,5 +30,6 @@ #define _MIPS_BUS_DMA_H_ #include +#include #endif /* _MIPS_BUS_DMA_H_ */ Index: sys/mips/mips/busdma_machdep.c =================================================================== --- sys/mips/mips/busdma_machdep.c +++ sys/mips/mips/busdma_machdep.c @@ -926,7 +926,7 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); return (EFBIG); /* XXX better return value here? */ } return (0); @@ -1024,14 +1024,14 @@ * Did we fit? */ if (buflen != 0) { - _bus_dmamap_unload(dmat, map); + bus_dmamap_unload(dmat, map); error = EFBIG; /* XXX better return value here? */ } return (error); } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { @@ -1056,7 +1056,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_page *bpage; @@ -1229,7 +1229,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct sync_list *sl, *end; int aligned; Index: sys/powerpc/include/bus_dma.h =================================================================== --- sys/powerpc/include/bus_dma.h +++ sys/powerpc/include/bus_dma.h @@ -29,6 +29,7 @@ #define _POWERPC_BUS_DMA_H_ #include +#include struct device; Index: sys/powerpc/powerpc/busdma_machdep.c =================================================================== --- sys/powerpc/powerpc/busdma_machdep.c +++ sys/powerpc/powerpc/busdma_machdep.c @@ -841,7 +841,7 @@ } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { @@ -879,7 +879,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bounce_page *bpage; @@ -895,7 +895,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; vm_offset_t datavaddr, tempvaddr; Index: sys/riscv/include/bus_dma.h =================================================================== --- sys/riscv/include/bus_dma.h +++ sys/riscv/include/bus_dma.h @@ -4,5 +4,6 @@ #define _MACHINE_BUS_DMA_H_ #include +#include #endif /* !_MACHINE_BUS_DMA_H_ */ Index: sys/riscv/riscv/busdma_machdep.c =================================================================== --- sys/riscv/riscv/busdma_machdep.c +++ sys/riscv/riscv/busdma_machdep.c @@ -69,7 +69,7 @@ } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { @@ -88,14 +88,14 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { panic("busdma"); } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { panic("busdma"); Index: sys/sparc64/include/bus_dma.h =================================================================== --- sys/sparc64/include/bus_dma.h +++ sys/sparc64/include/bus_dma.h @@ -124,29 +124,48 @@ struct bus_dma_methods *dt_mt; }; -#define bus_dmamap_create(t, f, p) \ - ((t)->dt_mt->dm_dmamap_create((t), (f), (p))) -#define bus_dmamap_destroy(t, p) \ - ((t)->dt_mt->dm_dmamap_destroy((t), (p))) -#define _bus_dmamap_load_phys(t, m, b, l, f, s, sp) \ - ((t)->dt_mt->dm_dmamap_load_phys((t), (m), (b), (l), \ - (f), (s), (sp))) -#define _bus_dmamap_load_buffer(t, m, b, l, p, f, s, sp) \ - ((t)->dt_mt->dm_dmamap_load_buffer((t), (m), (b), (l), (p), \ - (f), (s), (sp))) -#define _bus_dmamap_waitok(t, m, mem, c, ca) \ - ((t)->dt_mt->dm_dmamap_waitok((t), (m), (mem), (c), (ca))) -#define _bus_dmamap_complete(t, m, s, n, e) \ - ((t)->dt_mt->dm_dmamap_complete((t), (m), (s), (n), (e))) -#define bus_dmamap_unload(t, p) \ - ((t)->dt_mt->dm_dmamap_unload((t), (p))) -#define bus_dmamap_sync(t, m, op) \ - ((t)->dt_mt->dm_dmamap_sync((t), (m), (op))) -#define bus_dmamem_alloc(t, v, f, m) \ - ((t)->dt_mt->dm_dmamem_alloc((t), (v), (f), (m))) -#define bus_dmamem_free(t, v, m) \ - ((t)->dt_mt->dm_dmamem_free((t), (v), (m))) -#define _bus_dmamap_load_ma(t, m, a, tt, o, f, s, p) \ - bus_dmamap_load_ma_triv((t), (m), (a), (tt), (o), (f), (s), (p)) +static inline bus_dma_segment_t* +_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dma_segment_t *segs, int nsegs, int error) +{ + return (dmat->dt_mt->dm_dmamap_complete(dmat, map, segs, + nsegs, error)); +} + +static inline int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, + void *buf, bus_size_t buflen, struct pmap *pmap, + int flags, bus_dma_segment_t *segs, int *segp) +{ + return (dmat->dt_mt->dm_dmamap_load_buffer(dmat, map, buf, buflen, + pmap, flags, segs, segp)); +} + +static inline int +_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, + struct vm_page **ma, bus_size_t tlen, int ma_offs, + int flags, bus_dma_segment_t *segs, int *segp) +{ + return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, + segs, segp)); +} + +static inline int +_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t paddr, bus_size_t buflen, + int flags, bus_dma_segment_t *segs, int *segp) +{ + return (dmat->dt_mt->dm_dmamap_load_phys(dmat, map, paddr, buflen, + flags, segs, segp)); +} + +static inline void +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, + void *callback_arg) +{ + return (dmat->dt_mt->dm_dmamap_waitok(dmat, map, mem, callback, + callback_arg)); +} #endif /* !_SPARC64_BUS_DMA_H_ */ Index: sys/sparc64/sparc64/bus_machdep.c =================================================================== --- sys/sparc64/sparc64/bus_machdep.c +++ sys/sparc64/sparc64/bus_machdep.c @@ -276,6 +276,42 @@ return (0); } +int +bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) +{ + return (dmat->dt_mt->dm_dmamap_create(dmat, flags, mapp)); +} + +int +bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + return (dmat->dt_mt->dm_dmamap_destroy(dmat, map)); +} + +void +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +{ + dmat->dt_mt->dm_dmamap_sync(dmat, map, op); +} + +void +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + dmat->dt_mt->dm_dmamap_unload(dmat, map); +} + +int +bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp) +{ + return (dmat->dt_mt->dm_dmamem_alloc(dmat, vaddr, flags, mapp)); +} + +void +bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) +{ + dmat->dt_mt->dm_dmamem_free(dmat, vaddr, map); +} + /* Allocate/free a tag, and do the necessary management work. */ int sparc64_dma_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp) Index: sys/sys/bus_dma.h =================================================================== --- sys/sys/bus_dma.h +++ sys/sys/bus_dma.h @@ -249,13 +249,6 @@ bus_dma_segment_t *segs, int *segp); /* - * XXX sparc64 uses the same interface, but a much different implementation. - * for the sparc64 arch contains the equivalent - * declarations. - */ -#if !defined(__sparc64__) - -/* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. */ @@ -283,70 +276,13 @@ /* * Perform a synchronization operation on the given map. If the map - * is NULL we have a fully IO-coherent system. On every ARM architecture - * there must be a memory barrier placed to ensure that all data - * accesses are visible before going any further. - */ -void _bus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_dmasync_op_t); -#if defined(__arm__) - #define __BUS_DMAMAP_SYNC_DEFAULT mb() -#elif defined(__aarch64__) - #define __BUS_DMAMAP_SYNC_DEFAULT dmb(sy) -#else - #define __BUS_DMAMAP_SYNC_DEFAULT do {} while (0) -#endif -#define bus_dmamap_sync(dmat, dmamap, op) \ - do { \ - if ((dmamap) != NULL) \ - _bus_dmamap_sync(dmat, dmamap, op); \ - else \ - __BUS_DMAMAP_SYNC_DEFAULT; \ - } while (0) + * is NULL we have a fully IO-coherent system. + */ +void bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t dmamap, bus_dmasync_op_t op); /* * Release the mapping held by map. */ -void _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map); -#define bus_dmamap_unload(dmat, dmamap) \ - do { \ - if ((dmamap) != NULL) \ - _bus_dmamap_unload(dmat, dmamap); \ - } while (0) - -/* - * The following functions define the interface between the MD and MI - * busdma layers. These are not intended for consumption by driver - * software. - */ -void __bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, - struct memdesc *mem, - bus_dmamap_callback_t *callback, - void *callback_arg); - -#define _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg) \ - do { \ - if ((map) != NULL) \ - __bus_dmamap_waitok(dmat, map, mem, callback, \ - callback_arg); \ - } while (0); - -int _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, - void *buf, bus_size_t buflen, struct pmap *pmap, - int flags, bus_dma_segment_t *segs, int *segp); - -int _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_paddr_t paddr, bus_size_t buflen, - int flags, bus_dma_segment_t *segs, int *segp); - -int _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, - struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, - bus_dma_segment_t *segs, int *segp); - -bus_dma_segment_t *_bus_dmamap_complete(bus_dma_tag_t dmat, - bus_dmamap_t map, - bus_dma_segment_t *segs, - int nsegs, int error); - -#endif /* __sparc64__ */ +void bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t dmamap); #endif /* _BUS_DMA_H_ */ Index: sys/sys/bus_dma_internal.h =================================================================== --- /dev/null +++ sys/sys/bus_dma_internal.h @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2017 Jason A. Harmening. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _BUS_DMA_INTERNAL_H_ +#define _BUS_DMA_INTERNAL_H_ + +/* + * The following functions define the interface between the MD and MI + * busdma layers. These are not intended for consumption by driver + * software. + */ + +bus_dma_segment_t *_bus_dmamap_complete(bus_dma_tag_t dmat, + bus_dmamap_t map, bus_dma_segment_t *segs, + int nsegs, int error); + +int _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, + void *buf, bus_size_t buflen, struct pmap *pmap, + int flags, bus_dma_segment_t *segs, int *segp); + +int _bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, + struct vm_page **ma, bus_size_t tlen, int ma_offs, + int flags, bus_dma_segment_t *segs, int *segp); + +int _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t paddr, bus_size_t buflen, + int flags, bus_dma_segment_t *segs, int *segp); + +void _bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, + void *callback_arg); + +#endif /* _BUS_DMA_INTERNAL_H_ */ + Index: sys/x86/x86/busdma_machdep.c =================================================================== --- sys/x86/x86/busdma_machdep.c +++ sys/x86/x86/busdma_machdep.c @@ -324,7 +324,7 @@ } void -__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { struct bus_dma_tag_common *tc; @@ -347,7 +347,7 @@ * Release the mapping held by map. */ void -_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) { struct bus_dma_tag_common *tc; @@ -356,7 +356,7 @@ } void -_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bus_dma_tag_common *tc;