diff --git a/sys/arm/arm/busdma_machdep.c b/sys/arm/arm/busdma_machdep.c --- a/sys/arm/arm/busdma_machdep.c +++ b/sys/arm/arm/busdma_machdep.c @@ -169,6 +169,7 @@ #define dmat_alignment(dmat) ((dmat)->alignment) #define dmat_flags(dmat) ((dmat)->flags) +#define dmat_highaddr(dmat) ((dmat)->highaddr) #define dmat_lowaddr(dmat) ((dmat)->lowaddr) #define dmat_lockfunc(dmat) ((dmat)->lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->lockfuncarg) @@ -340,18 +341,10 @@ if (cacheline_bounce(map, paddr, size)) return (1); - /* - * The tag already contains ancestors' alignment restrictions so this - * check doesn't need to be inside the loop. - */ - if (alignment_bounce(dmat, paddr)) - return (1); - /* * Check the tag's exclusion zone. */ - if (exclusion_bounce(dmat) && - paddr >= dmat->lowaddr && paddr <= dmat->highaddr) + if (exclusion_bounce(dmat) && addr_needs_bounce(dmat, paddr)) return (1); return (0); diff --git a/sys/arm64/arm64/busdma_bounce.c b/sys/arm64/arm64/busdma_bounce.c --- a/sys/arm64/arm64/busdma_bounce.c +++ b/sys/arm64/arm64/busdma_bounce.c @@ -107,7 +107,6 @@ struct sync_list slist[]; }; -int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -120,6 +119,7 @@ #define dmat_alignment(dmat) ((dmat)->common.alignment) #define dmat_domain(dmat) ((dmat)->common.domain) #define dmat_flags(dmat) ((dmat)->common.flags) +#define dmat_highaddr(dmat) ((dmat)->common.highaddr) #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) @@ -225,11 +225,8 @@ if (cacheline_bounce(dmat, map, paddr, size)) return (true); - if (alignment_bounce(dmat, paddr)) - return (true); - if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && - bus_dma_run_filter(&dmat->common, paddr)) + addr_needs_bounce(dmat, paddr)) return (true); return (false); diff --git a/sys/arm64/arm64/busdma_machdep.c b/sys/arm64/arm64/busdma_machdep.c --- a/sys/arm64/arm64/busdma_machdep.c +++ b/sys/arm64/arm64/busdma_machdep.c @@ -49,24 +49,6 @@ #include #include -/* - * Return true if a match is made. - * - * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. - * - * If paddr is within the bounds of the dma tag then call the filter callback - * to check for a match, if there is no filter callback then assume a match. - */ -int -bus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) -{ - - if (paddr > tc->lowaddr && paddr <= tc->highaddr) - return (1); - - return (0); -} - int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, diff --git a/sys/arm64/include/bus_dma_impl.h b/sys/arm64/include/bus_dma_impl.h --- a/sys/arm64/include/bus_dma_impl.h +++ b/sys/arm64/include/bus_dma_impl.h @@ -77,7 +77,6 @@ bus_dmasync_op_t op); }; -int bus_dma_run_filter(struct bus_dma_tag_common *dmat, bus_addr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_size_t maxsize, int nsegments, diff --git a/sys/kern/subr_busdma_bounce.c b/sys/kern/subr_busdma_bounce.c --- a/sys/kern/subr_busdma_bounce.c +++ b/sys/kern/subr_busdma_bounce.c @@ -155,6 +155,22 @@ return (bz->sysctl_tree_top); } +/* + * Returns true if the address falls within the tag's exclusion window, or + * fails to meet its alignment requirements. + */ +static bool +addr_needs_bounce(bus_dma_tag_t dmat, bus_addr_t paddr) +{ + + if (paddr > dmat_lowaddr(dmat) && paddr <= dmat_highaddr(dmat)) + return (true); + if (!vm_addr_align_ok(paddr, dmat_alignment(dmat))) + return (true); + + return (false); +} + static int alloc_bounce_zone(bus_dma_tag_t dmat) { diff --git a/sys/powerpc/powerpc/busdma_machdep.c b/sys/powerpc/powerpc/busdma_machdep.c --- a/sys/powerpc/powerpc/busdma_machdep.c +++ b/sys/powerpc/powerpc/busdma_machdep.c @@ -99,10 +99,9 @@ static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); -static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); - #define dmat_alignment(dmat) ((dmat)->alignment) #define dmat_flags(dmat) ((dmat)->flags) +#define dmat_highaddr(dmat) ((dmat)->highaddr) #define dmat_lowaddr(dmat) ((dmat)->lowaddr) #define dmat_lockfunc(dmat) ((dmat)->lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->lockfuncarg) @@ -110,27 +109,20 @@ #include "../../kern/subr_busdma_bounce.c" /* - * Return true if a match is made. - * - * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. - * - * If paddr is within the bounds of the dma tag then call the filter callback - * to check for a match, if there is no filter callback then assume a match. + * Returns true if the address falls within the tag's exclusion window, or + * fails to meet its alignment requirements. */ -static __inline int -run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) +static __inline bool +must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr) { - int retval; - - retval = 0; - if (dmat->iommu == NULL && - paddr > dmat->lowaddr && paddr <= dmat->highaddr) - retval = 1; + if (dmat->iommu == NULL && paddr > dmat->lowaddr && + paddr <= dmat->highaddr) + return (true); if (!vm_addr_align_ok(paddr, dmat->alignment)) - retval = 1; + return (true); - return (retval); + return (false); } #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 @@ -492,7 +484,7 @@ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->maxsegsz); - if (run_filter(dmat, curaddr) != 0) { + if (must_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); map->pagesneeded++; @@ -532,7 +524,7 @@ paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (run_filter(dmat, paddr) != 0) { + if (must_bounce(dmat, paddr)) { sg_len = roundup2(sg_len, dmat->alignment); map->pagesneeded++; } @@ -614,7 +606,7 @@ while (buflen > 0) { curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); - if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); @@ -694,7 +686,7 @@ */ max_sgsize = MIN(buflen, dmat->maxsegsz); sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); - if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) { sgsize = roundup2(sgsize, dmat->alignment); sgsize = MIN(sgsize, max_sgsize); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, diff --git a/sys/riscv/include/bus_dma_impl.h b/sys/riscv/include/bus_dma_impl.h --- a/sys/riscv/include/bus_dma_impl.h +++ b/sys/riscv/include/bus_dma_impl.h @@ -74,7 +74,6 @@ bus_dmasync_op_t op); }; -int bus_dma_run_filter(struct bus_dma_tag_common *dmat, bus_addr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, diff --git a/sys/riscv/riscv/busdma_bounce.c b/sys/riscv/riscv/busdma_bounce.c --- a/sys/riscv/riscv/busdma_bounce.c +++ b/sys/riscv/riscv/busdma_bounce.c @@ -103,7 +103,6 @@ struct sync_list slist[]; }; -int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags); static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -113,6 +112,7 @@ #define dmat_alignment(dmat) ((dmat)->common.alignment) #define dmat_flags(dmat) ((dmat)->common.flags) +#define dmat_highaddr(dmat) ((dmat)->common.highaddr) #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) @@ -494,7 +494,7 @@ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->common.maxsegsz); - if (bus_dma_run_filter(&dmat->common, curaddr)) { + if (addr_needs_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); map->pagesneeded++; @@ -535,7 +535,7 @@ paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (bus_dma_run_filter(&dmat->common, paddr) != 0) { + if (addr_needs_bounce(dmat, paddr)) { sg_len = roundup2(sg_len, dmat->common.alignment); map->pagesneeded++; @@ -621,7 +621,7 @@ sgsize = MIN(buflen, dmat->common.maxsegsz); if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) && map->pagesneeded != 0 && - bus_dma_run_filter(&dmat->common, curaddr)) { + addr_needs_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); @@ -708,7 +708,7 @@ sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) && map->pagesneeded != 0 && - bus_dma_run_filter(&dmat->common, curaddr)) { + addr_needs_bounce(dmat, curaddr)) { sgsize = roundup2(sgsize, dmat->common.alignment); sgsize = MIN(sgsize, max_sgsize); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, diff --git a/sys/riscv/riscv/busdma_machdep.c b/sys/riscv/riscv/busdma_machdep.c --- a/sys/riscv/riscv/busdma_machdep.c +++ b/sys/riscv/riscv/busdma_machdep.c @@ -48,27 +48,6 @@ #include #include -/* - * Return true if a match is made. - * - * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. - * - * If paddr is within the bounds of the dma tag then call the filter callback - * to check for a match, if there is no filter callback then assume a match. - */ -int -bus_dma_run_filter(struct bus_dma_tag_common *tc, bus_addr_t paddr) -{ - int retval; - - retval = 0; - if ((paddr > tc->lowaddr && paddr <= tc->highaddr) || - !vm_addr_align_ok(paddr, tc->alignment)) - retval = 1; - - return (retval); -} - int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, diff --git a/sys/x86/include/busdma_impl.h b/sys/x86/include/busdma_impl.h --- a/sys/x86/include/busdma_impl.h +++ b/sys/x86/include/busdma_impl.h @@ -82,7 +82,6 @@ #endif }; -int bus_dma_run_filter(struct bus_dma_tag_common *dmat, vm_paddr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, diff --git a/sys/x86/x86/busdma_bounce.c b/sys/x86/x86/busdma_bounce.c --- a/sys/x86/x86/busdma_bounce.c +++ b/sys/x86/x86/busdma_bounce.c @@ -110,6 +110,7 @@ #define dmat_alignment(dmat) ((dmat)->common.alignment) #define dmat_domain(dmat) ((dmat)->common.domain) #define dmat_flags(dmat) ((dmat)->common.flags) +#define dmat_highaddr(dmat) ((dmat)->common.highaddr) #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) @@ -490,7 +491,7 @@ curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->common.maxsegsz); - if (bus_dma_run_filter(&dmat->common, curaddr)) { + if (addr_needs_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); if (pagesneeded == NULL) @@ -546,7 +547,7 @@ paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (bus_dma_run_filter(&dmat->common, paddr) != 0) { + if (addr_needs_bounce(dmat, paddr)) { sg_len = roundup2(sg_len, dmat->common.alignment); map->pagesneeded++; @@ -583,7 +584,7 @@ sg_len = PAGE_SIZE - ma_offs; max_sgsize = MIN(buflen, dmat->common.maxsegsz); sg_len = MIN(sg_len, max_sgsize); - if (bus_dma_run_filter(&dmat->common, paddr) != 0) { + if (addr_needs_bounce(dmat, paddr)) { sg_len = roundup2(sg_len, dmat->common.alignment); sg_len = MIN(sg_len, max_sgsize); @@ -684,7 +685,7 @@ sgsize = MIN(buflen, dmat->common.maxsegsz); if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0 && map->pagesneeded != 0 && - bus_dma_run_filter(&dmat->common, curaddr)) { + addr_needs_bounce(dmat, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, 0, sgsize); @@ -752,7 +753,7 @@ sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0 && map->pagesneeded != 0 && - bus_dma_run_filter(&dmat->common, curaddr)) { + addr_needs_bounce(dmat, curaddr)) { sgsize = roundup2(sgsize, dmat->common.alignment); sgsize = MIN(sgsize, max_sgsize); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 0, @@ -819,7 +820,7 @@ sgsize = PAGE_SIZE - ma_offs; if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0 && map->pagesneeded != 0 && - bus_dma_run_filter(&dmat->common, paddr)) { + addr_needs_bounce(dmat, paddr)) { sgsize = roundup2(sgsize, dmat->common.alignment); sgsize = MIN(sgsize, max_sgsize); KASSERT(vm_addr_align_ok(sgsize, diff --git a/sys/x86/x86/busdma_machdep.c b/sys/x86/x86/busdma_machdep.c --- a/sys/x86/x86/busdma_machdep.c +++ b/sys/x86/x86/busdma_machdep.c @@ -53,28 +53,6 @@ #include #include -/* - * Return true if a match is made. - * - * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. - * - * If paddr is within the bounds of the dma tag then call the filter callback - * to check for a match, if there is no filter callback then assume a match. - */ -int -bus_dma_run_filter(struct bus_dma_tag_common *tc, vm_paddr_t paddr) -{ - int retval; - - retval = 0; - if (paddr >= BUS_SPACE_MAXADDR || - (paddr > tc->lowaddr && paddr <= tc->highaddr) || - !vm_addr_align_ok(paddr, tc->alignment)) - retval = 1; - - return (retval); -} - int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr,