diff --git a/sys/arm64/arm64/busdma_bounce.h b/sys/arm64/arm64/busdma_bounce.h --- a/sys/arm64/arm64/busdma_bounce.h +++ b/sys/arm64/arm64/busdma_bounce.h @@ -31,7 +31,11 @@ * SUCH DAMAGE. */ +#ifdef BUS_DMA_NON_COHERENT_IMPL #define DMA_FUNC(f) f +#else +#define DMA_FUNC(f) f ## _coherent +#endif static bool DMA_FUNC(_bus_dmamap_pagesneeded)(bus_dma_tag_t, bus_dmamap_t, vm_paddr_t, bus_size_t, int *); @@ -48,8 +52,10 @@ if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) return (true); +#ifdef BUS_DMA_NON_COHERENT_IMPL if (cacheline_bounce(dmat, map, paddr, size)) return (true); +#endif if (alignment_bounce(dmat, paddr)) return (true); @@ -62,8 +68,10 @@ bus_size_t size) { +#ifdef BUS_DMA_NON_COHERENT_IMPL if (cacheline_bounce(dmat, map, paddr, size)) return (true); +#endif if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && addr_needs_bounce(dmat, paddr)) @@ -178,9 +186,12 @@ vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { +#ifdef BUS_DMA_NON_COHERENT_IMPL struct sync_list *sl; + bus_addr_t sl_end; +#endif bus_size_t sgsize; - bus_addr_t curaddr, sl_end; + bus_addr_t curaddr; int error; if (segs == NULL) @@ -195,8 +206,10 @@ } } +#ifdef BUS_DMA_NON_COHERENT_IMPL sl = map->slist + map->sync_count - 1; sl_end = 0; +#endif while (buflen > 0) { curaddr = buf; @@ -216,6 +229,7 @@ sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); +#ifdef BUS_DMA_NON_COHERENT_IMPL } else if ((map->flags & DMAMAP_COHERENT) == 0) { if (map->sync_count > 0) sl_end = sl->paddr + sl->datacount; @@ -233,6 +247,7 @@ sl->datacount = sgsize; } else sl->datacount += sgsize; +#endif } if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, segp)) @@ -260,10 +275,14 @@ void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { +#ifdef BUS_DMA_NON_COHERENT_IMPL struct sync_list *sl; + bus_addr_t sl_pend; + vm_offset_t sl_vend; +#endif bus_size_t sgsize; - bus_addr_t curaddr, sl_pend; - vm_offset_t kvaddr, vaddr, sl_vend; + bus_addr_t curaddr; + vm_offset_t kvaddr, vaddr; int error; KASSERT((map->flags & DMAMAP_FROM_DMAMEM) != 0 || @@ -292,10 +311,12 @@ * continuous segments first and then pass these segment into * load loop. */ +#ifdef BUS_DMA_NON_COHERENT_IMPL sl = map->slist + map->sync_count - 1; - vaddr = (vm_offset_t)buf; sl_pend = 0; sl_vend = 0; +#endif + vaddr = (vm_offset_t)buf; while (buflen > 0) { /* @@ -325,6 +346,7 @@ "than PAGE_SIZE: %lu", dmat->common.alignment)); curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, sgsize); +#ifdef BUS_DMA_NON_COHERENT_IMPL } else if ((map->flags & DMAMAP_COHERENT) == 0) { if (map->sync_count > 0) { sl_pend = sl->paddr + sl->datacount; @@ -351,6 +373,7 @@ sl->datacount = sgsize; } else sl->datacount += sgsize; +#endif } if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, segp)) @@ -374,9 +397,16 @@ bus_dmasync_op_t op) { struct bounce_page *bpage; +#ifdef BUS_DMA_NON_COHERENT_IMPL struct sync_list *sl, *end; +#endif vm_offset_t datavaddr, tempvaddr; +#ifndef BUS_DMA_NON_COHERENT_IMPL + /* The map should always have this flag set in the coherent busdma */ + MPASS((map->flags & DMAMAP_COHERENT) != 0); +#endif + if (op == BUS_DMASYNC_POSTWRITE) return; @@ -406,26 +436,32 @@ (void *)bpage->vaddr, bpage->datacount); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); +#ifdef BUS_DMA_NON_COHERENT_IMPL if ((map->flags & DMAMAP_COHERENT) == 0) cpu_dcache_wb_range((void *)bpage->vaddr, bpage->datacount); +#endif bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } else if ((op & BUS_DMASYNC_PREREAD) != 0) { +#ifdef BUS_DMA_NON_COHERENT_IMPL while (bpage != NULL) { if ((map->flags & DMAMAP_COHERENT) == 0) cpu_dcache_wbinv_range((void *)bpage->vaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } +#endif } if ((op & BUS_DMASYNC_POSTREAD) != 0) { while (bpage != NULL) { +#ifdef BUS_DMA_NON_COHERENT_IMPL if ((map->flags & DMAMAP_COHERENT) == 0) cpu_dcache_inv_range((void *)bpage->vaddr, bpage->datacount); +#endif tempvaddr = 0; datavaddr = bpage->datavaddr; if (datavaddr == 0) { @@ -445,6 +481,7 @@ } } +#ifdef BUS_DMA_NON_COHERENT_IMPL /* * Cache maintenance for normal (non-COHERENT non-bounce) buffers. */ @@ -457,6 +494,7 @@ for ( ; sl != end; ++sl) dma_dcache_sync(sl, op); } +#endif if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { /* diff --git a/sys/arm64/arm64/busdma_bounce.c b/sys/arm64/arm64/busdma_bounce.c --- a/sys/arm64/arm64/busdma_bounce.c +++ b/sys/arm64/arm64/busdma_bounce.c @@ -218,7 +218,7 @@ if (error != 0) return (error); - newtag->common.impl = &bus_dma_bounce_impl; + MPASS(newtag->common.impl != NULL); newtag->map_count = 0; newtag->segments = NULL; @@ -687,6 +687,32 @@ } #endif +/* Coherent bus support */ +#include "busdma_bounce.h" + +struct bus_dma_impl bus_dma_bounce_coherent_impl = { + .tag_create = bounce_bus_dma_tag_create, + .tag_destroy = bounce_bus_dma_tag_destroy, + .tag_set_domain = bounce_bus_dma_tag_set_domain, + .id_mapped = bounce_bus_dma_id_mapped_coherent, + .map_create = bounce_bus_dmamap_create, + .map_destroy = bounce_bus_dmamap_destroy, + .mem_alloc = bounce_bus_dmamem_alloc, + .mem_free = bounce_bus_dmamem_free, + .load_phys = bounce_bus_dmamap_load_phys_coherent, + .load_buffer = bounce_bus_dmamap_load_buffer_coherent, + .load_ma = bus_dmamap_load_ma_triv, + .map_waitok = bounce_bus_dmamap_waitok, + .map_complete = bounce_bus_dmamap_complete, + .map_unload = bounce_bus_dmamap_unload, + .map_sync = bounce_bus_dmamap_sync_coherent, +#ifdef KMSAN + .load_kmsan = bounce_bus_dmamap_load_kmsan, +#endif +}; + +/* busdma support for non-coherent devices */ +#define BUS_DMA_NON_COHERENT_IMPL #include "busdma_bounce.h" struct bus_dma_impl bus_dma_bounce_impl = { diff --git a/sys/arm64/arm64/busdma_machdep.c b/sys/arm64/arm64/busdma_machdep.c --- a/sys/arm64/arm64/busdma_machdep.c +++ b/sys/arm64/arm64/busdma_machdep.c @@ -76,7 +76,6 @@ } common = newtag; - common->impl = &bus_dma_bounce_impl; common->alignment = alignment; common->boundary = boundary; common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); @@ -89,6 +88,11 @@ if (parent != NULL) common->flags |= parent->flags & BUS_DMA_COHERENT; + if ((common->flags & BUS_DMA_COHERENT) == 0) + common->impl = &bus_dma_bounce_impl; + else + common->impl = &bus_dma_bounce_coherent_impl; + if (lockfunc != NULL) { common->lockfunc = lockfunc; common->lockfuncarg = lockfuncarg; @@ -136,9 +140,12 @@ return (EINVAL); if (parent == NULL) { - error = bus_dma_bounce_impl.tag_create(parent, alignment, - boundary, lowaddr, highaddr, maxsize, nsegments, maxsegsz, - flags, lockfunc, lockfuncarg, dmat); + MPASS(bus_dma_bounce_impl.tag_create == + bus_dma_bounce_coherent_impl.tag_create); + + error = bus_dma_bounce_impl.tag_create(parent, + alignment, boundary, lowaddr, highaddr, maxsize, nsegments, + maxsegsz, flags, lockfunc, lockfuncarg, dmat); } else { tc = (struct bus_dma_tag_common *)parent; error = tc->impl->tag_create(parent, alignment, diff --git a/sys/arm64/include/bus_dma_impl.h b/sys/arm64/include/bus_dma_impl.h --- a/sys/arm64/include/bus_dma_impl.h +++ b/sys/arm64/include/bus_dma_impl.h @@ -87,5 +87,6 @@ void *lockfuncarg, size_t sz, void **dmat); extern struct bus_dma_impl bus_dma_bounce_impl; +extern struct bus_dma_impl bus_dma_bounce_coherent_impl; #endif