Page MenuHomeFreeBSD
Differential D11201 Diff 29626 sys/arm/arm/busdma_machdep-v6.c

Changeset View

Standalone View

View Options

sys/arm/arm/busdma_machdep-v6.c

Show First 20 Lines • Show All 485 Lines • ▼ Show 20 Linesbus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
} }
/* Take into account any restrictions imposed by our parent tag */ /* Take into account any restrictions imposed by our parent tag */
if (parent != NULL) { if (parent != NULL) {
newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
newtag->alignment = MAX(parent->alignment, newtag->alignment); newtag->alignment = MAX(parent->alignment, newtag->alignment);
newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE;
newtag->flags |= parent->flags & BUS_DMA_COHERENT;
if (newtag->boundary == 0) if (newtag->boundary == 0)
newtag->boundary = parent->boundary; newtag->boundary = parent->boundary;
else if (parent->boundary != 0) else if (parent->boundary != 0)
newtag->boundary = MIN(parent->boundary, newtag->boundary = MIN(parent->boundary,
newtag->boundary); newtag->boundary);
if (newtag->filter == NULL) { if (newtag->filter == NULL) {
/* /*
* Short circuit to looking at our parent directly * Short circuit to looking at our parent directly
▲ Show 20 Lines • Show All 249 Lines • ▼ Show 20 Linesbus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags,
if (map == NULL) { if (map == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->flags, ENOMEM); __func__, dmat, dmat->flags, ENOMEM);
return (ENOMEM); return (ENOMEM);
} }
map->flags = DMAMAP_DMAMEM_ALLOC; map->flags = DMAMAP_DMAMEM_ALLOC;
/* Choose a busdma buffer allocator based on memory type flags. */ /* Choose a busdma buffer allocator based on memory type flags. */
if (flags & BUS_DMA_COHERENT) { if (flags & BUS_DMA_COHERENT)
map->flags |= DMAMAP_COHERENT;
if ((flags & BUS_DMA_COHERENT) &&
((dmat->flags & BUS_DMA_COHERENT) == 0)) {
ianUnsubmitted
Not Done
Inline Actions

This confused me for a while. Once I figured it out, I realized the problem is mostly that the comment is now a bit misleading. I think I'd make 2 comments now...

/* For coherent memory, set the map flag that disables sync ops. */       
if (flags & BUS_DMA_COHERENT)
    map->flags |= DMAMAP_COHERENT;
	​
/* 
 * Choose a busdma buffer allocator based on memory type flags.  If the tag's
 * COHERENT flag is set, that means normal memory is already coherent, use
 * the normal allocator.
 */
​if ((flags & BUS_DMA_COHERENT) &&
​    ((dmat->flags & BUS_DMA_COHERENT) == 0)) {
ian: This confused me for a while. Once I figured it out, I realized the problem is mostly that the…
memattr = VM_MEMATTR_UNCACHEABLE; memattr = VM_MEMATTR_UNCACHEABLE;
ba = coherent_allocator; ba = coherent_allocator;
map->flags |= DMAMAP_COHERENT;
} else { } else {
memattr = VM_MEMATTR_DEFAULT; memattr = VM_MEMATTR_DEFAULT;
ba = standard_allocator; ba = standard_allocator;
} }
/* /*
* Try to find a bufzone in the allocator that holds a cache of buffers * Try to find a bufzone in the allocator that holds a cache of buffers
* of the right size for this request. If the buffer is too big to be * of the right size for this request. If the buffer is too big to be
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
* its associated map. * its associated map.
*/ */
void void
bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
{ {
struct busdma_bufzone *bufzone; struct busdma_bufzone *bufzone;
busdma_bufalloc_t ba; busdma_bufalloc_t ba;
if (map->flags & DMAMAP_COHERENT) if ((map->flags & DMAMAP_COHERENT) &&
((dmat->flags & BUS_DMA_COHERENT) == 0))
ba = coherent_allocator; ba = coherent_allocator;
else else
ba = standard_allocator; ba = standard_allocator;
bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
if (bufzone != NULL && dmat->alignment <= bufzone->size && if (bufzone != NULL && dmat->alignment <= bufzone->size &&
!exclusion_bounce(dmat)) !exclusion_bounce(dmat))
▲ Show 20 Lines • Show All 184 Lines • ▼ Show 20 Lines_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
while (buflen > 0) { while (buflen > 0) {
curaddr = buf; curaddr = buf;
sgsize = MIN(buflen, dmat->maxsegsz); sgsize = MIN(buflen, dmat->maxsegsz);
if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
sgsize)) { sgsize)) {
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
curaddr = add_bounce_page(dmat, map, 0, curaddr, curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize); sgsize);
} else { } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) {
if (map->sync_count > 0) if (map->sync_count > 0)
sl_end = sl->paddr + sl->datacount; sl_end = sl->paddr + sl->datacount;
if (map->sync_count == 0 || curaddr != sl_end) { if (map->sync_count == 0 || curaddr != sl_end) {
if (++map->sync_count > dmat->nsegments) if (++map->sync_count > dmat->nsegments)
break; break;
sl++; sl++;
sl->vaddr = 0; sl->vaddr = 0;
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines if (sgsize > dmat->maxsegsz)
sgsize = dmat->maxsegsz; sgsize = dmat->maxsegsz;
if (buflen < sgsize) if (buflen < sgsize)
sgsize = buflen; sgsize = buflen;
if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
sgsize)) { sgsize)) {
curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize); sgsize);
} else { } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) {
if (map->sync_count > 0) { if (map->sync_count > 0) {
sl_pend = sl->paddr + sl->datacount; sl_pend = sl->paddr + sl->datacount;
sl_vend = sl->vaddr + sl->datacount; sl_vend = sl->vaddr + sl->datacount;
} }
if (map->sync_count == 0 || if (map->sync_count == 0 ||
(kvaddr != 0 && kvaddr != sl_vend) || (kvaddr != 0 && kvaddr != sl_vend) ||
(curaddr != sl_pend)) { (curaddr != sl_pend)) {
▲ Show 20 Lines • Show All 192 Lines • ▼ Show 20 Lines if (op & BUS_DMASYNC_PREWRITE) {
tempvaddr = pmap_quick_enter_page( tempvaddr = pmap_quick_enter_page(
bpage->datapage); bpage->datapage);
datavaddr = tempvaddr | bpage->dataoffs; datavaddr = tempvaddr | bpage->dataoffs;
} }
bcopy((void *)datavaddr, (void *)bpage->vaddr, bcopy((void *)datavaddr, (void *)bpage->vaddr,
bpage->datacount); bpage->datacount);
if (tempvaddr != 0) if (tempvaddr != 0)
pmap_quick_remove_page(tempvaddr); pmap_quick_remove_page(tempvaddr);
dcache_wb_poc(bpage->vaddr, bpage->busaddr, if ((dmat->flags & BUS_DMA_COHERENT) == 0)
bpage->datacount); dcache_wb_poc(bpage->vaddr,
bpage->busaddr, bpage->datacount);
bpage = STAILQ_NEXT(bpage, links); bpage = STAILQ_NEXT(bpage, links);
} }
dmat->bounce_zone->total_bounced++; dmat->bounce_zone->total_bounced++;
} }
/* /*
* Do an invalidate for PREREAD unless a writeback was already * Do an invalidate for PREREAD unless a writeback was already
* done above due to PREWRITE also being set. The reason for a * done above due to PREWRITE also being set. The reason for a
* PREREAD invalidate is to prevent dirty lines currently in the * PREREAD invalidate is to prevent dirty lines currently in the
* cache from being evicted during the DMA. If a writeback was * cache from being evicted during the DMA. If a writeback was
* done due to PREWRITE also being set there will be no dirty * done due to PREWRITE also being set there will be no dirty
* lines and the POSTREAD invalidate handles the rest. The * lines and the POSTREAD invalidate handles the rest. The
* invalidate is done from the innermost to outermost level. If * invalidate is done from the innermost to outermost level. If
* L2 were done first, a dirty cacheline could be automatically * L2 were done first, a dirty cacheline could be automatically
* evicted from L1 before we invalidated it, re-dirtying the L2. * evicted from L1 before we invalidated it, re-dirtying the L2.
*/ */
if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) { if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
bpage = STAILQ_FIRST(&map->bpages); bpage = STAILQ_FIRST(&map->bpages);
while (bpage != NULL) { while (bpage != NULL) {
dcache_inv_poc_dma(bpage->vaddr, bpage->busaddr, if ((dmat->flags & BUS_DMA_COHERENT) == 0)
bpage->datacount); dcache_inv_poc_dma(bpage->vaddr,
bpage->busaddr, bpage->datacount);
bpage = STAILQ_NEXT(bpage, links); bpage = STAILQ_NEXT(bpage, links);
} }
} }
/* /*
* Re-invalidate the caches on a POSTREAD, even though they were * Re-invalidate the caches on a POSTREAD, even though they were
* already invalidated at PREREAD time. Aggressive prefetching * already invalidated at PREREAD time. Aggressive prefetching
* due to accesses to other data near the dma buffer could have * due to accesses to other data near the dma buffer could have
* brought buffer data into the caches which is now stale. The * brought buffer data into the caches which is now stale. The
* caches are invalidated from the outermost to innermost; the * caches are invalidated from the outermost to innermost; the
* prefetches could be happening right now, and if L1 were * prefetches could be happening right now, and if L1 were
* invalidated first, stale L2 data could be prefetched into L1. * invalidated first, stale L2 data could be prefetched into L1.
*/ */
if (op & BUS_DMASYNC_POSTREAD) { if (op & BUS_DMASYNC_POSTREAD) {
while (bpage != NULL) { while (bpage != NULL) {
dcache_inv_poc(bpage->vaddr, bpage->busaddr, if ((dmat->flags & BUS_DMA_COHERENT) == 0)
bpage->datacount); dcache_inv_poc(bpage->vaddr,
bpage->busaddr, bpage->datacount);
tempvaddr = 0; tempvaddr = 0;
datavaddr = bpage->datavaddr; datavaddr = bpage->datavaddr;
if (datavaddr == 0) { if (datavaddr == 0) {
tempvaddr = pmap_quick_enter_page( tempvaddr = pmap_quick_enter_page(
bpage->datapage); bpage->datapage);
datavaddr = tempvaddr | bpage->dataoffs; datavaddr = tempvaddr | bpage->dataoffs;
} }
bcopy((void *)bpage->vaddr, (void *)datavaddr, bcopy((void *)bpage->vaddr, (void *)datavaddr,
Show All 12 Lines_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
* needed for a PREREAD without PREWRITE also set, because that would * needed for a PREREAD without PREWRITE also set, because that would
* imply that the cpu had written to the COHERENT buffer and expected * imply that the cpu had written to the COHERENT buffer and expected
* the dma device to see that change, and by definition a PREWRITE sync * the dma device to see that change, and by definition a PREWRITE sync
* is required to make that happen. * is required to make that happen.
*/ */
if (map->flags & DMAMAP_COHERENT) { if (map->flags & DMAMAP_COHERENT) {
if (op & BUS_DMASYNC_PREWRITE) { if (op & BUS_DMASYNC_PREWRITE) {
dsb(); dsb();
if ((dmat->flags & BUS_DMA_COHERENT) == 0)
cpu_l2cache_drain_writebuf(); cpu_l2cache_drain_writebuf();
} }
return; return;
} }
/* /*
* Cache maintenance for normal (non-COHERENT non-bounce) buffers. All * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All
* the comments about the sequences for flushing cache levels in the * the comments about the sequences for flushing cache levels in the
* bounce buffer code above apply here as well. In particular, the fact * bounce buffer code above apply here as well. In particular, the fact
▲ Show 20 Lines • Show All 265 Lines • Show Last 20 Lines