Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm/arm/busdma_machdep-v6.c
| Show First 20 Lines • Show All 485 Lines • ▼ Show 20 Lines | bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, | ||||
| } | } | ||||
| /* Take into account any restrictions imposed by our parent tag */ | /* Take into account any restrictions imposed by our parent tag */ | ||||
| if (parent != NULL) { | if (parent != NULL) { | ||||
| newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); | newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); | ||||
| newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); | newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); | ||||
| newtag->alignment = MAX(parent->alignment, newtag->alignment); | newtag->alignment = MAX(parent->alignment, newtag->alignment); | ||||
| newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; | newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; | ||||
| newtag->flags |= parent->flags & BUS_DMA_COHERENT; | |||||
| if (newtag->boundary == 0) | if (newtag->boundary == 0) | ||||
| newtag->boundary = parent->boundary; | newtag->boundary = parent->boundary; | ||||
| else if (parent->boundary != 0) | else if (parent->boundary != 0) | ||||
| newtag->boundary = MIN(parent->boundary, | newtag->boundary = MIN(parent->boundary, | ||||
| newtag->boundary); | newtag->boundary); | ||||
| if (newtag->filter == NULL) { | if (newtag->filter == NULL) { | ||||
| /* | /* | ||||
| * Short circuit to looking at our parent directly | * Short circuit to looking at our parent directly | ||||
| ▲ Show 20 Lines • Show All 248 Lines • ▼ Show 20 Lines | bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, | ||||
| *mapp = map = allocate_map(dmat, mflags); | *mapp = map = allocate_map(dmat, mflags); | ||||
| if (map == NULL) { | if (map == NULL) { | ||||
| CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", | CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", | ||||
| __func__, dmat, dmat->flags, ENOMEM); | __func__, dmat, dmat->flags, ENOMEM); | ||||
| return (ENOMEM); | return (ENOMEM); | ||||
| } | } | ||||
| map->flags = DMAMAP_DMAMEM_ALLOC; | map->flags = DMAMAP_DMAMEM_ALLOC; | ||||
| /* Choose a busdma buffer allocator based on memory type flags. */ | /* For coherent memory, set the map flag that disables sync ops. */ | ||||
| if (flags & BUS_DMA_COHERENT) { | if (flags & BUS_DMA_COHERENT) | ||||
| map->flags |= DMAMAP_COHERENT; | |||||
| /* | |||||
| * Choose a busdma buffer allocator based on memory type flags. | |||||
| * If the tag's COHERENT flag is set, that means normal memory | |||||
| * is already coherent, use the normal allocator. | |||||
| */ | |||||
| if ((flags & BUS_DMA_COHERENT) && | |||||
| ((dmat->flags & BUS_DMA_COHERENT) == 0)) { | |||||
| memattr = VM_MEMATTR_UNCACHEABLE; | memattr = VM_MEMATTR_UNCACHEABLE; | ||||
| ba = coherent_allocator; | ba = coherent_allocator; | ||||
| map->flags |= DMAMAP_COHERENT; | |||||
| } else { | } else { | ||||
| memattr = VM_MEMATTR_DEFAULT; | memattr = VM_MEMATTR_DEFAULT; | ||||
| ba = standard_allocator; | ba = standard_allocator; | ||||
| } | } | ||||
| /* | /* | ||||
| * Try to find a bufzone in the allocator that holds a cache of buffers | * Try to find a bufzone in the allocator that holds a cache of buffers | ||||
| * of the right size for this request. If the buffer is too big to be | * of the right size for this request. If the buffer is too big to be | ||||
| ▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
| * its associated map. | * its associated map. | ||||
| */ | */ | ||||
| void | void | ||||
| bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) | bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) | ||||
| { | { | ||||
| struct busdma_bufzone *bufzone; | struct busdma_bufzone *bufzone; | ||||
| busdma_bufalloc_t ba; | busdma_bufalloc_t ba; | ||||
| if (map->flags & DMAMAP_COHERENT) | if ((map->flags & DMAMAP_COHERENT) && | ||||
| ((dmat->flags & BUS_DMA_COHERENT) == 0)) | |||||
| ba = coherent_allocator; | ba = coherent_allocator; | ||||
| else | else | ||||
| ba = standard_allocator; | ba = standard_allocator; | ||||
| bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); | bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); | ||||
| if (bufzone != NULL && dmat->alignment <= bufzone->size && | if (bufzone != NULL && dmat->alignment <= bufzone->size && | ||||
| !exclusion_bounce(dmat)) | !exclusion_bounce(dmat)) | ||||
| ▲ Show 20 Lines • Show All 184 Lines • ▼ Show 20 Lines | _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, | ||||
| while (buflen > 0) { | while (buflen > 0) { | ||||
| curaddr = buf; | curaddr = buf; | ||||
| sgsize = MIN(buflen, dmat->maxsegsz); | sgsize = MIN(buflen, dmat->maxsegsz); | ||||
| if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, | if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, | ||||
| sgsize)) { | sgsize)) { | ||||
| sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); | sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); | ||||
| curaddr = add_bounce_page(dmat, map, 0, curaddr, | curaddr = add_bounce_page(dmat, map, 0, curaddr, | ||||
| sgsize); | sgsize); | ||||
| } else { | } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) { | ||||
| if (map->sync_count > 0) | if (map->sync_count > 0) | ||||
| sl_end = sl->paddr + sl->datacount; | sl_end = sl->paddr + sl->datacount; | ||||
| if (map->sync_count == 0 || curaddr != sl_end) { | if (map->sync_count == 0 || curaddr != sl_end) { | ||||
| if (++map->sync_count > dmat->nsegments) | if (++map->sync_count > dmat->nsegments) | ||||
| break; | break; | ||||
| sl++; | sl++; | ||||
| sl->vaddr = 0; | sl->vaddr = 0; | ||||
| ▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | if (sgsize > dmat->maxsegsz) | ||||
| sgsize = dmat->maxsegsz; | sgsize = dmat->maxsegsz; | ||||
| if (buflen < sgsize) | if (buflen < sgsize) | ||||
| sgsize = buflen; | sgsize = buflen; | ||||
| if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, | if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr, | ||||
| sgsize)) { | sgsize)) { | ||||
| curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, | curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, | ||||
| sgsize); | sgsize); | ||||
| } else { | } else if ((dmat->flags & BUS_DMA_COHERENT) == 0) { | ||||
| if (map->sync_count > 0) { | if (map->sync_count > 0) { | ||||
| sl_pend = sl->paddr + sl->datacount; | sl_pend = sl->paddr + sl->datacount; | ||||
| sl_vend = sl->vaddr + sl->datacount; | sl_vend = sl->vaddr + sl->datacount; | ||||
| } | } | ||||
| if (map->sync_count == 0 || | if (map->sync_count == 0 || | ||||
| (kvaddr != 0 && kvaddr != sl_vend) || | (kvaddr != 0 && kvaddr != sl_vend) || | ||||
| (curaddr != sl_pend)) { | (curaddr != sl_pend)) { | ||||
| ▲ Show 20 Lines • Show All 192 Lines • ▼ Show 20 Lines | if (op & BUS_DMASYNC_PREWRITE) { | ||||
| tempvaddr = pmap_quick_enter_page( | tempvaddr = pmap_quick_enter_page( | ||||
| bpage->datapage); | bpage->datapage); | ||||
| datavaddr = tempvaddr | bpage->dataoffs; | datavaddr = tempvaddr | bpage->dataoffs; | ||||
| } | } | ||||
| bcopy((void *)datavaddr, (void *)bpage->vaddr, | bcopy((void *)datavaddr, (void *)bpage->vaddr, | ||||
| bpage->datacount); | bpage->datacount); | ||||
| if (tempvaddr != 0) | if (tempvaddr != 0) | ||||
| pmap_quick_remove_page(tempvaddr); | pmap_quick_remove_page(tempvaddr); | ||||
| dcache_wb_poc(bpage->vaddr, bpage->busaddr, | if ((dmat->flags & BUS_DMA_COHERENT) == 0) | ||||
| bpage->datacount); | dcache_wb_poc(bpage->vaddr, | ||||
| bpage->busaddr, bpage->datacount); | |||||
| bpage = STAILQ_NEXT(bpage, links); | bpage = STAILQ_NEXT(bpage, links); | ||||
| } | } | ||||
| dmat->bounce_zone->total_bounced++; | dmat->bounce_zone->total_bounced++; | ||||
| } | } | ||||
| /* | /* | ||||
| * Do an invalidate for PREREAD unless a writeback was already | * Do an invalidate for PREREAD unless a writeback was already | ||||
| * done above due to PREWRITE also being set. The reason for a | * done above due to PREWRITE also being set. The reason for a | ||||
| * PREREAD invalidate is to prevent dirty lines currently in the | * PREREAD invalidate is to prevent dirty lines currently in the | ||||
| * cache from being evicted during the DMA. If a writeback was | * cache from being evicted during the DMA. If a writeback was | ||||
| * done due to PREWRITE also being set there will be no dirty | * done due to PREWRITE also being set there will be no dirty | ||||
| * lines and the POSTREAD invalidate handles the rest. The | * lines and the POSTREAD invalidate handles the rest. The | ||||
| * invalidate is done from the innermost to outermost level. If | * invalidate is done from the innermost to outermost level. If | ||||
| * L2 were done first, a dirty cacheline could be automatically | * L2 were done first, a dirty cacheline could be automatically | ||||
| * evicted from L1 before we invalidated it, re-dirtying the L2. | * evicted from L1 before we invalidated it, re-dirtying the L2. | ||||
| */ | */ | ||||
| if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) { | if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) { | ||||
| bpage = STAILQ_FIRST(&map->bpages); | bpage = STAILQ_FIRST(&map->bpages); | ||||
| while (bpage != NULL) { | while (bpage != NULL) { | ||||
| dcache_inv_poc_dma(bpage->vaddr, bpage->busaddr, | if ((dmat->flags & BUS_DMA_COHERENT) == 0) | ||||
| bpage->datacount); | dcache_inv_poc_dma(bpage->vaddr, | ||||
| bpage->busaddr, bpage->datacount); | |||||
| bpage = STAILQ_NEXT(bpage, links); | bpage = STAILQ_NEXT(bpage, links); | ||||
| } | } | ||||
| } | } | ||||
| /* | /* | ||||
| * Re-invalidate the caches on a POSTREAD, even though they were | * Re-invalidate the caches on a POSTREAD, even though they were | ||||
| * already invalidated at PREREAD time. Aggressive prefetching | * already invalidated at PREREAD time. Aggressive prefetching | ||||
| * due to accesses to other data near the dma buffer could have | * due to accesses to other data near the dma buffer could have | ||||
| * brought buffer data into the caches which is now stale. The | * brought buffer data into the caches which is now stale. The | ||||
| * caches are invalidated from the outermost to innermost; the | * caches are invalidated from the outermost to innermost; the | ||||
| * prefetches could be happening right now, and if L1 were | * prefetches could be happening right now, and if L1 were | ||||
| * invalidated first, stale L2 data could be prefetched into L1. | * invalidated first, stale L2 data could be prefetched into L1. | ||||
| */ | */ | ||||
| if (op & BUS_DMASYNC_POSTREAD) { | if (op & BUS_DMASYNC_POSTREAD) { | ||||
| while (bpage != NULL) { | while (bpage != NULL) { | ||||
| dcache_inv_poc(bpage->vaddr, bpage->busaddr, | if ((dmat->flags & BUS_DMA_COHERENT) == 0) | ||||
| bpage->datacount); | dcache_inv_poc(bpage->vaddr, | ||||
| bpage->busaddr, bpage->datacount); | |||||
| tempvaddr = 0; | tempvaddr = 0; | ||||
| datavaddr = bpage->datavaddr; | datavaddr = bpage->datavaddr; | ||||
| if (datavaddr == 0) { | if (datavaddr == 0) { | ||||
| tempvaddr = pmap_quick_enter_page( | tempvaddr = pmap_quick_enter_page( | ||||
| bpage->datapage); | bpage->datapage); | ||||
| datavaddr = tempvaddr | bpage->dataoffs; | datavaddr = tempvaddr | bpage->dataoffs; | ||||
| } | } | ||||
| bcopy((void *)bpage->vaddr, (void *)datavaddr, | bcopy((void *)bpage->vaddr, (void *)datavaddr, | ||||
| Show All 12 Lines | _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) | ||||
| * needed for a PREREAD without PREWRITE also set, because that would | * needed for a PREREAD without PREWRITE also set, because that would | ||||
| * imply that the cpu had written to the COHERENT buffer and expected | * imply that the cpu had written to the COHERENT buffer and expected | ||||
| * the dma device to see that change, and by definition a PREWRITE sync | * the dma device to see that change, and by definition a PREWRITE sync | ||||
| * is required to make that happen. | * is required to make that happen. | ||||
| */ | */ | ||||
| if (map->flags & DMAMAP_COHERENT) { | if (map->flags & DMAMAP_COHERENT) { | ||||
| if (op & BUS_DMASYNC_PREWRITE) { | if (op & BUS_DMASYNC_PREWRITE) { | ||||
| dsb(); | dsb(); | ||||
| if ((dmat->flags & BUS_DMA_COHERENT) == 0) | |||||
| cpu_l2cache_drain_writebuf(); | cpu_l2cache_drain_writebuf(); | ||||
| } | } | ||||
| return; | return; | ||||
| } | } | ||||
| /* | /* | ||||
| * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All | * Cache maintenance for normal (non-COHERENT non-bounce) buffers. All | ||||
| * the comments about the sequences for flushing cache levels in the | * the comments about the sequences for flushing cache levels in the | ||||
| * bounce buffer code above apply here as well. In particular, the fact | * bounce buffer code above apply here as well. In particular, the fact | ||||
| ▲ Show 20 Lines • Show All 265 Lines • Show Last 20 Lines | |||||