Differential D3986 Diff 9653 sys/mips/mips/busdma_machdep.c

Changeset View

Standalone View

sys/mips/mips/busdma_machdep.c

Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Lines	struct bus_dma_tag {
bus_dma_lock_t *lockfunc;		bus_dma_lock_t *lockfunc;
void *lockfuncarg;		void *lockfuncarg;
bus_dma_segment_t *segments;		bus_dma_segment_t *segments;
struct bounce_zone *bounce_zone;		struct bounce_zone *bounce_zone;
};		};

struct bounce_page {		struct bounce_page {
vm_offset_t vaddr; /* kva of bounce buffer */		vm_offset_t vaddr; /* kva of bounce buffer */
vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */
bus_addr_t busaddr; /* Physical address */		bus_addr_t busaddr; /* Physical address */
vm_offset_t datavaddr; /* kva of client data */		vm_offset_t datavaddr; /* kva of client data */
bus_addr_t dataaddr; /* client physical address */		vm_page_t datapage; /* physical page of client data */
		vm_offset_t dataoffs; /* page offset of client data */
bus_size_t datacount; /* client data count */		bus_size_t datacount; /* client data count */
STAILQ_ENTRY(bounce_page) links;		STAILQ_ENTRY(bounce_page) links;
};		};

struct sync_list {		struct sync_list {
vm_offset_t vaddr; /* kva of bounce buffer */		vm_offset_t vaddr; /* kva of client data */
bus_addr_t busaddr; /* Physical address */		vm_page_t pages; /* starting page of client data */
		vm_offset_t dataoffs; /* page offset of client data */
bus_size_t datacount; /* client data count */		bus_size_t datacount; /* client data count */
};		};

int busdma_swi_pending;		int busdma_swi_pending;

struct bounce_zone {		struct bounce_zone {
STAILQ_ENTRY(bounce_zone) links;		STAILQ_ENTRY(bounce_zone) links;
STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;		STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines
static int alloc_bounce_zone(bus_dma_tag_t dmat);		static int alloc_bounce_zone(bus_dma_tag_t dmat);
static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);		static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,		static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
int commit);		int commit);
static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,		static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
vm_offset_t vaddr, bus_addr_t addr,		vm_offset_t vaddr, bus_addr_t addr,
bus_size_t size);		bus_size_t size);
static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);		static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
		static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, int aligned);

/* Default tag, as most drivers provide no parent tag. */		/* Default tag, as most drivers provide no parent tag. */
bus_dma_tag_t mips_root_dma_tag;		bus_dma_tag_t mips_root_dma_tag;

static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */		static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */

static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */		static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */		static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
▲ Show 20 Lines • Show All 577 Lines • ▼ Show 20 Lines	if ((map->pagesneeded == 0)) {
/*		/*
* Count the number of bounce pages		* Count the number of bounce pages
* needed in order to complete this transfer		* needed in order to complete this transfer
*/		*/
curaddr = buf;		curaddr = buf;
while (buflen != 0) {		while (buflen != 0) {
sgsize = MIN(buflen, dmat->maxsegsz);		sgsize = MIN(buflen, dmat->maxsegsz);
if (run_filter(dmat, curaddr) != 0) {		if (run_filter(dmat, curaddr) != 0) {
sgsize = MIN(sgsize, PAGE_SIZE);		sgsize = MIN(sgsize,
		PAGE_SIZE - (curaddr & PAGE_MASK));
map->pagesneeded++;		map->pagesneeded++;
}		}
curaddr += sgsize;		curaddr += sgsize;
buflen -= sgsize;		buflen -= sgsize;
}		}
CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
}		}
}		}
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines
*/		*/
int		int
_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,		_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,		vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
int *segp)		int *segp)
{		{
bus_addr_t curaddr;		bus_addr_t curaddr;
bus_size_t sgsize;		bus_size_t sgsize;
		bus_addr_t sl_end = 0;
		struct sync_list *sl;
int error;		int error;

if (segs == NULL)		if (segs == NULL)
segs = dmat->segments;		segs = dmat->segments;

if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {		if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
if (map->pagesneeded != 0) {		if (map->pagesneeded != 0) {
error = _bus_dmamap_reserve_pages(dmat, map, flags);		error = _bus_dmamap_reserve_pages(dmat, map, flags);
if (error)		if (error)
return (error);		return (error);
}		}
}		}

		sl = map->slist + map->sync_count - 1;

while (buflen > 0) {		while (buflen > 0) {
curaddr = buf;		curaddr = buf;
sgsize = MIN(buflen, dmat->maxsegsz);		sgsize = MIN(buflen, dmat->maxsegsz);
if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
map->pagesneeded != 0 && run_filter(dmat, curaddr)) {		map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
sgsize = MIN(sgsize, PAGE_SIZE);		sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
curaddr = add_bounce_page(dmat, map, 0, curaddr,		curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize);		sgsize);
		} else {
		if (map->sync_count > 0)
		sl_end = VM_PAGE_TO_PHYS(sl->pages) +
		sl->dataoffs + sl->datacount;

		if (map->sync_count == 0 \|\| curaddr != sl_end) {
		if (++map->sync_count > dmat->nsegments)
		break;
		sl++;
		sl->vaddr = 0;
		sl->datacount = sgsize;
		sl->pages = PHYS_TO_VM_PAGE(curaddr);
		sl->dataoffs = curaddr & PAGE_MASK;
		} else
		sl->datacount += sgsize;
}		}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);		segp);
if (sgsize == 0)		if (sgsize == 0)
break;		break;
buf += sgsize;		buf += sgsize;
buflen -= sgsize;		buflen -= sgsize;
}		}
Show All 25 Lines
*/		*/
int		int
_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,		_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
bus_size_t buflen, struct pmap pmap, int flags, bus_dma_segment_t segs,		bus_size_t buflen, struct pmap pmap, int flags, bus_dma_segment_t segs,
int *segp)		int *segp)
{		{
bus_size_t sgsize;		bus_size_t sgsize;
bus_addr_t curaddr;		bus_addr_t curaddr;
		bus_addr_t sl_pend = 0;
		vm_offset_t kvaddr, vaddr, sl_vend = 0;
struct sync_list *sl;		struct sync_list *sl;
vm_offset_t vaddr = (vm_offset_t)buf;
int error = 0;		int error = 0;


if (segs == NULL)		if (segs == NULL)
segs = dmat->segments;		segs = dmat->segments;
if ((flags & BUS_DMA_LOAD_MBUF) != 0)		if ((flags & BUS_DMA_LOAD_MBUF) != 0)
map->flags \|= DMAMAP_CACHE_ALIGNED;		map->flags \|= DMAMAP_CACHE_ALIGNED;

if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {		if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);		_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
if (map->pagesneeded != 0) {		if (map->pagesneeded != 0) {
error = _bus_dmamap_reserve_pages(dmat, map, flags);		error = _bus_dmamap_reserve_pages(dmat, map, flags);
if (error)		if (error)
return (error);		return (error);
}		}
}		}
CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "		CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
"alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);		"alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);

		sl = map->slist + map->sync_count - 1;
		vaddr = (vm_offset_t)buf;

while (buflen > 0) {		while (buflen > 0) {
/*		/*
* Get the physical address for this segment.		* Get the physical address for this segment.
*
* XXX Don't support checking for coherent mappings
* XXX in user address space.
*/		*/
KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap"));		if (__predict_true(pmap == kernel_pmap)) {
curaddr = pmap_kextract(vaddr);		curaddr = pmap_kextract(vaddr);
		kvaddr = vaddr;
		} else {
		curaddr = pmap_extract(pmap, vaddr);
		kvaddr = 0;
		}

/*		/*
* Compute the segment size, and adjust counts.		* Compute the segment size, and adjust counts.
*/		*/
sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);		sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
if (sgsize > dmat->maxsegsz)		if (sgsize > dmat->maxsegsz)
sgsize = dmat->maxsegsz;		sgsize = dmat->maxsegsz;
if (buflen < sgsize)		if (buflen < sgsize)
sgsize = buflen;		sgsize = buflen;

if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
map->pagesneeded != 0 && run_filter(dmat, curaddr)) {		map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
curaddr = add_bounce_page(dmat, map, vaddr, curaddr,		curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize);		sgsize);
} else {		} else {
sl = &map->slist[map->sync_count - 1];		if (map->sync_count > 0) {
		sl_pend = VM_PAGE_TO_PHYS(sl->pages) +
		sl->dataoffs + sl->datacount;
		sl_vend = sl->vaddr + sl->datacount;
		}

if (map->sync_count == 0 \|\|		if (map->sync_count == 0 \|\|
vaddr != sl->vaddr + sl->datacount) {		(kvaddr != 0 && kvaddr != sl_vend) \|\|
		(kvaddr == 0 && curaddr != sl_pend)) {

if (++map->sync_count > dmat->nsegments)		if (++map->sync_count > dmat->nsegments)
goto cleanup;		goto cleanup;
sl++;		sl++;
sl->vaddr = vaddr;		sl->vaddr = kvaddr;
sl->datacount = sgsize;		sl->datacount = sgsize;
sl->busaddr = curaddr;		sl->pages = PHYS_TO_VM_PAGE(curaddr);
		sl->dataoffs = curaddr & PAGE_MASK;
} else		} else
sl->datacount += sgsize;		sl->datacount += sgsize;
}		}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);		segp);
if (sgsize == 0)		if (sgsize == 0)
break;		break;
vaddr += sgsize;		vaddr += sgsize;
▲ Show 20 Lines • Show All 153 Lines • ▼ Show 20 Lines	bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, int aligned)

case BUS_DMASYNC_PREWRITE:		case BUS_DMASYNC_PREWRITE:
mips_dcache_wb_range(buf, len);		mips_dcache_wb_range(buf, len);
break;		break;
}		}
}		}

static void		static void
		bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, int aligned)
		{
		bus_size_t synccount, pagecount;
		vm_offset_t syncoffs, tempvaddr;
		vm_page_t curpage;

		if (sl->vaddr != 0) {
		bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, aligned);
		return;
		}

		synccount = sl->datacount;
		syncoffs = sl->dataoffs;
		tempvaddr = 0;
		curpage = sl->pages;

		while (synccount > 0) {
		KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) +
		ptoa(curpage - sl->pages),
		("unexpected vm_page_t phys: 0x%lx != 0x%lx",
		(unsigned long)VM_PAGE_TO_PHYS(curpage),
		(unsigned long)(VM_PAGE_TO_PHYS(sl->pages) +
		ptoa(curpage - sl->pages))));
		pagecount = MIN(PAGE_SIZE - syncoffs, synccount);
		tempvaddr = pmap_quick_enter_page(curpage);
		bus_dmamap_sync_buf(tempvaddr \| syncoffs, pagecount, op, aligned);
		pmap_quick_remove_page(tempvaddr);
		syncoffs = 0;
		synccount -= pagecount;
		++curpage;
		}
		}

		static void
_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)		_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{		{
struct bounce_page *bpage;		struct bounce_page *bpage;
		vm_offset_t datavaddr, tempvaddr;

STAILQ_FOREACH(bpage, &map->bpages, links) {		STAILQ_FOREACH(bpage, &map->bpages, links) {
if (op & BUS_DMASYNC_PREWRITE) {		if (op & BUS_DMASYNC_PREWRITE) {
if (bpage->datavaddr != 0)		tempvaddr = 0;
bcopy((void *)bpage->datavaddr,		datavaddr = bpage->datavaddr;
(void *)(bpage->vaddr_nocache != 0 ?		if (datavaddr == 0) {
bpage->vaddr_nocache :		tempvaddr = pmap_quick_enter_page(
bpage->vaddr),		bpage->datapage);
bpage->datacount);		datavaddr = tempvaddr \| bpage->dataoffs;
else
physcopyout(bpage->dataaddr,
(void *)(bpage->vaddr_nocache != 0 ?
bpage->vaddr_nocache :
bpage->vaddr),
bpage->datacount);
if (bpage->vaddr_nocache == 0) {
mips_dcache_wb_range(bpage->vaddr,
bpage->datacount);
}		}
		bcopy((void )datavaddr, (void)bpage->vaddr,
		bpage->datacount);
		if (tempvaddr != 0)
		pmap_quick_remove_page(tempvaddr);
		mips_dcache_wb_range(bpage->vaddr, bpage->datacount);
dmat->bounce_zone->total_bounced++;		dmat->bounce_zone->total_bounced++;
}		}
if (op & BUS_DMASYNC_POSTREAD) {		if (op & BUS_DMASYNC_POSTREAD) {
if (bpage->vaddr_nocache == 0) {		mips_dcache_inv_range(bpage->vaddr, bpage->datacount);
mips_dcache_inv_range(bpage->vaddr,		tempvaddr = 0;
bpage->datacount);		datavaddr = bpage->datavaddr;
		if (datavaddr == 0) {
		tempvaddr = pmap_quick_enter_page(
		bpage->datapage);
		datavaddr = tempvaddr \| bpage->dataoffs;
}		}
if (bpage->datavaddr != 0)		bcopy((void )bpage->vaddr, (void )datavaddr,
bcopy((void *)(bpage->vaddr_nocache != 0 ?		bpage->datacount);
bpage->vaddr_nocache : bpage->vaddr),		if (tempvaddr != 0)
(void *)bpage->datavaddr, bpage->datacount);		pmap_quick_remove_page(tempvaddr);
else
physcopyin((void *)(bpage->vaddr_nocache != 0 ?
bpage->vaddr_nocache : bpage->vaddr),
bpage->dataaddr, bpage->datacount);
dmat->bounce_zone->total_bounced++;		dmat->bounce_zone->total_bounced++;
}		}
}		}
}		}

void		void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)		_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{		{
Show All 12 Lines	if (map->flags & DMAMAP_UNCACHEABLE)
return;		return;

aligned = (map->flags & DMAMAP_CACHE_ALIGNED) ? 1 : 0;		aligned = (map->flags & DMAMAP_CACHE_ALIGNED) ? 1 : 0;

CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);		CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
if (map->sync_count) {		if (map->sync_count) {
end = &map->slist[map->sync_count];		end = &map->slist[map->sync_count];
for (sl = &map->slist[0]; sl != end; sl++)		for (sl = &map->slist[0]; sl != end; sl++)
bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,		bus_dmamap_sync_sl(sl, op, aligned);
aligned);
}		}
}		}

static void		static void
init_bounce_pages(void *dummy __unused)		init_bounce_pages(void *dummy __unused)
{		{

total_bpages = 0;		total_bpages = 0;
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines	bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE,
bz->lowaddr,		bz->lowaddr,
PAGE_SIZE,		PAGE_SIZE,
0);		0);
if (bpage->vaddr == 0) {		if (bpage->vaddr == 0) {
free(bpage, M_BUSDMA);		free(bpage, M_BUSDMA);
break;		break;
}		}
bpage->busaddr = pmap_kextract(bpage->vaddr);		bpage->busaddr = pmap_kextract(bpage->vaddr);
bpage->vaddr_nocache =
(vm_offset_t)pmap_mapdev(bpage->busaddr, PAGE_SIZE);
mtx_lock(&bounce_lock);		mtx_lock(&bounce_lock);
STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);		STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
total_bpages++;		total_bpages++;
bz->total_bpages++;		bz->total_bpages++;
bz->free_bpages++;		bz->free_bpages++;
mtx_unlock(&bounce_lock);		mtx_unlock(&bounce_lock);
count++;		count++;
numpages--;		numpages--;
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
mtx_unlock(&bounce_lock);		mtx_unlock(&bounce_lock);

if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {		if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
/* Page offset needs to be preserved. */		/* Page offset needs to be preserved. */
bpage->vaddr \|= addr & PAGE_MASK;		bpage->vaddr \|= addr & PAGE_MASK;
bpage->busaddr \|= addr & PAGE_MASK;		bpage->busaddr \|= addr & PAGE_MASK;
}		}
bpage->datavaddr = vaddr;		bpage->datavaddr = vaddr;
bpage->dataaddr = addr;		bpage->datapage = PHYS_TO_VM_PAGE(addr);
		bpage->dataoffs = addr & PAGE_MASK;
bpage->datacount = size;		bpage->datacount = size;
STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);		STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
return (bpage->busaddr);		return (bpage->busaddr);
}		}

static void		static void
free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)		free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
{		{
▲ Show 20 Lines • Show All 52 Lines • Show Last 20 Lines