Differential D26494 Diff 77473 head/sys/arm64/arm64/busdma_bounce.c

Changeset View

Standalone View

head/sys/arm64/arm64/busdma_bounce.c

Show First 20 Lines • Show All 129 Lines • ▼ Show 20 Lines	struct bus_dmamap {
int pagesneeded;		int pagesneeded;
int pagesreserved;		int pagesreserved;
bus_dma_tag_t dmat;		bus_dma_tag_t dmat;
struct memdesc mem;		struct memdesc mem;
bus_dmamap_callback_t *callback;		bus_dmamap_callback_t *callback;
void *callback_arg;		void *callback_arg;
STAILQ_ENTRY(bus_dmamap) links;		STAILQ_ENTRY(bus_dmamap) links;
u_int flags;		u_int flags;
#define DMAMAP_COULD_BOUNCE (1 << 0)		#define DMAMAP_COHERENT (1 << 0)
#define DMAMAP_FROM_DMAMEM (1 << 1)		#define DMAMAP_FROM_DMAMEM (1 << 1)
int sync_count;		int sync_count;
struct sync_list slist[];		struct sync_list slist[];
};		};

static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;		static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;		static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;

▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 Lines	if (dmat->bounce_flags & BF_COULD_BOUNCE) {
if (dmat->bounce_zone == NULL) {		if (dmat->bounce_zone == NULL) {
if ((error = alloc_bounce_zone(dmat)) != 0) {		if ((error = alloc_bounce_zone(dmat)) != 0) {
free(*mapp, M_DEVBUF);		free(*mapp, M_DEVBUF);
return (error);		return (error);
}		}
}		}
bz = dmat->bounce_zone;		bz = dmat->bounce_zone;

(*mapp)->flags = DMAMAP_COULD_BOUNCE;

/*		/*
* Attempt to add pages to our pool on a per-instance		* Attempt to add pages to our pool on a per-instance
* basis up to a sane limit.		* basis up to a sane limit.
*/		*/
if (dmat->common.alignment > 1)		if (dmat->common.alignment > 1)
maxpages = MAX_BPAGES;		maxpages = MAX_BPAGES;
else		else
maxpages = MIN(MAX_BPAGES, Maxmem -		maxpages = MIN(MAX_BPAGES, Maxmem -
Show All 11 Lines	if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 \|\|
dmat->bounce_flags \|=		dmat->bounce_flags \|=
BF_MIN_ALLOC_COMP;		BF_MIN_ALLOC_COMP;
}		}
} else		} else
error = 0;		error = 0;
}		}
bz->map_count++;		bz->map_count++;
}		}
if (error == 0)		if (error == 0) {
dmat->map_count++;		dmat->map_count++;
else		if ((dmat->bounce_flags & BF_COHERENT) != 0)
		(*mapp)->flags \|= DMAMAP_COHERENT;
		} else {
free(*mapp, M_DEVBUF);		free(*mapp, M_DEVBUF);
		}
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->common.flags, error);		__func__, dmat, dmat->common.flags, error);
return (error);		return (error);
}		}

/*		/*
* Destroy a handle for mapping from kva/uva/physical		* Destroy a handle for mapping from kva/uva/physical
* address space into bus device space.		* address space into bus device space.
*/		*/
static int		static int
bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)		bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
{		{

/* Check we are destroying the correct map type */		/* Check we are destroying the correct map type */
if ((map->flags & DMAMAP_FROM_DMAMEM) != 0)		if ((map->flags & DMAMAP_FROM_DMAMEM) != 0)
panic("bounce_bus_dmamap_destroy: Invalid map freed\n");		panic("bounce_bus_dmamap_destroy: Invalid map freed\n");

if (STAILQ_FIRST(&map->bpages) != NULL \|\| map->sync_count != 0) {		if (STAILQ_FIRST(&map->bpages) != NULL \|\| map->sync_count != 0) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY);		CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY);
return (EBUSY);		return (EBUSY);
}		}
if (dmat->bounce_zone) {		if (dmat->bounce_zone)
KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0,
("%s: Bounce zone when cannot bounce", __func__));
dmat->bounce_zone->map_count--;		dmat->bounce_zone->map_count--;
}
free(map, M_DEVBUF);		free(map, M_DEVBUF);
dmat->map_count--;		dmat->map_count--;
CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);		CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
return (0);		return (0);
}		}

/*		/*
* Allocate a piece of memory that can be efficiently mapped into		* Allocate a piece of memory that can be efficiently mapped into
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines	bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
* this allocation should never bounce;		* this allocation should never bounce;
*/		*/
*mapp = alloc_dmamap(dmat, mflags);		*mapp = alloc_dmamap(dmat, mflags);
if (*mapp == NULL) {		if (*mapp == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->common.flags, ENOMEM);		__func__, dmat, dmat->common.flags, ENOMEM);
return (ENOMEM);		return (ENOMEM);
}		}
(*mapp)->flags = DMAMAP_FROM_DMAMEM;

/*		/*
		* Mark the map as coherent if we used uncacheable memory or the
		* tag was already marked as coherent.
		*/
		if (attr == VM_MEMATTR_UNCACHEABLE \|\|
		(dmat->bounce_flags & BF_COHERENT) != 0)
		(*mapp)->flags \|= DMAMAP_COHERENT;

		(*mapp)->flags \|= DMAMAP_FROM_DMAMEM;

		/*
* Allocate the buffer from the malloc(9) allocator if...		* Allocate the buffer from the malloc(9) allocator if...
* - It's small enough to fit into a single power of two sized bucket.		* - It's small enough to fit into a single power of two sized bucket.
* - The alignment is less than or equal to the maximum size		* - The alignment is less than or equal to the maximum size
* - The low address requirement is fulfilled.		* - The low address requirement is fulfilled.
* else allocate non-contiguous pages if...		* else allocate non-contiguous pages if...
* - The page count that could get allocated doesn't exceed		* - The page count that could get allocated doesn't exceed
* nsegments also when the maximum segment size is less		* nsegments also when the maximum segment size is less
* than PAGE_SIZE.		* than PAGE_SIZE.
▲ Show 20 Lines • Show All 251 Lines • ▼ Show 20 Lines	bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,

while (buflen > 0) {		while (buflen > 0) {
curaddr = buf;		curaddr = buf;
sgsize = MIN(buflen, dmat->common.maxsegsz);		sgsize = MIN(buflen, dmat->common.maxsegsz);
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {		if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));		sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
curaddr = add_bounce_page(dmat, map, 0, curaddr,		curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize);		sgsize);
} else if ((dmat->bounce_flags & BF_COHERENT) == 0) {		} else if ((map->flags & DMAMAP_COHERENT) == 0) {
if (map->sync_count > 0)		if (map->sync_count > 0)
sl_end = sl->paddr + sl->datacount;		sl_end = sl->paddr + sl->datacount;

if (map->sync_count == 0 \|\| curaddr != sl_end) {		if (map->sync_count == 0 \|\| curaddr != sl_end) {
if (++map->sync_count > dmat->common.nsegments)		if (++map->sync_count > dmat->common.nsegments)
break;		break;
sl++;		sl++;
sl->vaddr = 0;		sl->vaddr = 0;
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines	while (buflen > 0) {
*/		*/
max_sgsize = MIN(buflen, dmat->common.maxsegsz);		max_sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);		sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {		if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
sgsize = roundup2(sgsize, dmat->common.alignment);		sgsize = roundup2(sgsize, dmat->common.alignment);
sgsize = MIN(sgsize, max_sgsize);		sgsize = MIN(sgsize, max_sgsize);
curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,		curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize);		sgsize);
} else if ((dmat->bounce_flags & BF_COHERENT) == 0) {		} else if ((map->flags & DMAMAP_COHERENT) == 0) {
sgsize = MIN(sgsize, max_sgsize);		sgsize = MIN(sgsize, max_sgsize);
if (map->sync_count > 0) {		if (map->sync_count > 0) {
sl_pend = sl->paddr + sl->datacount;		sl_pend = sl->paddr + sl->datacount;
sl_vend = sl->vaddr + sl->datacount;		sl_vend = sl->vaddr + sl->datacount;
}		}

if (map->sync_count == 0 \|\|		if (map->sync_count == 0 \|\|
(kvaddr != 0 && kvaddr != sl_vend) \|\|		(kvaddr != 0 && kvaddr != sl_vend) \|\|
Show All 33 Lines	cleanup:
return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */		return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
}		}

static void		static void
bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,		bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
struct memdesc mem, bus_dmamap_callback_t callback, void *callback_arg)		struct memdesc mem, bus_dmamap_callback_t callback, void *callback_arg)
{		{

if ((map->flags & DMAMAP_COULD_BOUNCE) == 0)
return;
map->mem = *mem;		map->mem = *mem;
map->dmat = dmat;		map->dmat = dmat;
map->callback = callback;		map->callback = callback;
map->callback_arg = callback_arg;		map->callback_arg = callback_arg;
}		}

static bus_dma_segment_t *		static bus_dma_segment_t *
bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,		bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines	if ((op & BUS_DMASYNC_PREWRITE) != 0) {
bpage->datapage);		bpage->datapage);
datavaddr = tempvaddr \| bpage->dataoffs;		datavaddr = tempvaddr \| bpage->dataoffs;
}		}

bcopy((void *)datavaddr,		bcopy((void *)datavaddr,
(void *)bpage->vaddr, bpage->datacount);		(void *)bpage->vaddr, bpage->datacount);
if (tempvaddr != 0)		if (tempvaddr != 0)
pmap_quick_remove_page(tempvaddr);		pmap_quick_remove_page(tempvaddr);
if ((dmat->bounce_flags & BF_COHERENT) == 0)		if ((map->flags & DMAMAP_COHERENT) == 0)
cpu_dcache_wb_range(bpage->vaddr,		cpu_dcache_wb_range(bpage->vaddr,
bpage->datacount);		bpage->datacount);
bpage = STAILQ_NEXT(bpage, links);		bpage = STAILQ_NEXT(bpage, links);
}		}
dmat->bounce_zone->total_bounced++;		dmat->bounce_zone->total_bounced++;
} else if ((op & BUS_DMASYNC_PREREAD) != 0) {		} else if ((op & BUS_DMASYNC_PREREAD) != 0) {
while (bpage != NULL) {		while (bpage != NULL) {
if ((dmat->bounce_flags & BF_COHERENT) == 0)		if ((map->flags & DMAMAP_COHERENT) == 0)
cpu_dcache_wbinv_range(bpage->vaddr,		cpu_dcache_wbinv_range(bpage->vaddr,
bpage->datacount);		bpage->datacount);
bpage = STAILQ_NEXT(bpage, links);		bpage = STAILQ_NEXT(bpage, links);
}		}
}		}

if ((op & BUS_DMASYNC_POSTREAD) != 0) {		if ((op & BUS_DMASYNC_POSTREAD) != 0) {
while (bpage != NULL) {		while (bpage != NULL) {
if ((dmat->bounce_flags & BF_COHERENT) == 0)		if ((map->flags & DMAMAP_COHERENT) == 0)
cpu_dcache_inv_range(bpage->vaddr,		cpu_dcache_inv_range(bpage->vaddr,
bpage->datacount);		bpage->datacount);
tempvaddr = 0;		tempvaddr = 0;
datavaddr = bpage->datavaddr;		datavaddr = bpage->datavaddr;
if (datavaddr == 0) {		if (datavaddr == 0) {
tempvaddr = pmap_quick_enter_page(		tempvaddr = pmap_quick_enter_page(
bpage->datapage);		bpage->datapage);
datavaddr = tempvaddr \| bpage->dataoffs;		datavaddr = tempvaddr \| bpage->dataoffs;
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines
static bus_addr_t		static bus_addr_t
add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,		add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
bus_addr_t addr, bus_size_t size)		bus_addr_t addr, bus_size_t size)
{		{
struct bounce_zone *bz;		struct bounce_zone *bz;
struct bounce_page *bpage;		struct bounce_page *bpage;

KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));		KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0,
("add_bounce_page: bad map %p", map));

bz = dmat->bounce_zone;		bz = dmat->bounce_zone;
if (map->pagesneeded == 0)		if (map->pagesneeded == 0)
panic("add_bounce_page: map doesn't need any pages");		panic("add_bounce_page: map doesn't need any pages");
map->pagesneeded--;		map->pagesneeded--;

if (map->pagesreserved == 0)		if (map->pagesreserved == 0)
panic("add_bounce_page: map doesn't need any pages");		panic("add_bounce_page: map doesn't need any pages");
▲ Show 20 Lines • Show All 98 Lines • Show Last 20 Lines