Differential D26496 Diff 77475 head/sys/arm64/arm64/busdma_bounce.c

Changeset View

Standalone View

head/sys/arm64/arm64/busdma_bounce.c

Show First 20 Lines • Show All 133 Lines • ▼ Show 20 Lines	struct bus_dmamap {
bus_dma_tag_t dmat;		bus_dma_tag_t dmat;
struct memdesc mem;		struct memdesc mem;
bus_dmamap_callback_t *callback;		bus_dmamap_callback_t *callback;
void *callback_arg;		void *callback_arg;
STAILQ_ENTRY(bus_dmamap) links;		STAILQ_ENTRY(bus_dmamap) links;
u_int flags;		u_int flags;
#define DMAMAP_COHERENT (1 << 0)		#define DMAMAP_COHERENT (1 << 0)
#define DMAMAP_FROM_DMAMEM (1 << 1)		#define DMAMAP_FROM_DMAMEM (1 << 1)
		#define DMAMAP_MBUF (1 << 2)
int sync_count;		int sync_count;
struct sync_list slist[];		struct sync_list slist[];
};		};

static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;		static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;		static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;

static void init_bounce_pages(void *dummy);		static void init_bounce_pages(void *dummy);
static int alloc_bounce_zone(bus_dma_tag_t dmat);		static int alloc_bounce_zone(bus_dma_tag_t dmat);
static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);		static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,		static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
int commit);		int commit);
static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,		static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
vm_offset_t vaddr, bus_addr_t addr, bus_size_t size);		vm_offset_t vaddr, bus_addr_t addr, bus_size_t size);
static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);		static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);		int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, vm_paddr_t buf,		static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map,
bus_size_t buflen, int *pagesneeded);		vm_paddr_t buf, bus_size_t buflen, int *pagesneeded);
static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,		static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
pmap_t pmap, void *buf, bus_size_t buflen, int flags);		pmap_t pmap, void *buf, bus_size_t buflen, int flags);
static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,		static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
vm_paddr_t buf, bus_size_t buflen, int flags);		vm_paddr_t buf, bus_size_t buflen, int flags);
static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,		static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
int flags);		int flags);

		/*
		* Return true if the DMA should bounce because the start or end does not fall
		* on a cacheline boundary (which would require a partial cacheline flush).
		* COHERENT memory doesn't trigger cacheline flushes. Memory allocated by
		* bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a
		* strict rule that such memory cannot be accessed by the CPU while DMA is in
		* progress (or by multiple DMA engines at once), so that it's always safe to do
		* full cacheline flushes even if that affects memory outside the range of a
		* given DMA operation that doesn't involve the full allocated buffer. If we're
		* mapping an mbuf, that follows the same rules as a buffer we allocated.
		*/
static bool		static bool
might_bounce(bus_dma_tag_t dmat)		cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr,
		bus_size_t size)
{		{

		#define DMAMAP_CACHELINE_FLAGS \
		(DMAMAP_FROM_DMAMEM \| DMAMAP_COHERENT \| DMAMAP_MBUF)
		if ((dmat->bounce_flags & BF_COHERENT) != 0)
		return (false);
		if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0)
		return (false);
		return (((paddr \| size) & (dcache_line_size - 1)) != 0);
		#undef DMAMAP_CACHELINE_FLAGS
		}

		/*
		* Return true if the given address does not fall on the alignment boundary.
		*/
		static bool
		alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr)
		{

		return ((addr & (dmat->common.alignment - 1)) != 0);
		}

		static bool
		might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr,
		bus_size_t size)
		{

if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0)		if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0)
return (true);		return (true);

		if (cacheline_bounce(dmat, map, paddr, size))
		return (true);

		if (alignment_bounce(dmat, paddr))
		return (true);

return (false);		return (false);
}		}

static bool		static bool
must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr)		must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr,
		bus_size_t size)
{		{

		if (cacheline_bounce(dmat, map, paddr, size))
		return (true);

		if (alignment_bounce(dmat, paddr))
		return (true);

if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 &&		if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 &&
bus_dma_run_filter(&dmat->common, paddr))		bus_dma_run_filter(&dmat->common, paddr))
return (true);		return (true);

return (false);		return (false);
}		}

/*		/*
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines	if (parent != NULL) {
/* Copy some flags from the parent */		/* Copy some flags from the parent */
newtag->bounce_flags \|= parent->bounce_flags & BF_COHERENT;		newtag->bounce_flags \|= parent->bounce_flags & BF_COHERENT;
}		}

if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) \|\|		if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) \|\|
newtag->common.alignment > 1)		newtag->common.alignment > 1)
newtag->bounce_flags \|= BF_COULD_BOUNCE;		newtag->bounce_flags \|= BF_COULD_BOUNCE;

if (((newtag->bounce_flags & BF_COULD_BOUNCE) != 0) &&		if ((flags & BUS_DMA_ALLOCNOW) != 0) {
(flags & BUS_DMA_ALLOCNOW) != 0) {
struct bounce_zone *bz;		struct bounce_zone *bz;

/* Must bounce */		/* Must bounce */
if ((error = alloc_bounce_zone(newtag)) != 0) {		if ((error = alloc_bounce_zone(newtag)) != 0) {
free(newtag, M_DEVBUF);		free(newtag, M_DEVBUF);
return (error);		return (error);
}		}
bz = newtag->bounce_zone;		bz = newtag->bounce_zone;
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines	out:
CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);		CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
return (error);		return (error);
}		}

static bool		static bool
bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)		bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
{		{

if (!might_bounce(dmat))		if (!might_bounce(dmat, NULL, buf, buflen))
return (true);		return (true);
return (!_bus_dmamap_pagesneeded(dmat, buf, buflen, NULL));		return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL));
}		}

static bus_dmamap_t		static bus_dmamap_t
alloc_dmamap(bus_dma_tag_t dmat, int flags)		alloc_dmamap(bus_dma_tag_t dmat, int flags)
{		{
u_long mapsize;		u_long mapsize;
bus_dmamap_t map;		bus_dmamap_t map;

Show All 39 Lines	if (*mapp == NULL) {
return (ENOMEM);		return (ENOMEM);
}		}

/*		/*
* Bouncing might be required if the driver asks for an active		* Bouncing might be required if the driver asks for an active
* exclusion region, a data alignment that is stricter than 1, and/or		* exclusion region, a data alignment that is stricter than 1, and/or
* an active address boundary.		* an active address boundary.
*/		*/
if (dmat->bounce_flags & BF_COULD_BOUNCE) {
/* Must bounce */
if (dmat->bounce_zone == NULL) {		if (dmat->bounce_zone == NULL) {
if ((error = alloc_bounce_zone(dmat)) != 0) {		if ((error = alloc_bounce_zone(dmat)) != 0) {
free(*mapp, M_DEVBUF);		free(*mapp, M_DEVBUF);
return (error);		return (error);
}		}
}		}
bz = dmat->bounce_zone;		bz = dmat->bounce_zone;

/*		/*
* Attempt to add pages to our pool on a per-instance		* Attempt to add pages to our pool on a per-instance
* basis up to a sane limit.		* basis up to a sane limit.
*/		*/
if (dmat->common.alignment > 1)		if (dmat->common.alignment > 1)
maxpages = MAX_BPAGES;		maxpages = MAX_BPAGES;
else		else
maxpages = MIN(MAX_BPAGES, Maxmem -		maxpages = MIN(MAX_BPAGES, Maxmem -
atop(dmat->common.lowaddr));		atop(dmat->common.lowaddr));
if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 \|\|		if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 \|\|
(bz->map_count > 0 && bz->total_bpages < maxpages)) {		(bz->map_count > 0 && bz->total_bpages < maxpages)) {
pages = MAX(atop(dmat->common.maxsize), 1);		pages = MAX(atop(dmat->common.maxsize), 1);
pages = MIN(maxpages - bz->total_bpages, pages);		pages = MIN(maxpages - bz->total_bpages, pages);
pages = MAX(pages, 1);		pages = MAX(pages, 1);
if (alloc_bounce_pages(dmat, pages) < pages)		if (alloc_bounce_pages(dmat, pages) < pages)
error = ENOMEM;		error = ENOMEM;
if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP)		if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) {
== 0) {
if (error == 0) {		if (error == 0) {
dmat->bounce_flags \|=		dmat->bounce_flags \|= BF_MIN_ALLOC_COMP;
BF_MIN_ALLOC_COMP;
}		}
} else		} else
error = 0;		error = 0;
}		}
bz->map_count++;		bz->map_count++;
}
if (error == 0) {		if (error == 0) {
dmat->map_count++;		dmat->map_count++;
if ((dmat->bounce_flags & BF_COHERENT) != 0)		if ((dmat->bounce_flags & BF_COHERENT) != 0)
(*mapp)->flags \|= DMAMAP_COHERENT;		(*mapp)->flags \|= DMAMAP_COHERENT;
} else {		} else {
free(*mapp, M_DEVBUF);		free(*mapp, M_DEVBUF);
}		}
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines	else
kmem_free((vm_offset_t)vaddr, dmat->alloc_size);		kmem_free((vm_offset_t)vaddr, dmat->alloc_size);
free(map, M_DEVBUF);		free(map, M_DEVBUF);
dmat->map_count--;		dmat->map_count--;
CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,		CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
dmat->bounce_flags);		dmat->bounce_flags);
}		}

static bool		static bool
_bus_dmamap_pagesneeded(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen,		_bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
int *pagesneeded)		bus_size_t buflen, int *pagesneeded)
{		{
bus_addr_t curaddr;		bus_addr_t curaddr;
bus_size_t sgsize;		bus_size_t sgsize;
int count;		int count;

/*		/*
* Count the number of bounce pages needed in order to		* Count the number of bounce pages needed in order to
* complete this transfer		* complete this transfer
*/		*/
count = 0;		count = 0;
curaddr = buf;		curaddr = buf;
while (buflen != 0) {		while (buflen != 0) {
sgsize = MIN(buflen, dmat->common.maxsegsz);		sgsize = MIN(buflen, dmat->common.maxsegsz);
if (must_bounce(dmat, curaddr)) {		if (must_bounce(dmat, map, curaddr, sgsize)) {
sgsize = MIN(sgsize,		sgsize = MIN(sgsize,
PAGE_SIZE - (curaddr & PAGE_MASK));		PAGE_SIZE - (curaddr & PAGE_MASK));
if (pagesneeded == NULL)		if (pagesneeded == NULL)
return (true);		return (true);
count++;		count++;
}		}
curaddr += sgsize;		curaddr += sgsize;
buflen -= sgsize;		buflen -= sgsize;
}		}

if (pagesneeded != NULL)		if (pagesneeded != NULL)
*pagesneeded = count;		*pagesneeded = count;
return (count != 0);		return (count != 0);
}		}

static void		static void
_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,		_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
bus_size_t buflen, int flags)		bus_size_t buflen, int flags)
{		{

if (map->pagesneeded == 0) {		if (map->pagesneeded == 0) {
_bus_dmamap_pagesneeded(dmat, buf, buflen, &map->pagesneeded);		_bus_dmamap_pagesneeded(dmat, map, buf, buflen,
		&map->pagesneeded);
CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
}		}
}		}

static void		static void
_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,		_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
void *buf, bus_size_t buflen, int flags)		void *buf, bus_size_t buflen, int flags)
{		{
Show All 17 Lines	if (map->pagesneeded == 0) {
vendaddr = (vm_offset_t)buf + buflen;		vendaddr = (vm_offset_t)buf + buflen;

while (vaddr < vendaddr) {		while (vaddr < vendaddr) {
sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);		sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
if (pmap == kernel_pmap)		if (pmap == kernel_pmap)
paddr = pmap_kextract(vaddr);		paddr = pmap_kextract(vaddr);
else		else
paddr = pmap_extract(pmap, vaddr);		paddr = pmap_extract(pmap, vaddr);
if (must_bounce(dmat, paddr)) {		if (must_bounce(dmat, map, paddr,
		min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr &
		PAGE_MASK)))) != 0) {
sg_len = roundup2(sg_len,		sg_len = roundup2(sg_len,
dmat->common.alignment);		dmat->common.alignment);
map->pagesneeded++;		map->pagesneeded++;
}		}
vaddr += sg_len;		vaddr += sg_len;
}		}
CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
}		}
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines	bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
struct sync_list *sl;		struct sync_list *sl;
bus_size_t sgsize;		bus_size_t sgsize;
bus_addr_t curaddr, sl_end;		bus_addr_t curaddr, sl_end;
int error;		int error;

if (segs == NULL)		if (segs == NULL)
segs = dmat->segments;		segs = dmat->segments;

if (might_bounce(dmat)) {		if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
if (map->pagesneeded != 0) {		if (map->pagesneeded != 0) {
error = _bus_dmamap_reserve_pages(dmat, map, flags);		error = _bus_dmamap_reserve_pages(dmat, map, flags);
if (error)		if (error)
return (error);		return (error);
}		}
}		}

sl = map->slist + map->sync_count - 1;		sl = map->slist + map->sync_count - 1;
sl_end = 0;		sl_end = 0;

while (buflen > 0) {		while (buflen > 0) {
curaddr = buf;		curaddr = buf;
sgsize = MIN(buflen, dmat->common.maxsegsz);		sgsize = MIN(buflen, dmat->common.maxsegsz);
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {		if (map->pagesneeded != 0 &&
		must_bounce(dmat, map, curaddr, sgsize)) {
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));		sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
curaddr = add_bounce_page(dmat, map, 0, curaddr,		curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize);		sgsize);
} else if ((map->flags & DMAMAP_COHERENT) == 0) {		} else if ((map->flags & DMAMAP_COHERENT) == 0) {
if (map->sync_count > 0)		if (map->sync_count > 0)
sl_end = sl->paddr + sl->datacount;		sl_end = sl->paddr + sl->datacount;

if (map->sync_count == 0 \|\| curaddr != sl_end) {		if (map->sync_count == 0 \|\| curaddr != sl_end) {
Show All 37 Lines	bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
bus_size_t sgsize, max_sgsize;		bus_size_t sgsize, max_sgsize;
bus_addr_t curaddr, sl_pend;		bus_addr_t curaddr, sl_pend;
vm_offset_t kvaddr, vaddr, sl_vend;		vm_offset_t kvaddr, vaddr, sl_vend;
int error;		int error;

if (segs == NULL)		if (segs == NULL)
segs = dmat->segments;		segs = dmat->segments;

if (might_bounce(dmat)) {		if (flags & BUS_DMA_LOAD_MBUF)
		map->flags \|= DMAMAP_MBUF;

		if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);		_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
if (map->pagesneeded != 0) {		if (map->pagesneeded != 0) {
error = _bus_dmamap_reserve_pages(dmat, map, flags);		error = _bus_dmamap_reserve_pages(dmat, map, flags);
if (error)		if (error)
return (error);		return (error);
}		}
}		}

Show All 14 Lines	if (pmap == kernel_pmap) {
kvaddr = 0;		kvaddr = 0;
}		}

/*		/*
* Compute the segment size, and adjust counts.		* Compute the segment size, and adjust counts.
*/		*/
max_sgsize = MIN(buflen, dmat->common.maxsegsz);		max_sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);		sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {		if (map->pagesneeded != 0 &&
		must_bounce(dmat, map, curaddr, sgsize)) {
sgsize = roundup2(sgsize, dmat->common.alignment);		sgsize = roundup2(sgsize, dmat->common.alignment);
sgsize = MIN(sgsize, max_sgsize);		sgsize = MIN(sgsize, max_sgsize);
curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,		curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize);		sgsize);
} else if ((map->flags & DMAMAP_COHERENT) == 0) {		} else if ((map->flags & DMAMAP_COHERENT) == 0) {
sgsize = MIN(sgsize, max_sgsize);		sgsize = MIN(sgsize, max_sgsize);
if (map->sync_count > 0) {		if (map->sync_count > 0) {
sl_pend = sl->paddr + sl->datacount;		sl_pend = sl->paddr + sl->datacount;
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
struct bounce_page *bpage;		struct bounce_page *bpage;

while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {		while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
STAILQ_REMOVE_HEAD(&map->bpages, links);		STAILQ_REMOVE_HEAD(&map->bpages, links);
free_bounce_page(dmat, bpage);		free_bounce_page(dmat, bpage);
}		}

map->sync_count = 0;		map->sync_count = 0;
		map->flags &= ~DMAMAP_MBUF;
}		}

static void		static void
dma_preread_safe(vm_offset_t va, vm_size_t size)		dma_preread_safe(vm_offset_t va, vm_size_t size)
{		{
/*		/*
* Write back any partial cachelines immediately before and		* Write back any partial cachelines immediately before and
* after the DMA region.		* after the DMA region.
▲ Show 20 Lines • Show All 434 Lines • Show Last 20 Lines