Index: sys/mips/mips/busdma_machdep.c
===================================================================
--- sys/mips/mips/busdma_machdep.c
+++ sys/mips/mips/busdma_machdep.c
@@ -90,17 +90,18 @@
 
 struct bounce_page {
 	vm_offset_t	vaddr;		/* kva of bounce buffer */
-	vm_offset_t	vaddr_nocache;	/* kva of bounce buffer uncached */
 	bus_addr_t	busaddr;	/* Physical address */
 	vm_offset_t	datavaddr;	/* kva of client data */
-	bus_addr_t	dataaddr;	/* client physical address */
+	vm_page_t	datapage;	/* physical page of client data */
+	vm_offset_t	dataoffs;	/* page offset of client data */
 	bus_size_t	datacount;	/* client data count */
 	STAILQ_ENTRY(bounce_page) links;
 };
 
 struct sync_list {
-	vm_offset_t	vaddr;		/* kva of bounce buffer */
-	bus_addr_t	busaddr;	/* Physical address */
+	vm_offset_t	vaddr;		/* kva of client data */
+	vm_page_t	pages;		/* starting page of client data */
+	vm_offset_t	dataoffs;	/* page offset of client data */
 	bus_size_t	datacount;	/* client data count */
 };
 
@@ -165,6 +166,7 @@
 				  vm_offset_t vaddr, bus_addr_t addr,
 				  bus_size_t size);
 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
+static void bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, int aligned);
 
 /* Default tag, as most drivers provide no parent tag. */
 bus_dma_tag_t mips_root_dma_tag;
@@ -758,7 +760,8 @@
 		while (buflen != 0) {
 			sgsize = MIN(buflen, dmat->maxsegsz);
 			if (run_filter(dmat, curaddr) != 0) {
-				sgsize = MIN(sgsize, PAGE_SIZE);
+				sgsize = MIN(sgsize,
+				    PAGE_SIZE - (curaddr & PAGE_MASK));
 				map->pagesneeded++;
 			}
 			curaddr += sgsize;
@@ -881,6 +884,8 @@
 {
 	bus_addr_t curaddr;
 	bus_size_t sgsize;
+	bus_addr_t sl_end = 0;
+	struct sync_list *sl;
 	int error;
 
 	if (segs == NULL)
@@ -895,14 +900,31 @@
 		}
 	}
 
+	sl = map->slist + map->sync_count - 1;
+
 	while (buflen > 0) {
 		curaddr = buf;
 		sgsize = MIN(buflen, dmat->maxsegsz);
 		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
-			sgsize = MIN(sgsize, PAGE_SIZE);
+			sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
 			curaddr = add_bounce_page(dmat, map, 0, curaddr,
 			    sgsize);
+		} else {
+			if (map->sync_count > 0)
+				sl_end = VM_PAGE_TO_PHYS(sl->pages) +
+				    sl->dataoffs + sl->datacount;
+
+			if (map->sync_count == 0 || curaddr != sl_end) {
+				if (++map->sync_count > dmat->nsegments)
+					break;
+				sl++;
+				sl->vaddr = 0;
+				sl->datacount = sgsize;
+				sl->pages = PHYS_TO_VM_PAGE(curaddr);
+				sl->dataoffs = curaddr & PAGE_MASK;
+			} else
+				sl->datacount += sgsize;
 		}
 		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 		    segp);
@@ -944,8 +966,9 @@
 {
 	bus_size_t sgsize;
 	bus_addr_t curaddr;
+	bus_addr_t sl_pend = 0;
+	vm_offset_t kvaddr, vaddr, sl_vend = 0;
 	struct sync_list *sl;
-	vm_offset_t vaddr = (vm_offset_t)buf;
 	int error = 0;
 
 
@@ -965,20 +988,25 @@
 	CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
 	    "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
 
+	sl = map->slist + map->sync_count - 1;
+	vaddr = (vm_offset_t)buf;
+
 	while (buflen > 0) {
 		/*
 		 * Get the physical address for this segment.
-		 *
-		 * XXX Don't support checking for coherent mappings
-		 * XXX in user address space.
 		 */
-		KASSERT(kernel_pmap == pmap, ("pmap is not kernel pmap"));
-		curaddr = pmap_kextract(vaddr);
+		if (__predict_true(pmap == kernel_pmap)) {
+			curaddr = pmap_kextract(vaddr);
+			kvaddr = vaddr;
+		} else {
+			curaddr = pmap_extract(pmap, vaddr);
+			kvaddr = 0;
+		}
 
 		/*
 		 * Compute the segment size, and adjust counts.
 		 */
-		sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
+		sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
 		if (sgsize > dmat->maxsegsz)
 			sgsize = dmat->maxsegsz;
 		if (buflen < sgsize)
@@ -986,18 +1014,26 @@
 
 		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
-			curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
+			curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
 			    sgsize);
 		} else {
-			sl = &map->slist[map->sync_count - 1];
+			if (map->sync_count > 0) {
+				sl_pend = VM_PAGE_TO_PHYS(sl->pages) +
+				    sl->dataoffs + sl->datacount;
+				sl_vend = sl->vaddr + sl->datacount;
+			}
+
 			if (map->sync_count == 0 ||
-			    vaddr != sl->vaddr + sl->datacount) {
+			    (kvaddr != 0 && kvaddr != sl_vend) ||
+			    (kvaddr == 0 && curaddr != sl_pend)) {
+
 				if (++map->sync_count > dmat->nsegments)
 					goto cleanup;
 				sl++;
-				sl->vaddr = vaddr;
+				sl->vaddr = kvaddr;
 				sl->datacount = sgsize;
-				sl->busaddr = curaddr;
+				sl->pages = PHYS_TO_VM_PAGE(curaddr);
+				sl->dataoffs = curaddr & PAGE_MASK;
 			} else
 				sl->datacount += sgsize;
 		}
@@ -1167,43 +1203,74 @@
 }
 
 static void
+bus_dmamap_sync_sl(struct sync_list *sl, bus_dmasync_op_t op, int aligned)
+{
+	bus_size_t synccount, pagecount;
+	vm_offset_t syncoffs, tempvaddr;
+	vm_page_t curpage;
+
+	if (sl->vaddr != 0) {
+		bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, aligned);
+		return;
+	}
+
+	synccount = sl->datacount;
+	syncoffs = sl->dataoffs; 
+	tempvaddr = 0;
+	curpage = sl->pages;
+
+	while (synccount > 0) {
+		KASSERT(VM_PAGE_TO_PHYS(curpage) == VM_PAGE_TO_PHYS(sl->pages) +
+		    ptoa(curpage - sl->pages),
+		    ("unexpected vm_page_t phys: 0x%lx != 0x%lx",
+		    (unsigned long)VM_PAGE_TO_PHYS(curpage),
+		    (unsigned long)(VM_PAGE_TO_PHYS(sl->pages) +
+		    ptoa(curpage - sl->pages))));
+		pagecount = MIN(PAGE_SIZE - syncoffs, synccount);
+		tempvaddr = pmap_quick_enter_page(curpage);
+		bus_dmamap_sync_buf(tempvaddr | syncoffs, pagecount, op, aligned);
+		pmap_quick_remove_page(tempvaddr);
+		syncoffs = 0;
+		synccount -= pagecount;
+		++curpage;
+	}
+}
+
+static void
 _bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 {
 	struct bounce_page *bpage;
+	vm_offset_t datavaddr, tempvaddr;
 
 	STAILQ_FOREACH(bpage, &map->bpages, links) {
 		if (op & BUS_DMASYNC_PREWRITE) {
-			if (bpage->datavaddr != 0)
-				bcopy((void *)bpage->datavaddr,
-				    (void *)(bpage->vaddr_nocache != 0 ? 
-					     bpage->vaddr_nocache :
-					     bpage->vaddr),
-				    bpage->datacount);
-			else
-				physcopyout(bpage->dataaddr,
-				    (void *)(bpage->vaddr_nocache != 0 ? 
-					     bpage->vaddr_nocache :
-					     bpage->vaddr),
-				    bpage->datacount);
-			if (bpage->vaddr_nocache == 0) {
-				mips_dcache_wb_range(bpage->vaddr,
-				    bpage->datacount);
+			tempvaddr = 0;
+			datavaddr = bpage->datavaddr;
+			if (datavaddr == 0) {
+				tempvaddr = pmap_quick_enter_page(
+				    bpage->datapage);
+				datavaddr = tempvaddr | bpage->dataoffs;
 			}
+			bcopy((void *)datavaddr, (void*)bpage->vaddr,
+			    bpage->datacount);
+			if (tempvaddr != 0)
+				pmap_quick_remove_page(tempvaddr);
+			mips_dcache_wb_range(bpage->vaddr, bpage->datacount);
 			dmat->bounce_zone->total_bounced++;
 		}
 		if (op & BUS_DMASYNC_POSTREAD) {
-			if (bpage->vaddr_nocache == 0) {
-				mips_dcache_inv_range(bpage->vaddr,
-				    bpage->datacount);
+			mips_dcache_inv_range(bpage->vaddr, bpage->datacount);
+			tempvaddr = 0;
+			datavaddr = bpage->datavaddr;
+			if (datavaddr == 0) {
+				tempvaddr = pmap_quick_enter_page(
+				    bpage->datapage);
+				datavaddr = tempvaddr | bpage->dataoffs;
 			}
-			if (bpage->datavaddr != 0)
-				bcopy((void *)(bpage->vaddr_nocache != 0 ? 
-				    bpage->vaddr_nocache : bpage->vaddr),
-				    (void *)bpage->datavaddr, bpage->datacount);
-			else
-				physcopyin((void *)(bpage->vaddr_nocache != 0 ? 
-				    bpage->vaddr_nocache : bpage->vaddr),
-				    bpage->dataaddr, bpage->datacount);
+			bcopy((void *)bpage->vaddr, (void *)datavaddr,
+			    bpage->datacount);
+			if (tempvaddr != 0)
+				pmap_quick_remove_page(tempvaddr);
 			dmat->bounce_zone->total_bounced++;
 		}
 	}
@@ -1232,8 +1299,7 @@
 	if (map->sync_count) {
 		end = &map->slist[map->sync_count];
 		for (sl = &map->slist[0]; sl != end; sl++)
-			bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, 
-			    aligned);
+			bus_dmamap_sync_sl(sl, op, aligned);
 	}
 }
 
@@ -1361,8 +1427,6 @@
 			break;
 		}
 		bpage->busaddr = pmap_kextract(bpage->vaddr);
-		bpage->vaddr_nocache = 
-		    (vm_offset_t)pmap_mapdev(bpage->busaddr, PAGE_SIZE);
 		mtx_lock(&bounce_lock);
 		STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 		total_bpages++;
@@ -1429,7 +1493,8 @@
 		bpage->busaddr |= addr & PAGE_MASK;
 	}
 	bpage->datavaddr = vaddr;
-	bpage->dataaddr = addr;
+	bpage->datapage = PHYS_TO_VM_PAGE(addr);
+	bpage->dataoffs = addr & PAGE_MASK;
 	bpage->datacount = size;
 	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 	return (bpage->busaddr);