Index: ioat.h =================================================================== --- ioat.h +++ ioat.h @@ -52,7 +52,6 @@ * may be prefetched before operation 1 completes. */ #define DMA_FENCE 0x4 -#define _DMA_GENERIC_FLAGS (DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE) /* * Emit a CRC32C as the result of a ioat_copy_crc() or ioat_crc(). @@ -71,9 +70,21 @@ * without one of STORE or TEST. */ #define DMA_CRC_INLINE 0x20 -#define _DMA_CRC_FLAGS (DMA_CRC_STORE | DMA_CRC_TEST | DMA_CRC_INLINE) /* + * src, dst and/or crc address arguments are already loaded into bus_dma(9) + * map, parented by tag returned by ioat_get_dma_tag() for the device. + */ +#define DMA_SRC_LOADED 0x100 +#define DMA_DST_LOADED 0x200 +#define DMA_CRC_LOADED 0x400 + +#define _DMA_GENERIC_FLAGS (DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE | \ + DMA_SRC_LOADED | DMA_DST_LOADED) +#define _DMA_CRC_FLAGS (DMA_CRC_STORE | DMA_CRC_TEST | DMA_CRC_INLINE | \ + DMA_CRC_LOADED) + +/* * Hardware revision number. Different hardware revisions support different * features. For example, 3.2 cannot read from MMIO space, while 3.3 can. */ @@ -131,6 +142,8 @@ int ioat_get_hwversion(bus_dmaengine_t dmaengine); size_t ioat_get_max_io_size(bus_dmaengine_t dmaengine); uint32_t ioat_get_capabilities(bus_dmaengine_t dmaengine); +bus_dma_tag_t ioat_get_dma_tag(bus_dmaengine_t dmaengine); +int ioat_get_domain(bus_dmaengine_t dmaengine, int *domain); /* * Set interrupt coalescing on a DMA channel. Index: ioat.c =================================================================== --- ioat.c +++ ioat.c @@ -1045,7 +1045,25 @@ return (ioat->capabilities); } +bus_dma_tag_t +ioat_get_dma_tag(bus_dmaengine_t dmaengine) +{ + struct ioat_softc *ioat; + + ioat = to_ioat_softc(dmaengine); + return (ioat->data_tag); +} + int +ioat_get_domain(bus_dmaengine_t dmaengine, int *domain) +{ + struct ioat_softc *ioat; + + ioat = to_ioat_softc(dmaengine); + return (bus_get_domain(ioat->device, domain)); +} + +int ioat_set_interrupt_coalesce(bus_dmaengine_t dmaengine, uint16_t delay) { struct ioat_softc *ioat; @@ -1161,7 +1179,9 @@ hw_desc->size = size; - if (src != 0) { + if (flags & DMA_SRC_LOADED) { + hw_desc->src_addr = src; + } else if (src != 0) { nseg = -1; error = _bus_dmamap_load_phys(ioat->data_tag, desc->src_dmamap, src, size, 0, &seg, &nseg); @@ -1173,7 +1193,9 @@ hw_desc->src_addr = seg.ds_addr; } - if (dst != 0) { + if (flags & DMA_DST_LOADED) { + hw_desc->dest_addr = dst; + } else if (dst != 0) { nseg = -1; error = _bus_dmamap_load_phys(ioat->data_tag, desc->dst_dmamap, dst, size, 0, &seg, &nseg); @@ -1268,51 +1290,69 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; - src1_len = (src2 != src1 + PAGE_SIZE) ? PAGE_SIZE : 2 * PAGE_SIZE; - nseg = -1; - error = _bus_dmamap_load_phys(ioat->data_tag, - desc->src_dmamap, src1, src1_len, 0, &seg, &nseg); - if (error != 0) { - ioat_log_message(0, "%s: _bus_dmamap_load_phys" - " failed %d\n", __func__, error); - return (NULL); - } - hw_desc->src_addr = seg.ds_addr; - if (src1_len != 2 * PAGE_SIZE) { - hw_desc->u.control.src_page_break = 1; + if (flags & DMA_SRC_LOADED) { + hw_desc->src_addr = src1; + } else { + src1_len = (src2 != src1 + PAGE_SIZE) ? + PAGE_SIZE : 2 * PAGE_SIZE; nseg = -1; error = _bus_dmamap_load_phys(ioat->data_tag, - desc->src2_dmamap, src2, PAGE_SIZE, 0, &seg, &nseg); + desc->src_dmamap, src1, src1_len, 0, &seg, &nseg); if (error != 0) { ioat_log_message(0, "%s: _bus_dmamap_load_phys" " failed %d\n", __func__, error); return (NULL); } - hw_desc->next_src_addr = seg.ds_addr; + hw_desc->src_addr = seg.ds_addr; } - - dst1_len = (dst2 != dst1 + PAGE_SIZE) ? PAGE_SIZE : 2 * PAGE_SIZE; - nseg = -1; - error = _bus_dmamap_load_phys(ioat->data_tag, - desc->dst_dmamap, dst1, dst1_len, 0, &seg, &nseg); - if (error != 0) { - ioat_log_message(0, "%s: _bus_dmamap_load_phys" - " failed %d\n", __func__, error); - return (NULL); + if (src2 != src1 + PAGE_SIZE) { + hw_desc->u.control.src_page_break = 1; + if (flags & DMA_SRC_LOADED) { + hw_desc->next_src_addr = src2; + } else { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->src2_dmamap, src2, PAGE_SIZE, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->next_src_addr = seg.ds_addr; + } } - hw_desc->dest_addr = seg.ds_addr; - if (dst1_len != 2 * PAGE_SIZE) { - hw_desc->u.control.dest_page_break = 1; + + if (flags & DMA_DST_LOADED) { + hw_desc->dest_addr = dst1; + } else { + dst1_len = (dst2 != dst1 + PAGE_SIZE) ? + PAGE_SIZE : 2 * PAGE_SIZE; nseg = -1; error = _bus_dmamap_load_phys(ioat->data_tag, - desc->dst2_dmamap, dst2, PAGE_SIZE, 0, &seg, &nseg); + desc->dst_dmamap, dst1, dst1_len, 0, &seg, &nseg); if (error != 0) { ioat_log_message(0, "%s: _bus_dmamap_load_phys" " failed %d\n", __func__, error); return (NULL); } - hw_desc->next_dest_addr = seg.ds_addr; + hw_desc->dest_addr = seg.ds_addr; } + if (dst2 != dst1 + PAGE_SIZE) { + hw_desc->u.control.dest_page_break = 1; + if (flags & DMA_DST_LOADED) { + hw_desc->next_dest_addr = dst2; + } else { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->dst2_dmamap, dst2, PAGE_SIZE, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->next_dest_addr = seg.ds_addr; + } + } if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); @@ -1371,7 +1411,9 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; - if ((flags & DMA_CRC_INLINE) == 0) { + if ((flags & (DMA_CRC_INLINE | DMA_CRC_LOADED)) == DMA_CRC_LOADED) { + hw_desc->crc_address = crcptr; + } else if ((flags & DMA_CRC_INLINE) == 0) { nseg = -1; error = _bus_dmamap_load_phys(ioat->data_crc_tag, desc->crc_dmamap, crcptr, sizeof(uint32_t), 0, @@ -1447,7 +1489,9 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; - if ((flags & DMA_CRC_INLINE) == 0) { + if ((flags & (DMA_CRC_INLINE | DMA_CRC_LOADED)) == DMA_CRC_LOADED) { + hw_desc->crc_address = crcptr; + } else if ((flags & DMA_CRC_INLINE) == 0) { nseg = -1; error = _bus_dmamap_load_phys(ioat->data_crc_tag, desc->crc_dmamap, crcptr, sizeof(uint32_t), 0,