Index: sys/dev/ioat/ioat.c =================================================================== --- sys/dev/ioat/ioat.c +++ sys/dev/ioat/ioat.c @@ -324,6 +324,7 @@ ioat_detach(device_t device) { struct ioat_softc *ioat; + int i, error; ioat = DEVICE2SOFTC(device); @@ -359,6 +360,69 @@ bus_release_resource(device, SYS_RES_MEMORY, ioat->pci_resource_id, ioat->pci_resource); + for (i = 0; i < 1 << ioat->ring_size_order; i++) { + if (ioat->ring[i].src_dmamap == NULL) + continue; + error = bus_dmamap_destroy(ioat->data_tag, + ioat->ring[i].src_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_destroy failed %d\n", __func__, + error); + return (error); + } + } + for (i = 0; i < 1 << ioat->ring_size_order; i++) { + if (ioat->ring[i].dst_dmamap == NULL) + continue; + error = bus_dmamap_destroy(ioat->data_tag, + ioat->ring[i].dst_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_destroy failed %d\n", __func__, + error); + return (error); + } + } + for (i = 0; i < 1 << ioat->ring_size_order; i++) { + if (ioat->ring[i].src2_dmamap == NULL) + continue; + error = bus_dmamap_destroy(ioat->data_tag, + ioat->ring[i].src2_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_destroy failed %d\n", __func__, + error); + return (error); + } + } + for (i = 0; i < 1 << ioat->ring_size_order; i++) { + if (ioat->ring[i].dst2_dmamap == NULL) + continue; + error = bus_dmamap_destroy(ioat->data_tag, + ioat->ring[i].dst2_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_destroy failed %d\n", __func__, + error); + return (error); + } + } + for (i = 0; i < 1 << ioat->ring_size_order; i++) { + if (ioat->ring[i].crc_dmamap == NULL) + continue; + error = bus_dmamap_destroy(ioat->data_tag, + ioat->ring[i].crc_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_destroy failed %d\n", __func__, + error); + return (error); + } + } + if (ioat->data_tag != NULL) + bus_dma_tag_destroy(ioat->data_tag); + if (ioat->ring != NULL) ioat_free_ring(ioat, 1 << ioat->ring_size_order, ioat->ring); @@ -523,6 +587,15 @@ ioat->hw_desc_ring = hw_desc; + error = bus_dma_tag_create(bus_get_dma_tag(ioat->device), + 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, + ioat->max_xfer_size, 1, ioat->max_xfer_size, 0, NULL, NULL, + &ioat->data_tag); + if (error != 0) { + ioat_log_message(0, "%s: bus_dma_tag_create failed %d\n", + __func__, error); + return (error); + } ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, M_ZERO | M_WAITOK); @@ -530,6 +603,46 @@ for (i = 0; i < num_descriptors; i++) { memset(&ring[i].bus_dmadesc, 0, sizeof(ring[i].bus_dmadesc)); ring[i].id = i; + error = bus_dmamap_create(ioat->data_tag, 0, + &ring[i].src_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_create failed %d\n", __func__, + error); + return (error); + } + error = bus_dmamap_create(ioat->data_tag, 0, + &ring[i].dst_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_create failed %d\n", __func__, + error); + return (error); + } + error = bus_dmamap_create(ioat->data_tag, 0, + &ring[i].src2_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_create failed %d\n", __func__, + error); + return (error); + } + error = bus_dmamap_create(ioat->data_tag, 0, + &ring[i].dst2_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_create failed %d\n", __func__, + error); + return (error); + } + error = bus_dmamap_create(ioat->data_tag, 0, + &ring[i].crc_dmamap); + if (error != 0) { + ioat_log_message(0, + "%s: bus_dmamap_create failed %d\n", __func__, + error); + return (error); + } } for (i = 0; i < num_descriptors; i++) { @@ -724,6 +837,12 @@ ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn, dmadesc->callback_arg); + bus_dmamap_unload(ioat->data_tag, desc->src_dmamap); + bus_dmamap_unload(ioat->data_tag, desc->dst_dmamap); + bus_dmamap_unload(ioat->data_tag, desc->src2_dmamap); + bus_dmamap_unload(ioat->data_tag, desc->dst2_dmamap); + bus_dmamap_unload(ioat->data_tag, desc->crc_dmamap); + if (dmadesc->callback_fn != NULL) dmadesc->callback_fn(dmadesc->callback_arg, 0); @@ -999,7 +1118,8 @@ { struct ioat_generic_hw_descriptor *hw_desc; struct ioat_descriptor *desc; - int mflags; + bus_dma_segment_t seg; + int mflags, nseg, error; mtx_assert(&ioat->submit_lock, MA_OWNED); @@ -1032,9 +1152,31 @@ hw_desc->u.control_generic.fence = 1; hw_desc->size = size; - hw_desc->src_addr = src; - hw_desc->dest_addr = dst; + if (src != 0) { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, desc->src_dmamap, + src, size, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->src_addr = seg.ds_addr; + } + + if (dst != 0) { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, desc->dst_dmamap, + dst, size, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->dest_addr = seg.ds_addr; + } + desc->bus_dmadesc.callback_fn = callback_fn; desc->bus_dmadesc.callback_arg = callback_arg; return (desc); @@ -1102,6 +1244,8 @@ struct ioat_dma_hw_descriptor *hw_desc; struct ioat_descriptor *desc; struct ioat_softc *ioat; + bus_dma_segment_t seg; + int nseg, error; ioat = to_ioat_softc(dmaengine); CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); @@ -1125,11 +1269,27 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; if (src2 != src1 + PAGE_SIZE) { hw_desc->u.control.src_page_break = 1; - hw_desc->next_src_addr = src2; + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->src2_dmamap, src2, PAGE_SIZE, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->next_src_addr = seg.ds_addr; } if (dst2 != dst1 + PAGE_SIZE) { hw_desc->u.control.dest_page_break = 1; - hw_desc->next_dest_addr = dst2; + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->dst2_dmamap, dst2, PAGE_SIZE, 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->next_dest_addr = seg.ds_addr; } if (g_ioat_debug_level >= 3) @@ -1149,6 +1309,8 @@ struct ioat_softc *ioat; uint32_t teststore; uint8_t op; + bus_dma_segment_t seg; + int nseg, error; ioat = to_ioat_softc(dmaengine); CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); @@ -1201,9 +1363,17 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; - if ((flags & DMA_CRC_INLINE) == 0) - hw_desc->crc_address = crcptr; - else + if ((flags & DMA_CRC_INLINE) == 0) { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->crc_dmamap, crcptr, sizeof(crcptr), 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->crc_address = seg.ds_addr; + } else hw_desc->u.control.crc_location = 1; if (initialseed != NULL) { @@ -1228,6 +1398,8 @@ struct ioat_softc *ioat; uint32_t teststore; uint8_t op; + bus_dma_segment_t seg; + int nseg, error; ioat = to_ioat_softc(dmaengine); CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); @@ -1280,9 +1452,17 @@ hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; - if ((flags & DMA_CRC_INLINE) == 0) - hw_desc->crc_address = crcptr; - else + if ((flags & DMA_CRC_INLINE) == 0) { + nseg = -1; + error = _bus_dmamap_load_phys(ioat->data_tag, + desc->crc_dmamap, crcptr, sizeof(crcptr), 0, &seg, &nseg); + if (error != 0) { + ioat_log_message(0, "%s: _bus_dmamap_load_phys" + " failed %d\n", __func__, error); + return (NULL); + } + hw_desc->crc_address = seg.ds_addr; + } else hw_desc->u.control.crc_location = 1; if (initialseed != NULL) { @@ -1321,12 +1501,13 @@ return (NULL); } - desc = ioat_op_generic(ioat, IOAT_OP_FILL, len, fillpattern, dst, + desc = ioat_op_generic(ioat, IOAT_OP_FILL, len, 0, dst, callback_fn, callback_arg, flags); if (desc == NULL) return (NULL); hw_desc = &ioat_get_descriptor(ioat, desc->id)->fill; + hw_desc->src_data = fillpattern; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); Index: sys/dev/ioat/ioat_internal.h =================================================================== --- sys/dev/ioat/ioat_internal.h +++ sys/dev/ioat/ioat_internal.h @@ -414,6 +414,11 @@ struct ioat_descriptor { struct bus_dmadesc bus_dmadesc; uint32_t id; + bus_dmamap_t src_dmamap; + bus_dmamap_t dst_dmamap; + bus_dmamap_t src2_dmamap; + bus_dmamap_t dst2_dmamap; + bus_dmamap_t crc_dmamap; }; /* Unused by this driver at this time. */ @@ -457,6 +462,8 @@ bus_dma_tag_t hw_desc_tag; bus_dmamap_t hw_desc_map; + bus_dma_tag_t data_tag; + bus_dma_tag_t comp_update_tag; bus_dmamap_t comp_update_map; uint64_t *comp_update;