Index: head/sys/dev/ioat/ioat.c =================================================================== --- head/sys/dev/ioat/ioat.c (revision 290020) +++ head/sys/dev/ioat/ioat.c (revision 290021) @@ -1,1427 +1,1458 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ioat.h" #include "ioat_hw.h" #include "ioat_internal.h" #define IOAT_INTR_TIMO (hz / 10) #define IOAT_REFLK (&ioat->submit_lock) static int ioat_probe(device_t device); static int ioat_attach(device_t device); static int ioat_detach(device_t device); static int ioat_setup_intr(struct ioat_softc *ioat); static int ioat_teardown_intr(struct ioat_softc *ioat); static int ioat3_attach(device_t device); static int ioat_start_channel(struct ioat_softc *ioat); static int ioat_map_pci_bar(struct ioat_softc *ioat); static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error); static void ioat_interrupt_handler(void *arg); static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat); static void ioat_process_events(struct ioat_softc *ioat); static inline uint32_t ioat_get_active(struct ioat_softc *ioat); static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); static void ioat_free_ring(struct ioat_softc *, uint32_t size, struct ioat_descriptor **); static void ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc); static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *, int mflags); static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags); static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index); static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *, uint32_t size, boolean_t need_dscr, int mflags); static int ring_grow(struct ioat_softc *, uint32_t oldorder, struct ioat_descriptor **); static int ring_shrink(struct ioat_softc *, uint32_t oldorder, struct ioat_descriptor **); static void ioat_timer_callback(void *arg); static void dump_descriptor(void *hw_desc); static void ioat_submit_single(struct ioat_softc *ioat); static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error); static int ioat_reset_hw(struct ioat_softc *ioat); static void ioat_setup_sysctl(device_t device); static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); static inline struct ioat_softc *ioat_get(struct ioat_softc *, enum ioat_ref_kind); static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind); static inline void ioat_putn(struct ioat_softc *, uint32_t, enum ioat_ref_kind); static void ioat_drain(struct ioat_softc *); #define ioat_log_message(v, ...) do { \ if ((v) <= g_ioat_debug_level) { \ device_printf(ioat->device, __VA_ARGS__); \ } \ } while (0) MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations"); SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node"); static int g_force_legacy_interrupts; SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN, &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled"); int g_ioat_debug_level = 0; SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level, 0, "Set log level (0-3) for ioat(4). Higher is more verbose."); /* * OS <-> Driver interface structures */ static device_method_t ioat_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ioat_probe), DEVMETHOD(device_attach, ioat_attach), DEVMETHOD(device_detach, ioat_detach), { 0, 0 } }; static driver_t ioat_pci_driver = { "ioat", ioat_pci_methods, sizeof(struct ioat_softc), }; static devclass_t ioat_devclass; DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0); /* * Private data structures */ static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; static int ioat_channel_index = 0; SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, "Number of IOAT channels attached"); static struct _pcsid { u_int32_t type; const char *desc; } pci_ids[] = { { 0x34308086, "TBG IOAT Ch0" }, { 0x34318086, "TBG IOAT Ch1" }, { 0x34328086, "TBG IOAT Ch2" }, { 0x34338086, "TBG IOAT Ch3" }, { 0x34298086, "TBG IOAT Ch4" }, { 0x342a8086, "TBG IOAT Ch5" }, { 0x342b8086, "TBG IOAT Ch6" }, { 0x342c8086, "TBG IOAT Ch7" }, { 0x37108086, "JSF IOAT Ch0" }, { 0x37118086, "JSF IOAT Ch1" }, { 0x37128086, "JSF IOAT Ch2" }, { 0x37138086, "JSF IOAT Ch3" }, { 0x37148086, "JSF IOAT Ch4" }, { 0x37158086, "JSF IOAT Ch5" }, { 0x37168086, "JSF IOAT Ch6" }, { 0x37178086, "JSF IOAT Ch7" }, { 0x37188086, "JSF IOAT Ch0 (RAID)" }, { 0x37198086, "JSF IOAT Ch1 (RAID)" }, { 0x3c208086, "SNB IOAT Ch0" }, { 0x3c218086, "SNB IOAT Ch1" }, { 0x3c228086, "SNB IOAT Ch2" }, { 0x3c238086, "SNB IOAT Ch3" }, { 0x3c248086, "SNB IOAT Ch4" }, { 0x3c258086, "SNB IOAT Ch5" }, { 0x3c268086, "SNB IOAT Ch6" }, { 0x3c278086, "SNB IOAT Ch7" }, { 0x3c2e8086, "SNB IOAT Ch0 (RAID)" }, { 0x3c2f8086, "SNB IOAT Ch1 (RAID)" }, { 0x0e208086, "IVB IOAT Ch0" }, { 0x0e218086, "IVB IOAT Ch1" }, { 0x0e228086, "IVB IOAT Ch2" }, { 0x0e238086, "IVB IOAT Ch3" }, { 0x0e248086, "IVB IOAT Ch4" }, { 0x0e258086, "IVB IOAT Ch5" }, { 0x0e268086, "IVB IOAT Ch6" }, { 0x0e278086, "IVB IOAT Ch7" }, { 0x0e2e8086, "IVB IOAT Ch0 (RAID)" }, { 0x0e2f8086, "IVB IOAT Ch1 (RAID)" }, { 0x2f208086, "HSW IOAT Ch0" }, { 0x2f218086, "HSW IOAT Ch1" }, { 0x2f228086, "HSW IOAT Ch2" }, { 0x2f238086, "HSW IOAT Ch3" }, { 0x2f248086, "HSW IOAT Ch4" }, { 0x2f258086, "HSW IOAT Ch5" }, { 0x2f268086, "HSW IOAT Ch6" }, { 0x2f278086, "HSW IOAT Ch7" }, { 0x2f2e8086, "HSW IOAT Ch0 (RAID)" }, { 0x2f2f8086, "HSW IOAT Ch1 (RAID)" }, { 0x0c508086, "BWD IOAT Ch0" }, { 0x0c518086, "BWD IOAT Ch1" }, { 0x0c528086, "BWD IOAT Ch2" }, { 0x0c538086, "BWD IOAT Ch3" }, { 0x6f508086, "BDXDE IOAT Ch0" }, { 0x6f518086, "BDXDE IOAT Ch1" }, { 0x6f528086, "BDXDE IOAT Ch2" }, { 0x6f538086, "BDXDE IOAT Ch3" }, { 0x00000000, NULL } }; /* * OS <-> Driver linkage functions */ static int ioat_probe(device_t device) { struct _pcsid *ep; u_int32_t type; type = pci_get_devid(device); for (ep = pci_ids; ep->type; ep++) { if (ep->type == type) { device_set_desc(device, ep->desc); return (0); } } return (ENXIO); } static int ioat_attach(device_t device) { struct ioat_softc *ioat; int error; ioat = DEVICE2SOFTC(device); ioat->device = device; error = ioat_map_pci_bar(ioat); if (error != 0) goto err; ioat->version = ioat_read_cbver(ioat); if (ioat->version < IOAT_VER_3_0) { error = ENODEV; goto err; } error = ioat3_attach(device); if (error != 0) goto err; error = pci_enable_busmaster(device); if (error != 0) goto err; error = ioat_setup_intr(ioat); if (error != 0) goto err; error = ioat_reset_hw(ioat); if (error != 0) goto err; ioat_process_events(ioat); ioat_setup_sysctl(device); ioat_channel[ioat_channel_index++] = ioat; ioat_test_attach(); err: if (error != 0) ioat_detach(device); return (error); } static int ioat_detach(device_t device) { struct ioat_softc *ioat; ioat = DEVICE2SOFTC(device); ioat_test_detach(); ioat_drain(ioat); ioat_teardown_intr(ioat); callout_drain(&ioat->timer); pci_disable_busmaster(device); if (ioat->pci_resource != NULL) bus_release_resource(device, SYS_RES_MEMORY, ioat->pci_resource_id, ioat->pci_resource); if (ioat->ring != NULL) ioat_free_ring(ioat, 1 << ioat->ring_size_order, ioat->ring); if (ioat->comp_update != NULL) { bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map); bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update, ioat->comp_update_map); bus_dma_tag_destroy(ioat->comp_update_tag); } bus_dma_tag_destroy(ioat->hw_desc_tag); return (0); } static int ioat_teardown_intr(struct ioat_softc *ioat) { if (ioat->tag != NULL) bus_teardown_intr(ioat->device, ioat->res, ioat->tag); if (ioat->res != NULL) bus_release_resource(ioat->device, SYS_RES_IRQ, rman_get_rid(ioat->res), ioat->res); pci_release_msi(ioat->device); return (0); } static int ioat_start_channel(struct ioat_softc *ioat) { uint64_t status; uint32_t chanerr; int i; ioat_acquire(&ioat->dmaengine); ioat_null(&ioat->dmaengine, NULL, NULL, 0); ioat_release(&ioat->dmaengine); for (i = 0; i < 100; i++) { DELAY(1); status = ioat_get_chansts(ioat); if (is_ioat_idle(status)) return (0); } chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); ioat_log_message(0, "could not start channel: " "status = %#jx error = %b\n", (uintmax_t)status, (int)chanerr, IOAT_CHANERR_STR); return (ENXIO); } /* * Initialize Hardware */ static int ioat3_attach(device_t device) { struct ioat_softc *ioat; struct ioat_descriptor **ring; struct ioat_descriptor *next; struct ioat_dma_hw_descriptor *dma_hw_desc; uint32_t capabilities; int i, num_descriptors; int error; uint8_t xfercap; error = 0; ioat = DEVICE2SOFTC(device); capabilities = ioat_read_dmacapability(ioat); xfercap = ioat_read_xfercap(ioat); ioat->max_xfer_size = 1 << xfercap; /* TODO: need to check DCA here if we ever do XOR/PQ */ mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF); callout_init(&ioat->timer, 1); ioat->is_resize_pending = FALSE; ioat->is_completion_pending = FALSE; ioat->is_reset_pending = FALSE; ioat->is_channel_running = FALSE; bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, &ioat->comp_update_tag); error = bus_dmamem_alloc(ioat->comp_update_tag, (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); if (ioat->comp_update == NULL) return (ENOMEM); error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map, ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat, 0); if (error != 0) return (error); ioat->ring_size_order = IOAT_MIN_ORDER; num_descriptors = 1 << ioat->ring_size_order; bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, sizeof(struct ioat_dma_hw_descriptor), 1, sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, &ioat->hw_desc_tag); ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, M_ZERO | M_WAITOK); if (ioat->ring == NULL) return (ENOMEM); ring = ioat->ring; for (i = 0; i < num_descriptors; i++) { ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK); if (ring[i] == NULL) return (ENOMEM); ring[i]->id = i; } for (i = 0; i < num_descriptors - 1; i++) { next = ring[i + 1]; dma_hw_desc = ring[i]->u.dma; dma_hw_desc->next = next->hw_desc_bus_addr; } ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; ioat->head = ioat->hw_head = 0; ioat->tail = 0; ioat->last_seen = 0; return (0); } static int ioat_map_pci_bar(struct ioat_softc *ioat) { ioat->pci_resource_id = PCIR_BAR(0); ioat->pci_resource = bus_alloc_resource_any(ioat->device, SYS_RES_MEMORY, &ioat->pci_resource_id, RF_ACTIVE); if (ioat->pci_resource == NULL) { ioat_log_message(0, "unable to allocate pci resource\n"); return (ENODEV); } ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource); ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource); return (0); } static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) { struct ioat_softc *ioat = arg; KASSERT(error == 0, ("%s: error:%d", __func__, error)); ioat->comp_update_bus_addr = seg[0].ds_addr; } static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *baddr; KASSERT(error == 0, ("%s: error:%d", __func__, error)); baddr = arg; *baddr = segs->ds_addr; } /* * Interrupt setup and handlers */ static int ioat_setup_intr(struct ioat_softc *ioat) { uint32_t num_vectors; int error; boolean_t use_msix; boolean_t force_legacy_interrupts; use_msix = FALSE; force_legacy_interrupts = FALSE; if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) { num_vectors = 1; pci_alloc_msix(ioat->device, &num_vectors); if (num_vectors == 1) use_msix = TRUE; } if (use_msix) { ioat->rid = 1; ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, &ioat->rid, RF_ACTIVE); } else { ioat->rid = 0; ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, &ioat->rid, RF_SHAREABLE | RF_ACTIVE); } if (ioat->res == NULL) { ioat_log_message(0, "bus_alloc_resource failed\n"); return (ENOMEM); } ioat->tag = NULL; error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE | INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag); if (error != 0) { ioat_log_message(0, "bus_setup_intr failed\n"); return (error); } ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN); return (0); } static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat) { u_int32_t pciid; pciid = pci_get_devid(ioat->device); switch (pciid) { /* BWD: */ case 0x0c508086: case 0x0c518086: case 0x0c528086: case 0x0c538086: /* BDXDE: */ case 0x6f508086: case 0x6f518086: case 0x6f528086: case 0x6f538086: return (TRUE); } return (FALSE); } static void ioat_interrupt_handler(void *arg) { struct ioat_softc *ioat = arg; ioat_process_events(ioat); } static void ioat_process_events(struct ioat_softc *ioat) { struct ioat_descriptor *desc; struct bus_dmadesc *dmadesc; uint64_t comp_update, status; uint32_t completed; mtx_lock(&ioat->cleanup_lock); completed = 0; comp_update = *ioat->comp_update; status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; CTR0(KTR_IOAT, __func__); if (status == ioat->last_seen) goto out; while (1) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; CTR1(KTR_IOAT, "completing desc %d", ioat->tail); if (dmadesc->callback_fn) (*dmadesc->callback_fn)(dmadesc->callback_arg); completed++; ioat->tail++; if (desc->hw_desc_bus_addr == status) break; } ioat->last_seen = desc->hw_desc_bus_addr; if (ioat->head == ioat->tail) { ioat->is_completion_pending = FALSE; callout_reset(&ioat->timer, IOAT_INTR_TIMO, ioat_timer_callback, ioat); } out: ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); mtx_unlock(&ioat->cleanup_lock); ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); wakeup(&ioat->tail); } /* * User API functions */ bus_dmaengine_t ioat_get_dmaengine(uint32_t index) { if (index >= ioat_channel_index) return (NULL); return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine); } void ioat_put_dmaengine(bus_dmaengine_t dmaengine) { struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); ioat_put(ioat, IOAT_DMAENGINE_REF); } void ioat_acquire(bus_dmaengine_t dmaengine) { struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); mtx_lock(&ioat->submit_lock); CTR0(KTR_IOAT, __func__); } void ioat_release(bus_dmaengine_t dmaengine) { struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); CTR0(KTR_IOAT, __func__); ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head); mtx_unlock(&ioat->submit_lock); } static struct ioat_descriptor * ioat_op_generic(struct ioat_softc *ioat, uint8_t op, uint32_t size, uint64_t src, uint64_t dst, bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags) { struct ioat_generic_hw_descriptor *hw_desc; struct ioat_descriptor *desc; int mflags; mtx_assert(&ioat->submit_lock, MA_OWNED); KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x", flags & ~DMA_ALL_FLAGS)); if ((flags & DMA_NO_WAIT) != 0) mflags = M_NOWAIT; else mflags = M_WAITOK; if (size > ioat->max_xfer_size) { ioat_log_message(0, "%s: max_xfer_size = %d, requested = %u\n", __func__, ioat->max_xfer_size, (unsigned)size); return (NULL); } if (ioat_reserve_space(ioat, 1, mflags) != 0) return (NULL); desc = ioat_get_ring_entry(ioat, ioat->head); hw_desc = desc->u.generic; hw_desc->u.control_raw = 0; hw_desc->u.control_generic.op = op; hw_desc->u.control_generic.completion_update = 1; if ((flags & DMA_INT_EN) != 0) hw_desc->u.control_generic.int_enable = 1; hw_desc->size = size; hw_desc->src_addr = src; hw_desc->dest_addr = dst; desc->bus_dmadesc.callback_fn = callback_fn; desc->bus_dmadesc.callback_arg = callback_arg; return (desc); } struct bus_dmadesc * ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags) { struct ioat_dma_hw_descriptor *hw_desc; struct ioat_descriptor *desc; struct ioat_softc *ioat; CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn, callback_arg, flags); if (desc == NULL) return (NULL); hw_desc = desc->u.dma; hw_desc->u.control.null = 1; ioat_submit_single(ioat); return (&desc->bus_dmadesc); } struct bus_dmadesc * ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags) { struct ioat_dma_hw_descriptor *hw_desc; struct ioat_descriptor *desc; struct ioat_softc *ioat; CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); if (((src | dst) & (0xffffull << 48)) != 0) { ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n", __func__); return (NULL); } desc = ioat_op_generic(ioat, IOAT_OP_COPY, len, src, dst, callback_fn, callback_arg, flags); if (desc == NULL) return (NULL); hw_desc = desc->u.dma; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); ioat_submit_single(ioat); return (&desc->bus_dmadesc); } +struct bus_dmadesc * +ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, uint64_t fillpattern, + bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg, + uint32_t flags) +{ + struct ioat_fill_hw_descriptor *hw_desc; + struct ioat_descriptor *desc; + struct ioat_softc *ioat; + + CTR0(KTR_IOAT, __func__); + ioat = to_ioat_softc(dmaengine); + + if ((dst & (0xffffull << 48)) != 0) { + ioat_log_message(0, "%s: High 16 bits of dst invalid\n", + __func__); + return (NULL); + } + + desc = ioat_op_generic(ioat, IOAT_OP_FILL, len, fillpattern, dst, + callback_fn, callback_arg, flags); + if (desc == NULL) + return (NULL); + + hw_desc = desc->u.fill; + if (g_ioat_debug_level >= 3) + dump_descriptor(hw_desc); + + ioat_submit_single(ioat); + return (&desc->bus_dmadesc); +} + /* * Ring Management */ static inline uint32_t ioat_get_active(struct ioat_softc *ioat) { return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1)); } static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat) { return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); } static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags) { struct ioat_generic_hw_descriptor *hw_desc; struct ioat_descriptor *desc; int error, busdmaflag; error = ENOMEM; hw_desc = NULL; if ((mflags & M_WAITOK) != 0) busdmaflag = BUS_DMA_WAITOK; else busdmaflag = BUS_DMA_NOWAIT; desc = malloc(sizeof(*desc), M_IOAT, mflags); if (desc == NULL) goto out; bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map); if (hw_desc == NULL) goto out; desc->u.generic = hw_desc; error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, busdmaflag); if (error) goto out; out: if (error) { ioat_free_ring_entry(ioat, desc); return (NULL); } return (desc); } static void ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) { if (desc == NULL) return; if (desc->u.generic) bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic, ioat->hw_desc_map); free(desc, M_IOAT); } /* * Reserves space in this IOAT descriptor ring by ensuring enough slots remain * for 'num_descs'. * * If mflags contains M_WAITOK, blocks until enough space is available. * * Returns zero on success, or an errno on error. If num_descs is beyond the * maximum ring size, returns EINVAl; if allocation would block and mflags * contains M_NOWAIT, returns EAGAIN. * * Must be called with the submit_lock held; returns with the lock held. The * lock may be dropped to allocate the ring. * * (The submit_lock is needed to add any entries to the ring, so callers are * assured enough room is available.) */ static int ioat_reserve_space(struct ioat_softc *ioat, uint32_t num_descs, int mflags) { struct ioat_descriptor **new_ring; uint32_t order; int error; mtx_assert(&ioat->submit_lock, MA_OWNED); error = 0; if (num_descs < 1 || num_descs > (1 << IOAT_MAX_ORDER)) { error = EINVAL; goto out; } for (;;) { if (ioat_get_ring_space(ioat) >= num_descs) goto out; order = ioat->ring_size_order; if (ioat->is_resize_pending || order == IOAT_MAX_ORDER) { if ((mflags & M_WAITOK) != 0) { msleep(&ioat->tail, &ioat->submit_lock, 0, "ioat_rsz", 0); continue; } error = EAGAIN; break; } ioat->is_resize_pending = TRUE; for (;;) { mtx_unlock(&ioat->submit_lock); new_ring = ioat_prealloc_ring(ioat, 1 << (order + 1), TRUE, mflags); mtx_lock(&ioat->submit_lock); KASSERT(ioat->ring_size_order == order, ("is_resize_pending should protect order")); if (new_ring == NULL) { KASSERT((mflags & M_WAITOK) == 0, ("allocation failed")); error = EAGAIN; break; } error = ring_grow(ioat, order, new_ring); if (error == 0) break; } ioat->is_resize_pending = FALSE; wakeup(&ioat->tail); if (error) break; } out: mtx_assert(&ioat->submit_lock, MA_OWNED); return (error); } static struct ioat_descriptor ** ioat_prealloc_ring(struct ioat_softc *ioat, uint32_t size, boolean_t need_dscr, int mflags) { struct ioat_descriptor **ring; uint32_t i; int error; KASSERT(size > 0 && powerof2(size), ("bogus size")); ring = malloc(size * sizeof(*ring), M_IOAT, M_ZERO | mflags); if (ring == NULL) return (NULL); if (need_dscr) { error = ENOMEM; for (i = size / 2; i < size; i++) { ring[i] = ioat_alloc_ring_entry(ioat, mflags); if (ring[i] == NULL) goto out; ring[i]->id = i; } } error = 0; out: if (error != 0 && ring != NULL) { ioat_free_ring(ioat, size, ring); ring = NULL; } return (ring); } static void ioat_free_ring(struct ioat_softc *ioat, uint32_t size, struct ioat_descriptor **ring) { uint32_t i; for (i = 0; i < size; i++) { if (ring[i] != NULL) ioat_free_ring_entry(ioat, ring[i]); } free(ring, M_IOAT); } static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) { return (ioat->ring[index % (1 << ioat->ring_size_order)]); } static int ring_grow(struct ioat_softc *ioat, uint32_t oldorder, struct ioat_descriptor **newring) { struct ioat_descriptor *tmp, *next; struct ioat_dma_hw_descriptor *hw; uint32_t oldsize, newsize, head, tail, i, end; int error; CTR0(KTR_IOAT, __func__); mtx_assert(&ioat->submit_lock, MA_OWNED); if (oldorder != ioat->ring_size_order || oldorder >= IOAT_MAX_ORDER) { error = EINVAL; goto out; } oldsize = (1 << oldorder); newsize = (1 << (oldorder + 1)); mtx_lock(&ioat->cleanup_lock); head = ioat->head & (oldsize - 1); tail = ioat->tail & (oldsize - 1); /* Copy old descriptors to new ring */ for (i = 0; i < oldsize; i++) newring[i] = ioat->ring[i]; /* * If head has wrapped but tail hasn't, we must swap some descriptors * around so that tail can increment directly to head. */ if (head < tail) { for (i = 0; i <= head; i++) { tmp = newring[oldsize + i]; newring[oldsize + i] = newring[i]; newring[oldsize + i]->id = oldsize + i; newring[i] = tmp; newring[i]->id = i; } head += oldsize; } KASSERT(head >= tail, ("invariants")); /* Head didn't wrap; we only need to link in oldsize..newsize */ if (head < oldsize) { i = oldsize - 1; end = newsize; } else { /* Head did wrap; link newhead..newsize and 0..oldhead */ i = head; end = newsize + (head - oldsize) + 1; } /* * Fix up hardware ring, being careful not to trample the active * section (tail -> head). */ for (; i < end; i++) { KASSERT((i & (newsize - 1)) < tail || (i & (newsize - 1)) >= head, ("trampling snake")); next = newring[(i + 1) & (newsize - 1)]; hw = newring[i & (newsize - 1)]->u.dma; hw->next = next->hw_desc_bus_addr; } free(ioat->ring, M_IOAT); ioat->ring = newring; ioat->ring_size_order = oldorder + 1; ioat->tail = tail; ioat->head = head; error = 0; mtx_unlock(&ioat->cleanup_lock); out: if (error) ioat_free_ring(ioat, (1 << (oldorder + 1)), newring); return (error); } static int ring_shrink(struct ioat_softc *ioat, uint32_t oldorder, struct ioat_descriptor **newring) { struct ioat_dma_hw_descriptor *hw; struct ioat_descriptor *ent, *next; uint32_t oldsize, newsize, current_idx, new_idx, i; int error; CTR0(KTR_IOAT, __func__); mtx_assert(&ioat->submit_lock, MA_OWNED); if (oldorder != ioat->ring_size_order || oldorder <= IOAT_MIN_ORDER) { error = EINVAL; goto out_unlocked; } oldsize = (1 << oldorder); newsize = (1 << (oldorder - 1)); mtx_lock(&ioat->cleanup_lock); /* Can't shrink below current active set! */ if (ioat_get_active(ioat) >= newsize) { error = ENOMEM; goto out; } /* * Copy current descriptors to the new ring, dropping the removed * descriptors. */ for (i = 0; i < newsize; i++) { current_idx = (ioat->tail + i) & (oldsize - 1); new_idx = (ioat->tail + i) & (newsize - 1); newring[new_idx] = ioat->ring[current_idx]; newring[new_idx]->id = new_idx; } /* Free deleted descriptors */ for (i = newsize; i < oldsize; i++) { ent = ioat_get_ring_entry(ioat, ioat->tail + i); ioat_free_ring_entry(ioat, ent); } /* Fix up hardware ring. */ hw = newring[(ioat->tail + newsize - 1) & (newsize - 1)]->u.dma; next = newring[(ioat->tail + newsize) & (newsize - 1)]; hw->next = next->hw_desc_bus_addr; free(ioat->ring, M_IOAT); ioat->ring = newring; ioat->ring_size_order = oldorder - 1; error = 0; out: mtx_unlock(&ioat->cleanup_lock); out_unlocked: if (error) ioat_free_ring(ioat, (1 << (oldorder - 1)), newring); return (error); } static void ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr) { struct ioat_descriptor *desc; ioat_log_message(0, "Channel halted (%b)\n", (int)chanerr, IOAT_CHANERR_STR); if (chanerr == 0) return; mtx_lock(&ioat->submit_lock); desc = ioat_get_ring_entry(ioat, ioat->tail + 0); dump_descriptor(desc->u.raw); desc = ioat_get_ring_entry(ioat, ioat->tail + 1); dump_descriptor(desc->u.raw); mtx_unlock(&ioat->submit_lock); } static void ioat_timer_callback(void *arg) { struct ioat_descriptor **newring; struct ioat_softc *ioat; uint64_t status; uint32_t chanerr, order; ioat = arg; ioat_log_message(1, "%s\n", __func__); if (ioat->is_completion_pending) { status = ioat_get_chansts(ioat); /* * When halted due to errors, check for channel programming * errors before advancing the completion state. */ if (is_ioat_halted(status)) { chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); ioat_halted_debug(ioat, chanerr); } ioat_process_events(ioat); } else { mtx_lock(&ioat->submit_lock); order = ioat->ring_size_order; if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) { mtx_unlock(&ioat->submit_lock); goto out; } ioat->is_resize_pending = TRUE; mtx_unlock(&ioat->submit_lock); newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE, M_NOWAIT); mtx_lock(&ioat->submit_lock); KASSERT(ioat->ring_size_order == order, ("resize_pending protects order")); if (newring != NULL) ring_shrink(ioat, order, newring); ioat->is_resize_pending = FALSE; mtx_unlock(&ioat->submit_lock); out: /* Slowly scale the ring down if idle. */ if (ioat->ring_size_order > IOAT_MIN_ORDER) callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback, ioat); } } /* * Support Functions */ static void ioat_submit_single(struct ioat_softc *ioat) { ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); atomic_add_rel_int(&ioat->head, 1); atomic_add_rel_int(&ioat->hw_head, 1); if (!ioat->is_completion_pending) { ioat->is_completion_pending = TRUE; callout_reset(&ioat->timer, IOAT_INTR_TIMO, ioat_timer_callback, ioat); } } static int ioat_reset_hw(struct ioat_softc *ioat) { uint64_t status; uint32_t chanerr; unsigned timeout; status = ioat_get_chansts(ioat); if (is_ioat_active(status) || is_ioat_idle(status)) ioat_suspend(ioat); /* Wait at most 20 ms */ for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) && timeout < 20; timeout++) { DELAY(1000); status = ioat_get_chansts(ioat); } if (timeout == 20) return (ETIMEDOUT); KASSERT(ioat_get_active(ioat) == 0, ("active after quiesce")); chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); /* * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors * that can cause stability issues for IOAT v3. */ pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07, 4); chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4); pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4); /* * BDXDE and BWD models reset MSI-X registers on device reset. * Save/restore their contents manually. */ if (ioat_model_resets_msix(ioat)) { ioat_log_message(1, "device resets MSI-X registers; saving\n"); pci_save_state(ioat->device); } ioat_reset(ioat); /* Wait at most 20 ms */ for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++) DELAY(1000); if (timeout == 20) return (ETIMEDOUT); if (ioat_model_resets_msix(ioat)) { ioat_log_message(1, "device resets registers; restored\n"); pci_restore_state(ioat->device); } /* Reset attempts to return the hardware to "halted." */ status = ioat_get_chansts(ioat); if (is_ioat_active(status) || is_ioat_idle(status)) { /* So this really shouldn't happen... */ ioat_log_message(0, "Device is active after a reset?\n"); ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); return (0); } chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); ioat_halted_debug(ioat, chanerr); if (chanerr != 0) return (EIO); /* * Bring device back online after reset. Writing CHAINADDR brings the * device back to active. * * The internal ring counter resets to zero, so we have to start over * at zero as well. */ ioat->tail = ioat->head = ioat->hw_head = 0; ioat->last_seen = 0; ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); ioat_write_chainaddr(ioat, ioat->ring[0]->hw_desc_bus_addr); return (ioat_start_channel(ioat)); } static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS) { struct ioat_softc *ioat; int error, arg; ioat = arg1; arg = 0; error = SYSCTL_OUT(req, &arg, sizeof(arg)); if (error != 0 || req->newptr == NULL) return (error); error = SYSCTL_IN(req, &arg, sizeof(arg)); if (error != 0) return (error); if (arg != 0) error = ioat_reset_hw(ioat); return (error); } static void dump_descriptor(void *hw_desc) { int i, j; for (i = 0; i < 2; i++) { for (j = 0; j < 8; j++) printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]); printf("\n"); } } static void ioat_setup_sysctl(device_t device) { struct sysctl_oid_list *par; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct ioat_softc *ioat; ioat = DEVICE2SOFTC(device); ctx = device_get_sysctl_ctx(device); tree = device_get_sysctl_tree(device); par = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, par, OID_AUTO, "version", CTLFLAG_RD, &ioat->version, 0, "HW version (0xMM form)"); SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "max_xfer_size", CTLFLAG_RD, &ioat->max_xfer_size, 0, "HW maximum transfer size"); SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "ring_size_order", CTLFLAG_RD, &ioat->ring_size_order, 0, "SW descriptor ring size order"); SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, 0, "SW descriptor head pointer index"); SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0, "SW descriptor tail pointer index"); SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "hw_head", CTLFLAG_RD, &ioat->hw_head, 0, "HW DMACOUNT"); SYSCTL_ADD_UQUAD(ctx, par, OID_AUTO, "last_completion", CTLFLAG_RD, ioat->comp_update, "HW addr of last completion"); SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_resize_pending", CTLFLAG_RD, &ioat->is_resize_pending, 0, "resize pending"); SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_completion_pending", CTLFLAG_RD, &ioat->is_completion_pending, 0, "completion pending"); SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_reset_pending", CTLFLAG_RD, &ioat->is_reset_pending, 0, "reset pending"); SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_channel_running", CTLFLAG_RD, &ioat->is_channel_running, 0, "channel running"); SYSCTL_ADD_PROC(ctx, par, OID_AUTO, "force_hw_reset", CTLTYPE_INT | CTLFLAG_RW, ioat, 0, sysctl_handle_reset, "I", "Set to non-zero to reset the hardware"); } static inline struct ioat_softc * ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind) { uint32_t old; KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); old = atomic_fetchadd_32(&ioat->refcnt, 1); KASSERT(old < UINT32_MAX, ("refcnt overflow")); #ifdef INVARIANTS old = atomic_fetchadd_32(&ioat->refkinds[kind], 1); KASSERT(old < UINT32_MAX, ("refcnt kind overflow")); #endif return (ioat); } static inline void ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) { uint32_t old; KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); if (n == 0) return; #ifdef INVARIANTS old = atomic_fetchadd_32(&ioat->refkinds[kind], -n); KASSERT(old >= n, ("refcnt kind underflow")); #endif /* Skip acquiring the lock if resulting refcnt > 0. */ for (;;) { old = ioat->refcnt; if (old <= n) break; if (atomic_cmpset_32(&ioat->refcnt, old, old - n)) return; } mtx_lock(IOAT_REFLK); old = atomic_fetchadd_32(&ioat->refcnt, -n); KASSERT(old >= n, ("refcnt error")); if (old == n) wakeup(IOAT_REFLK); mtx_unlock(IOAT_REFLK); } static inline void ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind) { ioat_putn(ioat, 1, kind); } static void ioat_drain(struct ioat_softc *ioat) { mtx_lock(IOAT_REFLK); while (ioat->refcnt > 0) msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0); mtx_unlock(IOAT_REFLK); } Index: head/sys/dev/ioat/ioat.h =================================================================== --- head/sys/dev/ioat/ioat.h (revision 290020) +++ head/sys/dev/ioat/ioat.h (revision 290021) @@ -1,85 +1,93 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ __FBSDID("$FreeBSD$"); #ifndef __IOAT_H__ #define __IOAT_H__ #include #include /* * This file defines the public interface to the IOAT driver. */ /* * Enables an interrupt for this operation. Typically, you would only enable * this on the last operation in a group */ #define DMA_INT_EN 0x1 /* * Like M_NOWAIT. Operations will return NULL if they cannot allocate a * descriptor without blocking. */ #define DMA_NO_WAIT 0x2 #define DMA_ALL_FLAGS (DMA_INT_EN | DMA_NO_WAIT) typedef void *bus_dmaengine_t; struct bus_dmadesc; typedef void (*bus_dmaengine_callback_t)(void *arg); /* * Called first to acquire a reference to the DMA channel */ bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index); /* Release the DMA channel */ void ioat_put_dmaengine(bus_dmaengine_t dmaengine); /* * Acquire must be called before issuing an operation to perform. Release is * called after. Multiple operations can be issued within the context of one * acquire and release */ void ioat_acquire(bus_dmaengine_t dmaengine); void ioat_release(bus_dmaengine_t dmaengine); +/* + * Issue a blockfill operation. The 64-bit pattern 'fillpattern' is written to + * 'len' physically contiguous bytes at 'dst'. + */ +struct bus_dmadesc *ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, + uint64_t fillpattern, bus_size_t len, bus_dmaengine_callback_t callback_fn, + void *callback_arg, uint32_t flags); + /* Issues the copy data operation */ struct bus_dmadesc *ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags); /* * Issues a null operation. This issues the operation to the hardware, but the * hardware doesn't do anything with it. */ struct bus_dmadesc *ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags); #endif /* __IOAT_H__ */ Index: head/sys/dev/ioat/ioat_test.c =================================================================== --- head/sys/dev/ioat/ioat_test.c (revision 290020) +++ head/sys/dev/ioat/ioat_test.c (revision 290021) @@ -1,442 +1,475 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ioat.h" #include "ioat_hw.h" #include "ioat_internal.h" #include "ioat_test.h" #ifndef time_after #define time_after(a,b) ((long)(b) - (long)(a) < 0) #endif MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations"); #define IOAT_MAX_BUFS 256 struct test_transaction { void *buf[IOAT_MAX_BUFS]; uint32_t length; uint32_t depth; struct ioat_test *test; TAILQ_ENTRY(test_transaction) entry; }; #define IT_LOCK() mtx_lock(&ioat_test_lk) #define IT_UNLOCK() mtx_unlock(&ioat_test_lk) #define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED) static struct mtx ioat_test_lk; MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF); static int g_thread_index = 1; static struct cdev *g_ioat_cdev = NULL; #define ioat_test_log(v, ...) _ioat_test_log((v), "ioat_test: " __VA_ARGS__) static inline void _ioat_test_log(int verbosity, const char *fmt, ...); static void ioat_test_transaction_destroy(struct test_transaction *tx) { int i; for (i = 0; i < IOAT_MAX_BUFS; i++) { if (tx->buf[i] != NULL) { contigfree(tx->buf[i], tx->length, M_IOAT_TEST); tx->buf[i] = NULL; } } free(tx, M_IOAT_TEST); } static struct test_transaction *ioat_test_transaction_create(unsigned num_buffers, uint32_t buffer_size) { struct test_transaction *tx; unsigned i; tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO); if (tx == NULL) return (NULL); tx->length = buffer_size; for (i = 0; i < num_buffers; i++) { tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); if (tx->buf[i] == NULL) { ioat_test_transaction_destroy(tx); return (NULL); } } return (tx); } static bool ioat_compare_ok(struct test_transaction *tx) { - uint32_t i; + struct ioat_test *test; + char *dst, *src; + uint32_t i, j; + test = tx->test; + for (i = 0; i < tx->depth; i++) { - if (memcmp(tx->buf[2*i], tx->buf[2*i+1], tx->length) != 0) - return (false); + dst = tx->buf[2 * i + 1]; + src = tx->buf[2 * i]; + + if (test->testkind == IOAT_TEST_FILL) { + for (j = 0; j < tx->length; j += sizeof(uint64_t)) { + if (memcmp(src, &dst[j], + MIN(sizeof(uint64_t), tx->length - j)) + != 0) + return (false); + } + } else if (test->testkind == IOAT_TEST_DMA) + if (memcmp(src, dst, tx->length) != 0) + return (false); } return (true); } static void ioat_dma_test_callback(void *arg) { struct test_transaction *tx; struct ioat_test *test; tx = arg; test = tx->test; if (test->verify && !ioat_compare_ok(tx)) { ioat_test_log(0, "miscompare found\n"); atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth); } else if (!test->too_late) atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth); IT_LOCK(); TAILQ_REMOVE(&test->pend_q, tx, entry); TAILQ_INSERT_TAIL(&test->free_q, tx, entry); wakeup(&test->free_q); IT_UNLOCK(); } static int ioat_test_prealloc_memory(struct ioat_test *test, int index) { uint32_t i, j, k; struct test_transaction *tx; for (i = 0; i < test->transactions; i++) { tx = ioat_test_transaction_create(test->chain_depth * 2, test->buffer_size); if (tx == NULL) { ioat_test_log(0, "tx == NULL - memory exhausted\n"); test->status[IOAT_TEST_NO_MEMORY]++; return (ENOMEM); } TAILQ_INSERT_HEAD(&test->free_q, tx, entry); tx->test = test; tx->depth = test->chain_depth; /* fill in source buffers */ for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) { uint32_t val = j + (index << 28); for (k = 0; k < test->chain_depth; k++) { ((uint32_t *)tx->buf[2*k])[j] = ~val; ((uint32_t *)tx->buf[2*k+1])[j] = val; } } } return (0); } static void ioat_test_release_memory(struct ioat_test *test) { struct test_transaction *tx, *s; TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s) ioat_test_transaction_destroy(tx); TAILQ_INIT(&test->free_q); TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s) ioat_test_transaction_destroy(tx); TAILQ_INIT(&test->pend_q); } static void ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma) { struct test_transaction *tx; struct bus_dmadesc *desc; bus_dmaengine_callback_t cb; bus_addr_t src, dest; + uint64_t fillpattern; uint32_t i, flags; + desc = NULL; + IT_LOCK(); while (TAILQ_EMPTY(&test->free_q)) msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0); tx = TAILQ_FIRST(&test->free_q); TAILQ_REMOVE(&test->free_q, tx, entry); TAILQ_INSERT_HEAD(&test->pend_q, tx, entry); IT_UNLOCK(); ioat_acquire(dma); for (i = 0; i < tx->depth; i++) { src = vtophys((vm_offset_t)tx->buf[2*i]); dest = vtophys((vm_offset_t)tx->buf[2*i+1]); if (i == tx->depth - 1) { cb = ioat_dma_test_callback; flags = DMA_INT_EN; } else { cb = NULL; flags = 0; } - desc = ioat_copy(dma, src, dest, tx->length, cb, tx, flags); + if (test->testkind == IOAT_TEST_DMA) + desc = ioat_copy(dma, dest, src, tx->length, cb, tx, + flags); + else if (test->testkind == IOAT_TEST_FILL) { + fillpattern = *(uint64_t *)tx->buf[2*i]; + desc = ioat_blockfill(dma, dest, fillpattern, + tx->length, cb, tx, flags); + } + if (desc == NULL) panic("Failed to allocate a ring slot " "-- this shouldn't happen!"); } ioat_release(dma); } static void ioat_dma_test(void *arg) { struct ioat_test *test; bus_dmaengine_t dmaengine; uint32_t loops; int index, rc, start, end; test = arg; memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status)); if (test->buffer_size > 1024 * 1024) { ioat_test_log(0, "Buffer size too large >1MB\n"); test->status[IOAT_TEST_NO_MEMORY]++; return; } if (test->chain_depth * 2 > IOAT_MAX_BUFS) { ioat_test_log(0, "Depth too large (> %u)\n", (unsigned)IOAT_MAX_BUFS / 2); test->status[IOAT_TEST_NO_MEMORY]++; return; } if (btoc((uint64_t)test->buffer_size * test->chain_depth * test->transactions) > (physmem / 4)) { ioat_test_log(0, "Sanity check failed -- test would " "use more than 1/4 of phys mem.\n"); test->status[IOAT_TEST_NO_MEMORY]++; return; } if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) { ioat_test_log(0, "Sanity check failed -- test would " "use more than available IOAT ring space.\n"); test->status[IOAT_TEST_NO_MEMORY]++; + return; + } + + if (test->testkind >= IOAT_NUM_TESTKINDS) { + ioat_test_log(0, "Invalid kind %u\n", + (unsigned)test->testkind); + test->status[IOAT_TEST_INVALID_INPUT]++; return; } dmaengine = ioat_get_dmaengine(test->channel_index); if (dmaengine == NULL) { ioat_test_log(0, "Couldn't acquire dmaengine\n"); test->status[IOAT_TEST_NO_DMA_ENGINE]++; return; } index = g_thread_index++; TAILQ_INIT(&test->free_q); TAILQ_INIT(&test->pend_q); if (test->duration == 0) ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n", index, test->transactions); else ioat_test_log(1, "Thread %d: starting\n", index); rc = ioat_test_prealloc_memory(test, index); if (rc != 0) { ioat_test_log(0, "prealloc_memory: %d\n", rc); goto out; } wmb(); test->too_late = false; start = ticks; end = start + (((sbintime_t)test->duration * hz) / 1000); for (loops = 0;; loops++) { if (test->duration == 0 && loops >= test->transactions) break; else if (test->duration != 0 && time_after(ticks, end)) { test->too_late = true; break; } ioat_test_submit_1_tx(test, dmaengine); } ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n", ticks - start, ticks - end, (ticks - start) / hz); IT_LOCK(); while (!TAILQ_EMPTY(&test->pend_q)) msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz); IT_UNLOCK(); ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n", ticks - start, ticks - end, (ticks - start) / hz); ioat_test_release_memory(test); out: ioat_put_dmaengine(dmaengine); } static int ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td) { return (0); } static int ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td) { return (0); } static int ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag, struct thread *td) { switch (cmd) { case IOAT_DMATEST: ioat_dma_test(arg); break; default: return (EINVAL); } return (0); } static struct cdevsw ioat_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_open = ioat_test_open, .d_close = ioat_test_close, .d_ioctl = ioat_test_ioctl, .d_name = "ioat_test", }; static int enable_ioat_test(bool enable) { mtx_assert(&Giant, MA_OWNED); if (enable && g_ioat_cdev == NULL) { g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "ioat_test"); } else if (!enable && g_ioat_cdev != NULL) { destroy_dev(g_ioat_cdev); g_ioat_cdev = NULL; } return (0); } static int sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS) { int error, enabled; enabled = (g_ioat_cdev != NULL); error = sysctl_handle_int(oidp, &enabled, 0, req); if (error != 0 || req->newptr == NULL) return (error); enable_ioat_test(enabled); return (0); } SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_enable_ioat_test, "I", "Non-zero: Enable the /dev/ioat_test device"); void ioat_test_attach(void) { char *val; val = kern_getenv("hw.ioat.enable_ioat_test"); if (val != NULL && strcmp(val, "0") != 0) { mtx_lock(&Giant); enable_ioat_test(true); mtx_unlock(&Giant); } freeenv(val); } void ioat_test_detach(void) { mtx_lock(&Giant); enable_ioat_test(false); mtx_unlock(&Giant); } static inline void _ioat_test_log(int verbosity, const char *fmt, ...) { va_list argp; if (verbosity > g_ioat_debug_level) return; va_start(argp, fmt); vprintf(fmt, argp); va_end(argp); } Index: head/sys/dev/ioat/ioat_test.h =================================================================== --- head/sys/dev/ioat/ioat_test.h (revision 290020) +++ head/sys/dev/ioat/ioat_test.h (revision 290021) @@ -1,68 +1,77 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ __FBSDID("$FreeBSD$"); #ifndef __IOAT_TEST_H__ #define __IOAT_TEST_H__ enum ioat_res { IOAT_TEST_OK = 0, IOAT_TEST_NO_DMA_ENGINE, IOAT_TEST_NO_MEMORY, IOAT_TEST_MISCOMPARE, + IOAT_TEST_INVALID_INPUT, IOAT_NUM_RES }; +enum ioat_test_kind { + IOAT_TEST_FILL = 0, + IOAT_TEST_DMA, + IOAT_NUM_TESTKINDS +}; + struct test_transaction; struct ioat_test { volatile uint32_t status[IOAT_NUM_RES]; uint32_t channel_index; + + enum ioat_test_kind testkind; /* HW max of 1MB */ uint32_t buffer_size; uint32_t chain_depth; uint32_t transactions; /* * If non-zero, duration is time in ms; * If zero, bounded by 'transactions' above. */ uint32_t duration; /* If true, check for miscompares after a copy. */ bool verify; /* Internal usage -- not test inputs */ TAILQ_HEAD(, test_transaction) free_q; TAILQ_HEAD(, test_transaction) pend_q; volatile bool too_late; }; #define IOAT_DMATEST _IOWR('i', 0, struct ioat_test) #endif /* __IOAT_TEST_H__ */ Index: head/tools/tools/ioat/ioatcontrol.8 =================================================================== --- head/tools/tools/ioat/ioatcontrol.8 (revision 290020) +++ head/tools/tools/ioat/ioatcontrol.8 (revision 290021) @@ -1,125 +1,132 @@ .\" Copyright (c) 2015 EMC / Isilon Storage Division .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd October 21, 2015 +.Dd October 26, 2015 .Dt IOATCONTROL 8 .Os .Sh NAME .Nm ioatcontrol .Nd Userspace tool to test .Xr ioat 4 .Sh SYNOPSIS .Nm +.Op Fl f .Op Fl V .Ar channel_number .Ar num_txns .Ar [ bufsize .Ar [ chain-len .Ar [ duration ] ] ] .Sh DESCRIPTION .Nm allows one to issue some number of test operations to the .Xr ioat 4 driver on a specific hardware channel. The arguments are as follows: .Bl -tag -width Ds +.It Fl f +Test block fill (by default, +.Nm +tests copy) .It Fl V -Verify copies for accuracy +Verify copies/fills for accuracy .El .Pp .Nm operates in one of two modes; if the .Ar duration argument is passed, .Nm tries to estimate the copy rate in bytes per second by running .Ar num_txns repeatedly in loop. If .Ar duration is not passed, .Nm only runs through .Ar num_txns once and prints the total bytes copied, as well as error information. .Pp The .Ar bufsize argument determines the size of buffers to use for each .Fn ioat_copy invocation. The default is 256 KB. .Pp The .Ar chain-len argument determines the number of copies to chain together in a single DMA transaction. The default is 1, and the maximum is currently 4. .Pp The .Ar duration argument specifies an approximate time limit for the test, in milliseconds. .Pp The test will allocate two chunks of memory for each component of each transaction's chain. It will initialize them with specific data patterns. During the test, it submits DMA requests to copy between pairs of buffers. If the .Fl V flag was specified, it will compare the contents in the callback for a copy error. .Sh FILES .Pa /dev/ioat_test .Pp The interface between .Nm and .Xr ioat 4 . -.Nm +.Xr ioat 4 exposes it with .Cd hw.ioat.enable_ioat_test=1 . .Sh DIAGNOSTICS The wait channel .Va test_submit -indicates that the test code is keeping the DMA engine full of work. +indicates that the test code has enqueued all requested transactions and is +waiting on the IOAT hardware to complete one before issuing another operation. .Sh SEE ALSO .Xr ioat 4 .Sh HISTORY The .Xr ioat 4 driver first appeared in .Fx 11.0 . .Sh AUTHORS The .Xr ioat 4 driver and .Nm tool were developed by .An \&Jim Harris Aq Mt jimharris@FreeBSD.org , +.An \&Carl Delsey Aq Mt carl.r.delsey@intel.com , and -.An \&Carl Delsey Aq Mt carl.r.delsey@intel.com . +.An \&Conrad Meyer Aq Mt cem@FreeBSD.org . This manual page was written by .An \&Conrad Meyer Aq Mt cem@FreeBSD.org . Index: head/tools/tools/ioat/ioatcontrol.c =================================================================== --- head/tools/tools/ioat/ioatcontrol.c (revision 290020) +++ head/tools/tools/ioat/ioatcontrol.c (revision 290021) @@ -1,163 +1,171 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "ioat_test.h" static int prettyprint(struct ioat_test *); static void usage(void) { - printf("Usage: %s [-V] [ " + printf("Usage: %s [-fV] [ " "[ [duration]]]\n", getprogname()); exit(EX_USAGE); } int main(int argc, char **argv) { struct ioat_test t; int fd, ch; + bool fflag; - while ((ch = getopt(argc, argv, "V")) != -1) { + while ((ch = getopt(argc, argv, "fV")) != -1) { switch (ch) { + case 'f': + fflag = true; + break; case 'V': t.verify = true; break; default: usage(); } } argc -= optind; argv += optind; if (argc < 2) usage(); /* Defaults for optional args */ t.buffer_size = 256 * 1024; t.chain_depth = 2; t.duration = 0; + t.testkind = IOAT_TEST_DMA; + + if (fflag) + t.testkind = IOAT_TEST_FILL; t.channel_index = atoi(argv[0]); if (t.channel_index > 8) { printf("Channel number must be between 0 and 7.\n"); return (EX_USAGE); } t.transactions = atoi(argv[1]); if (argc >= 3) { t.buffer_size = atoi(argv[2]); if (t.buffer_size == 0) { printf("Buffer size must be greater than zero\n"); return (EX_USAGE); } } if (argc >= 4) { t.chain_depth = atoi(argv[3]); if (t.chain_depth < 1) { printf("Chain length must be greater than zero\n"); return (EX_USAGE); } } if (argc >= 5) { t.duration = atoi(argv[4]); if (t.duration < 1) { printf("Duration must be greater than zero\n"); return (EX_USAGE); } } fd = open("/dev/ioat_test", O_RDWR); if (fd < 0) { printf("Cannot open /dev/ioat_test\n"); return (EX_UNAVAILABLE); } (void)ioctl(fd, IOAT_DMATEST, &t); close(fd); return (prettyprint(&t)); } static int prettyprint(struct ioat_test *t) { char bps[10], bytesh[10]; uintmax_t bytes; if (t->status[IOAT_TEST_NO_DMA_ENGINE] != 0 || t->status[IOAT_TEST_NO_MEMORY] != 0 || t->status[IOAT_TEST_MISCOMPARE] != 0) { printf("Errors:\n"); if (t->status[IOAT_TEST_NO_DMA_ENGINE] != 0) printf("\tNo DMA engine present: %u\n", (unsigned)t->status[IOAT_TEST_NO_DMA_ENGINE]); if (t->status[IOAT_TEST_NO_MEMORY] != 0) printf("\tOut of memory: %u\n", (unsigned)t->status[IOAT_TEST_NO_MEMORY]); if (t->status[IOAT_TEST_MISCOMPARE] != 0) printf("\tMiscompares: %u\n", (unsigned)t->status[IOAT_TEST_MISCOMPARE]); } printf("Processed %u txns\n", (unsigned)t->status[IOAT_TEST_OK] / t->chain_depth); bytes = (uintmax_t)t->buffer_size * t->status[IOAT_TEST_OK]; humanize_number(bytesh, sizeof(bytesh), (int64_t)bytes, "B", HN_AUTOSCALE, HN_DECIMAL); if (t->duration) { humanize_number(bps, sizeof(bps), (int64_t)1000 * bytes / t->duration, "B/s", HN_AUTOSCALE, HN_DECIMAL); printf("%ju (%s) copied in %u ms (%s)\n", bytes, bytesh, (unsigned)t->duration, bps); } else printf("%ju (%s) copied\n", bytes, bytesh); return (EX_OK); }