Page MenuHomeFreeBSD
Files F145618538

D3457.id8132.diff
No OneTemporary
Actions

D3457.id8132.diff
View Options

Index: sys/amd64/conf/NOTES
===================================================================
--- sys/amd64/conf/NOTES
+++ sys/amd64/conf/NOTES
@@ -468,6 +468,7 @@
device asmc
#device si
device tpm
+device ioat # Intel I/OAT DMA engine
#
# Laptop/Notebook options:
Index: sys/conf/files.amd64
===================================================================
--- sys/conf/files.amd64
+++ sys/conf/files.amd64
@@ -175,6 +175,8 @@
dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci
dev/if_ndis/if_ndis_usb.c optional ndis usb
dev/io/iodev.c optional io
+dev/ioat/ioat.c optional ioat pci
+dev/ioat/ioat_test.c optional ioat pci
dev/ipmi/ipmi.c optional ipmi
dev/ipmi/ipmi_acpi.c optional ipmi acpi
dev/ipmi/ipmi_isa.c optional ipmi isa
Index: sys/dev/ioat/ioat.h
===================================================================
--- sys/dev/ioat/ioat.h
+++ sys/dev/ioat/ioat.h
@@ -34,7 +34,7 @@
* This file defines the public interface to the IOAT driver.
*/
-/*
+/*
* Enables an interrupt for this operation. Typically, you would only enable
* this on the last operation in a group
*/
@@ -44,12 +44,14 @@
struct bus_dmadesc;
typedef void (*bus_dmaengine_callback_t)(void *arg);
-/* Called first to acquire a reference to the DMA channel */
+/*
+ * Called first to acquire a reference to the DMA channel
+ */
bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index);
-/*
+/*
* Acquire must be called before issuing an operation to perform. Release is
- * called after. Multiple operations can be issued within the context of one
+ * called after. Multiple operations can be issued within the context of one
* acquire and release
*/
void ioat_acquire(bus_dmaengine_t dmaengine);
Index: sys/dev/ioat/ioat.c
===================================================================
--- sys/dev/ioat/ioat.c
+++ sys/dev/ioat/ioat.c
@@ -39,14 +39,16 @@
#include <sys/mutex.h>
#include <sys/rman.h>
#include <sys/sysctl.h>
+#include <sys/time.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <machine/bus.h>
#include <machine/resource.h>
+#include <machine/stdarg.h>
#include "ioat.h"
+#include "ioat_hw.h"
#include "ioat_internal.h"
-#include "ioat_logger.h"
static int ioat_probe(device_t device);
static int ioat_attach(device_t device);
@@ -55,7 +57,7 @@
static int ioat_map_pci_bar(struct ioat_softc *ioat);
static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
int error);
-static void ioat_interrupt_setup(struct ioat_softc *ioat);
+static int ioat_interrupt_setup(struct ioat_softc *ioat);
static void ioat_interrupt_handler(void *arg);
static void ioat_process_events(struct ioat_softc *ioat);
static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
@@ -76,8 +78,16 @@
static void ioat_setup_sysctl(device_t device);
MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
+SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
+
+static int g_force_legacy_interrupts;
+SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
+ &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
+
+static int g_ioat_debug_level = 0;
+SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
+ 0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
-//#define IOAT_LOGGING
/*
* OS <-> Driver interface structures
*/
@@ -101,25 +111,80 @@
/*
* Private data structures
*/
-struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
-int ioat_channel_index = 0;
+static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
+static int ioat_channel_index = 0;
+SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
+ "Number of IOAT channels attached");
static struct _pcsid
{
u_int32_t type;
const char *desc;
} pci_ids[] = {
- { 0x3c208086, "SNB IOAT Ch0" },
- { 0x3c218086, "SNB IOAT Ch1" },
- { 0x3c228086, "SNB IOAT Ch2" },
- { 0x3c238086, "SNB IOAT Ch3" },
- { 0x3c248086, "SNB IOAT Ch4" },
- { 0x3c258086, "SNB IOAT Ch5" },
- { 0x3c268086, "SNB IOAT Ch6" },
- { 0x3c278086, "SNB IOAT Ch7" },
- { 0x3c2e8086, "SNB IOAT Ch0" },
- { 0x3c2f8086, "SNB IOAT Ch1" },
- { 0x00000000, NULL }
+ { 0x34308086, "TBG IOAT Ch0" },
+ { 0x34318086, "TBG IOAT Ch1" },
+ { 0x34328086, "TBG IOAT Ch2" },
+ { 0x34338086, "TBG IOAT Ch3" },
+ { 0x34298086, "TBG IOAT Ch4" },
+ { 0x342a8086, "TBG IOAT Ch5" },
+ { 0x342b8086, "TBG IOAT Ch6" },
+ { 0x342c8086, "TBG IOAT Ch7" },
+
+ { 0x37108086, "JSF IOAT Ch0" },
+ { 0x37118086, "JSF IOAT Ch1" },
+ { 0x37128086, "JSF IOAT Ch2" },
+ { 0x37138086, "JSF IOAT Ch3" },
+ { 0x37148086, "JSF IOAT Ch4" },
+ { 0x37158086, "JSF IOAT Ch5" },
+ { 0x37168086, "JSF IOAT Ch6" },
+ { 0x37178086, "JSF IOAT Ch7" },
+ { 0x37188086, "JSF IOAT Ch0 (RAID)" },
+ { 0x37198086, "JSF IOAT Ch1 (RAID)" },
+
+ { 0x3c208086, "SNB IOAT Ch0" },
+ { 0x3c218086, "SNB IOAT Ch1" },
+ { 0x3c228086, "SNB IOAT Ch2" },
+ { 0x3c238086, "SNB IOAT Ch3" },
+ { 0x3c248086, "SNB IOAT Ch4" },
+ { 0x3c258086, "SNB IOAT Ch5" },
+ { 0x3c268086, "SNB IOAT Ch6" },
+ { 0x3c278086, "SNB IOAT Ch7" },
+ { 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
+ { 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
+
+ { 0x0e208086, "IVB IOAT Ch0" },
+ { 0x0e218086, "IVB IOAT Ch1" },
+ { 0x0e228086, "IVB IOAT Ch2" },
+ { 0x0e238086, "IVB IOAT Ch3" },
+ { 0x0e248086, "IVB IOAT Ch4" },
+ { 0x0e258086, "IVB IOAT Ch5" },
+ { 0x0e268086, "IVB IOAT Ch6" },
+ { 0x0e278086, "IVB IOAT Ch7" },
+ { 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
+ { 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
+
+ { 0x2f208086, "HSW IOAT Ch0" },
+ { 0x2f218086, "HSW IOAT Ch1" },
+ { 0x2f228086, "HSW IOAT Ch2" },
+ { 0x2f238086, "HSW IOAT Ch3" },
+ { 0x2f248086, "HSW IOAT Ch4" },
+ { 0x2f258086, "HSW IOAT Ch5" },
+ { 0x2f268086, "HSW IOAT Ch6" },
+ { 0x2f278086, "HSW IOAT Ch7" },
+ { 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
+ { 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
+
+ { 0x0c508086, "BWD IOAT Ch0" },
+ { 0x0c518086, "BWD IOAT Ch1" },
+ { 0x0c528086, "BWD IOAT Ch2" },
+ { 0x0c538086, "BWD IOAT Ch3" },
+
+ { 0x6f508086, "BDXDE IOAT Ch0" },
+ { 0x6f518086, "BDXDE IOAT Ch1" },
+ { 0x6f528086, "BDXDE IOAT Ch2" },
+ { 0x6f538086, "BDXDE IOAT Ch3" },
+
+ { 0x00000000, NULL }
};
/*
@@ -128,15 +193,15 @@
static int
ioat_probe(device_t device)
{
- u_int32_t type = pci_get_devid(device);
- struct _pcsid *ep = pci_ids;
+ struct _pcsid *ep;
+ u_int32_t type;
- while (ep->type) {
+ type = pci_get_devid(device);
+ for (ep = pci_ids; ep->type; ep++) {
if (ep->type == type) {
device_set_desc(device, ep->desc);
return (0);
}
- ++ep;
}
return (ENXIO);
}
@@ -144,54 +209,62 @@
static int
ioat_attach(device_t device)
{
- int error = 0;
- struct ioat_softc *ioat = DEVICE2SOFTC(device);
+ struct ioat_softc *ioat;
+ int error;
+ ioat = DEVICE2SOFTC(device);
ioat->device = device;
- /*
- * TODO: this needs to be moved into a driver init function
- * so that it only gets executed once overall, rather than
- * once per channel. But this works fine for now.
- */
- TUNABLE_INT_FETCH("hw.ioat.debug_level", &g_ioat_debug_level);
+ error = ioat_map_pci_bar(ioat);
+ if (error != 0)
+ goto err;
- ioat_map_pci_bar(ioat);
ioat->version = ioat_read_cbver(ioat);
ioat_interrupt_setup(ioat);
- if (ioat->version >= IOAT_VER_3_0) {
- error = ioat3_attach(device);
- } else {
- ioat_detach(device);
- error = -1;
+ if (ioat->version < IOAT_VER_3_0) {
+ error = ENODEV;
+ goto err;
}
- if (error == 0)
- ioat_channel[ioat_channel_index++] = ioat;
+ error = ioat3_attach(device);
+ if (error != 0)
+ goto err;
+
+ error = pci_enable_busmaster(device);
+ if (error != 0)
+ goto err;
+
+ ioat_channel[ioat_channel_index++] = ioat;
+err:
+ if (error != 0)
+ ioat_detach(device);
return (error);
}
static int
ioat_detach(device_t device)
{
- struct ioat_softc *ioat = DEVICE2SOFTC(device);
+ struct ioat_softc *ioat;
+ uint32_t i;
+
+ ioat = DEVICE2SOFTC(device);
+ callout_drain(&ioat->timer);
- if (ioat->pci_resource != NULL) {
+ pci_disable_busmaster(device);
+
+ if (ioat->pci_resource != NULL)
bus_release_resource(device, SYS_RES_MEMORY,
ioat->pci_resource_id, ioat->pci_resource);
- }
- if (ioat->ring) {
- int i;
+ if (ioat->ring != NULL) {
for (i = 0; i < (1 << ioat->ring_size_order); i++)
ioat_free_ring_entry(ioat, ioat->ring[i]);
-
free(ioat->ring, M_IOAT);
}
- if (ioat->comp_update) {
+ if (ioat->comp_update != NULL) {
bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
ioat->comp_update_map);
@@ -200,35 +273,59 @@
bus_dma_tag_destroy(ioat->hw_desc_tag);
- if(ioat->tag != NULL)
+ if (ioat->tag != NULL)
bus_teardown_intr(device, ioat->res, ioat->tag);
- if(ioat->res != NULL)
+ if (ioat->res != NULL)
bus_release_resource(device, SYS_RES_IRQ,
rman_get_rid(ioat->res), ioat->res);
pci_release_msi(device);
- callout_drain(&ioat->timer);
return (0);
}
+static int
+ioat3_selftest(struct ioat_softc *ioat)
+{
+ uint64_t status;
+ uint32_t chanerr;
+ int i;
+
+ ioat_acquire(&ioat->dmaengine);
+ ioat_null(&ioat->dmaengine, NULL, NULL, 0);
+ ioat_release(&ioat->dmaengine);
+
+ for (i = 0; i < 100; i++) {
+ DELAY(1);
+ status = ioat_get_chansts(ioat);
+ if (is_ioat_idle(status))
+ return (0);
+ }
+
+ chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
+ ioat_log_message(0, "could not start channel: "
+ "status = %#jx error = %x\n", (uintmax_t)status, chanerr);
+ return (ENXIO);
+}
+
/*
* Initialize Hardware
*/
static int
ioat3_attach(device_t device)
{
- struct ioat_softc *ioat = DEVICE2SOFTC(device);
+ struct ioat_softc *ioat;
struct ioat_descriptor **ring;
- int error = 0;
- uint8_t xfercap;
- uint32_t capabilities, chanerr;
- uint64_t status;
- int i, num_descriptors;
struct ioat_descriptor *next;
struct ioat_dma_hw_descriptor *dma_hw_desc;
+ uint32_t capabilities;
+ int i, num_descriptors;
+ int error;
+ uint8_t xfercap;
+ error = 0;
+ ioat = DEVICE2SOFTC(device);
capabilities = ioat_read_dmacapability(ioat);
xfercap = ioat_read_xfercap(ioat);
@@ -238,7 +335,7 @@
ioat->max_xfer_size = 1 << xfercap;
/* TODO: need to check DCA here if we ever do XOR/PQ */
-
+
mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
callout_init(&ioat->timer, CALLOUT_MPSAFE);
@@ -256,15 +353,14 @@
error = bus_dmamem_alloc(ioat->comp_update_tag,
(void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
-
- if (!ioat->comp_update) {
- ioat_detach(ioat->device);
- return (-1);
- }
+ if (ioat->comp_update == NULL)
+ return (ENOMEM);
error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
- ioat->comp_update, sizeof(uint64_t),
- ioat_comp_update_map, ioat, 0);
+ ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
+ 0);
+ if (error != 0)
+ return (error);
ioat->ring_size_order = IOAT_MIN_ORDER;
@@ -278,27 +374,20 @@
ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
M_ZERO | M_NOWAIT);
-
- if (!ioat->ring) {
- ioat_detach(ioat->device);
- return (-1);
- }
+ if (ioat->ring == NULL)
+ return (ENOMEM);
ring = ioat->ring;
-
for (i = 0; i < num_descriptors; i++) {
ring[i] = ioat_alloc_ring_entry(ioat);
-
- if (!ring[i]) {
- ioat_detach(ioat->device);
- return (-1);
- }
+ if (ring[i] == NULL)
+ return (ENOMEM);
ring[i]->id = i;
}
- for (i = 0; i < num_descriptors-1; i++) {
- next = ring[i+1];
+ for (i = 0; i < num_descriptors - 1; i++) {
+ next = ring[i + 1];
dma_hw_desc = ring[i]->u.dma;
dma_hw_desc->next = next->hw_desc_bus_addr;
@@ -310,46 +399,20 @@
ioat->tail = 0;
ioat->last_seen = 0;
- status = ioat_get_chansts(ioat);
-
- ioat_reset_hw(ioat);
+ error = ioat_reset_hw(ioat);
+ if (error != 0)
+ return (error);
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
- ioat_acquire(&ioat->dmaengine);
- ioat_null(&ioat->dmaengine, NULL, NULL, 0);
- ioat_release(&ioat->dmaengine);
-
- i = 100;
- while (i-- > 0) {
- DELAY(1);
- status = ioat_get_chansts(ioat);
- if (is_ioat_idle(status))
- break;
- }
-
- if (is_ioat_idle(status)) {
- ioat_process_events(ioat);
- } else {
- chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
-
- ioat_log_message(0, "could not start channel: "
- "status = %p\n error = %x\n",
- (void *)status, chanerr);
-
- error = -1;
- }
-
- if (error)
- {
- ioat_detach(device);
+ error = ioat3_selftest(ioat);
+ if (error != 0)
return (error);
- }
+ ioat_process_events(ioat);
ioat_setup_sysctl(device);
-
return (0);
}
@@ -359,16 +422,15 @@
ioat->pci_resource_id = PCIR_BAR(0);
ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY,
- &ioat->pci_resource_id, 0, ~0,
- 1, RF_ACTIVE);
+ &ioat->pci_resource_id, 0, ~0, 1, RF_ACTIVE);
- if(ioat->pci_resource == NULL)
+ if (ioat->pci_resource == NULL) {
ioat_log_message(0, "unable to allocate pci resource\n");
- else {
- ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
- ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
+ return (ENODEV);
}
+ ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
+ ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
return (0);
}
@@ -383,58 +445,57 @@
static void
ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
+ bus_addr_t *baddr;
- bus_addr_t *baddr = arg;
+ baddr = arg;
*baddr = segs->ds_addr;
}
/*
* Interrupt setup and handlers
*/
-static void
+static int
ioat_interrupt_setup(struct ioat_softc *ioat)
{
- boolean_t use_msix = 0;
- boolean_t force_legacy_interrupts = 0;
-
- TUNABLE_INT_FETCH("hw.ioat.force_legacy_interrupts",
- &force_legacy_interrupts);
+ uint32_t num_vectors;
+ int error;
+ boolean_t use_msix;
+ boolean_t force_legacy_interrupts;
- if (!force_legacy_interrupts &&
- pci_msix_count(ioat->device) >= 1) {
- uint32_t num_vectors = 1;
+ use_msix = FALSE;
+ force_legacy_interrupts = FALSE;
+ if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
+ num_vectors = 1;
pci_alloc_msix(ioat->device, &num_vectors);
if (num_vectors == 1)
use_msix = TRUE;
}
- if (use_msix == TRUE) {
+ if (use_msix) {
ioat->rid = 1;
ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
&ioat->rid, RF_ACTIVE);
- }
- else {
+ } else {
ioat->rid = 0;
ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
&ioat->rid, RF_SHAREABLE | RF_ACTIVE);
}
-
if (ioat->res == NULL) {
ioat_log_message(0, "bus_alloc_resource failed\n");
- return;
+ return (ENOMEM);
}
ioat->tag = NULL;
-
- if (bus_setup_intr(ioat->device, ioat->res,
- INTR_MPSAFE | INTR_TYPE_MISC, NULL,
- ioat_interrupt_handler, ioat, &ioat->tag)) {
+ error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
+ INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
+ if (error != 0) {
ioat_log_message(0, "bus_setup_intr failed\n");
- return;
+ return (error);
}
ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
+ return (0);
}
static void
@@ -449,9 +510,9 @@
ioat_process_events(struct ioat_softc *ioat)
{
struct ioat_descriptor *desc;
+ struct bus_dmadesc *dmadesc;
uint64_t comp_update, status;
uint32_t completed;
- struct bus_dmadesc *dmadesc;
mtx_lock(&ioat->cleanup_lock);
@@ -469,30 +530,24 @@
while (1) {
desc = ioat_get_ring_entry(ioat, ioat->tail);
dmadesc = &desc->bus_dmadesc;
-
ioat_log_message(3, "completing desc %d\n", ioat->tail);
- if (dmadesc->callback_fn) {
+ if (dmadesc->callback_fn)
(*dmadesc->callback_fn)(dmadesc->callback_arg);
- }
ioat->tail++;
-
if (desc->hw_desc_bus_addr == status)
break;
-
}
ioat->last_seen = desc->hw_desc_bus_addr;
if (ioat->head == ioat->tail) {
ioat->is_completion_pending = FALSE;
- callout_reset(&ioat->timer, 5000*hz/1000, ioat_timer_callback,
- ioat);
+ callout_reset(&ioat->timer, 5 * hz, ioat_timer_callback, ioat);
}
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
-
mtx_unlock(&ioat->cleanup_lock);
}
@@ -505,7 +560,6 @@
if (index < ioat_channel_index)
return (&ioat_channel[index]->dmaengine);
-
return (NULL);
}
@@ -514,7 +568,7 @@
{
struct ioat_softc *ioat;
- ioat = (struct ioat_softc *)dmaengine;
+ ioat = to_ioat_softc(dmaengine);
mtx_lock(&ioat->submit_lock);
ioat_log_message(3, "%s\n", __func__);
}
@@ -525,7 +579,7 @@
struct ioat_softc *ioat;
ioat_log_message(3, "%s\n", __func__);
- ioat = (struct ioat_softc *)dmaengine;
+ ioat = to_ioat_softc(dmaengine);
ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
mtx_unlock(&ioat->submit_lock);
}
@@ -538,7 +592,7 @@
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
- ioat = (struct ioat_softc *)dmaengine;
+ ioat = to_ioat_softc(dmaengine);
if (ioat_reserve_space_and_lock(ioat, 1) != 0)
return (NULL);
@@ -552,7 +606,7 @@
hw_desc->u.control.null = 1;
hw_desc->u.control.completion_update = 1;
- if (flags & DMA_INT_EN)
+ if ((flags & DMA_INT_EN) != 0)
hw_desc->u.control.int_enable = 1;
hw_desc->size = 8;
@@ -563,7 +617,6 @@
desc->bus_dmadesc.callback_arg = callback_arg;
ioat_submit_single(ioat);
-
return (&desc->bus_dmadesc);
}
@@ -576,11 +629,13 @@
struct ioat_dma_hw_descriptor *hw_desc;
struct ioat_softc *ioat;
- ioat = (struct ioat_softc *)dmaengine;
+ ioat = to_ioat_softc(dmaengine);
- if (len > ioat->max_xfer_size)
- panic("ioat_copy: max_xfer_size = %d, requested = %d\n",
- ioat->max_xfer_size, (int)len);
+ if (len > ioat->max_xfer_size) {
+ ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
+ __func__, ioat->max_xfer_size, (int)len);
+ return (NULL);
+ }
if (ioat_reserve_space_and_lock(ioat, 1) != 0)
return (NULL);
@@ -593,23 +648,20 @@
hw_desc->u.control_raw = 0;
hw_desc->u.control.completion_update = 1;
- if (flags & DMA_INT_EN)
+ if ((flags & DMA_INT_EN) != 0)
hw_desc->u.control.int_enable = 1;
hw_desc->size = len;
hw_desc->src_addr = src;
hw_desc->dest_addr = dst;
-#ifdef IOAT_LOGGING
if (g_ioat_debug_level >= 3)
dump_descriptor(hw_desc);
-#endif
desc->bus_dmadesc.callback_fn = callback_fn;
desc->bus_dmadesc.callback_arg = callback_arg;
ioat_submit_single(ioat);
-
return (&desc->bus_dmadesc);
}
@@ -633,17 +685,15 @@
static struct ioat_descriptor *
ioat_alloc_ring_entry(struct ioat_softc *ioat)
{
- struct ioat_descriptor *desc;
- struct ioat_dma_hw_descriptor *hw_desc;
+ struct ioat_dma_hw_descriptor *hw_desc;
+ struct ioat_descriptor *desc;
desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT);
-
if (desc == NULL)
return (NULL);
bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO,
&ioat->hw_desc_map);
-
if (hw_desc == NULL) {
free(desc, M_IOAT);
return (NULL);
@@ -653,7 +703,6 @@
sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0);
desc->u.dma = hw_desc;
-
return (desc);
}
@@ -661,34 +710,31 @@
ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
{
- if (desc != NULL) {
- if (desc->u.dma)
- bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
- ioat->hw_desc_map);
- free(desc, M_IOAT);
- }
+ if (desc == NULL)
+ return;
+
+ if (desc->u.dma)
+ bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
+ ioat->hw_desc_map);
+ free(desc, M_IOAT);
}
static int
ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
{
- int retry;
+ boolean_t retry;
- do {
- if (ioat_get_ring_space(ioat) >= num_descs) {
+ while (1) {
+ if (ioat_get_ring_space(ioat) >= num_descs)
return (0);
- }
mtx_lock(&ioat->cleanup_lock);
retry = resize_ring(ioat, ioat->ring_size_order + 1);
mtx_unlock(&ioat->cleanup_lock);
- if (retry)
- continue;
- else {
+ if (!retry)
return (ENOMEM);
- }
- } while (1);
+ }
}
static struct ioat_descriptor *
@@ -724,19 +770,18 @@
/* allocate the array to hold the software ring */
ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
-
- if (!ring)
+ if (ring == NULL)
return (FALSE);
- ioat_log_message(1, "ring resize: new: %d old: %d\n",
+ ioat_log_message(2, "ring resize: new: %d old: %d\n",
new_size, current_size);
/* allocate/trim descriptors as needed */
if (new_size > current_size) {
/* copy current descriptors to the new ring */
for (i = 0; i < current_size; i++) {
- current_idx = (ioat->tail+i) & (current_size-1);
- new_idx = (ioat->tail+i) & (new_size-1);
+ current_idx = (ioat->tail + i) & (current_size - 1);
+ new_idx = (ioat->tail + i) & (new_size - 1);
ring[new_idx] = ioat->ring[current_idx];
ring[new_idx]->id = new_idx;
@@ -744,15 +789,15 @@
/* add new descriptors to the ring */
for (i = current_size; i < new_size; i++) {
- new_idx = (ioat->tail+i) & (new_size-1);
+ new_idx = (ioat->tail + i) & (new_size - 1);
ring[new_idx] = ioat_alloc_ring_entry(ioat);
if (!ring[new_idx]) {
while (i--) {
- new_idx2 = (ioat->tail+i) &
- (new_size-1);
+ new_idx2 = (ioat->tail + i) &
+ (new_size - 1);
- ioat_free_ring_entry(ioat,
+ ioat_free_ring_entry(ioat,
ring[new_idx2]);
}
free(ring, M_IOAT);
@@ -761,9 +806,9 @@
ring[new_idx]->id = new_idx;
}
- for (i = current_size-1; i < new_size; i++) {
- new_idx = (ioat->tail+i) & (new_size-1);
- next = ring[(new_idx+1) & (new_size-1)];
+ for (i = current_size - 1; i < new_size; i++) {
+ new_idx = (ioat->tail + i) & (new_size - 1);
+ next = ring[(new_idx + 1) & (new_size - 1)];
hw = ring[new_idx]->u.dma;
hw->next = next->hw_desc_bus_addr;
@@ -774,22 +819,22 @@
* removed descriptors
*/
for (i = 0; i < new_size; i++) {
- uint32_t curr_idx = (ioat->tail+i) & (current_size-1);
- uint32_t new_idx = (ioat->tail+i) & (new_size-1);
+ current_idx = (ioat->tail + i) & (current_size - 1);
+ new_idx = (ioat->tail + i) & (new_size - 1);
- ring[new_idx] = ioat->ring[curr_idx];
+ ring[new_idx] = ioat->ring[current_idx];
ring[new_idx]->id = new_idx;
}
/* free deleted descriptors */
for (i = new_size; i < current_size; i++) {
- ent = ioat_get_ring_entry(ioat, ioat->tail+i);
+ ent = ioat_get_ring_entry(ioat, ioat->tail + i);
ioat_free_ring_entry(ioat, ent);
}
/* fix up hardware ring */
- hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->u.dma;
- next = ring[(ioat->tail+new_size) & (new_size-1)];
+ hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
+ next = ring[(ioat->tail + new_size) & (new_size - 1)];
hw->next = next->hw_desc_bus_addr;
}
@@ -803,28 +848,29 @@
static void
ioat_timer_callback(void *arg)
{
- struct ioat_softc *ioat = arg;
- uint64_t status;
struct ioat_descriptor *desc;
+ struct ioat_softc *ioat;
+ uint64_t status;
uint32_t chanerr;
+ ioat = arg;
ioat_log_message(2, "%s\n", __func__);
if (ioat->is_completion_pending) {
status = ioat_get_chansts(ioat);
/*
- * when halted due to errors check for channel
- * programming errors before advancing the completion state
+ * When halted due to errors, check for channel programming
+ * errors before advancing the completion state.
*/
if (is_ioat_halted(status)) {
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
ioat_log_message(0, "Channel halted (%x)\n", chanerr);
- desc = ioat_get_ring_entry(ioat, ioat->tail+0);
+ desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
dump_descriptor(desc->u.raw);
- desc = ioat_get_ring_entry(ioat, ioat->tail+1);
+ desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
dump_descriptor(desc->u.raw);
}
ioat_process_events(ioat);
@@ -834,17 +880,17 @@
if (ioat_get_active(ioat) == 0 &&
ioat->ring_size_order > IOAT_MIN_ORDER)
- resize_ring(ioat, ioat->ring_size_order-1);
+ resize_ring(ioat, ioat->ring_size_order - 1);
mtx_unlock(&ioat->cleanup_lock);
mtx_unlock(&ioat->submit_lock);
- if (ioat->ring_size_order > IOAT_MIN_ORDER) {
- callout_reset(&ioat->timer, 5000*hz/1000,
+ if (ioat->ring_size_order > IOAT_MIN_ORDER)
+ callout_reset(&ioat->timer, 5 * hz,
ioat_timer_callback, ioat);
- }
}
}
+
/*
* Support Functions
*/
@@ -856,28 +902,30 @@
if (!ioat->is_completion_pending) {
ioat->is_completion_pending = TRUE;
- callout_reset(&ioat->timer, 10000*hz/1000, ioat_timer_callback,
+ callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback,
ioat);
}
}
-static int ioat_reset_hw(struct ioat_softc *ioat)
+static int
+ioat_reset_hw(struct ioat_softc *ioat)
{
- int timeout = 20; /* in milliseconds */
uint64_t status;
uint32_t chanerr;
+ int timeout;
status = ioat_get_chansts(ioat);
if (is_ioat_active(status) || is_ioat_idle(status))
ioat_suspend(ioat);
- while (is_ioat_active(status) || is_ioat_idle(status)) {
+ /* Wait at most 20 ms */
+ for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
+ timeout < 20; timeout++) {
DELAY(1000);
- timeout--;
- if (timeout == 0)
- return (-1);
status = ioat_get_chansts(ioat);
}
+ if (timeout == 20)
+ return (ETIMEDOUT);
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
@@ -893,14 +941,11 @@
ioat_reset(ioat);
- timeout = 20;
-
- while (ioat_reset_pending(ioat)) {
+ /* Wait at most 20 ms */
+ for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
DELAY(1000);
- timeout--;
- if (timeout == 0)
- return (-1);
- }
+ if (timeout == 20)
+ return (ETIMEDOUT);
return (0);
}
@@ -912,7 +957,7 @@
for (i = 0; i < 2; i++) {
for (j = 0; j < 8; j++)
- printf("%08x ", ((uint32_t *)hw_desc)[i*8+j]);
+ printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
printf("\n");
}
}
@@ -920,19 +965,39 @@
static void
ioat_setup_sysctl(device_t device)
{
- struct ioat_softc *ioat = DEVICE2SOFTC(device);
- struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(device);
- struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(device);
+ struct sysctl_ctx_list *sysctl_ctx;
+ struct sysctl_oid *sysctl_tree;
+ struct ioat_softc *ioat;
+
+ ioat = DEVICE2SOFTC(device);
+ sysctl_ctx = device_get_sysctl_ctx(device);
+ sysctl_tree = device_get_sysctl_tree(device);
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
"ring_size_order", CTLFLAG_RD, &ioat->ring_size_order,
0, "HW descriptor ring size order");
-
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
"head", CTLFLAG_RD, &ioat->head,
0, "HW descriptor head pointer index");
-
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
"tail", CTLFLAG_RD, &ioat->tail,
0, "HW descriptor tail pointer index");
}
+
+void
+ioat_log_message(int verbosity, char *fmt, ...)
+{
+ va_list argp;
+ char buffer[512];
+ struct timeval tv;
+
+ if (verbosity > g_ioat_debug_level)
+ return;
+
+ va_start(argp, fmt);
+ vsnprintf(buffer, sizeof(buffer) - 1, fmt, argp);
+ va_end(argp);
+ microuptime(&tv);
+
+ printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
+}
Index: sys/dev/ioat/ioat_hw.h
===================================================================
--- /dev/null
+++ sys/dev/ioat/ioat_hw.h
@@ -0,0 +1,102 @@
+/*-
+ * Copyright (C) 2012 Intel Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __IOAT_HW_H__
+#define __IOAT_HW_H__
+
+#define IOAT_MAX_CHANNELS 32
+
+#define IOAT_CHANCNT_OFFSET 0x00
+
+#define IOAT_XFERCAP_OFFSET 0x01
+
+#define IOAT_GENCTRL_OFFSET 0x02
+
+#define IOAT_INTRCTRL_OFFSET 0x03
+#define IOAT_INTRCTRL_MASTER_INT_EN 0x01
+
+#define IOAT_ATTNSTATUS_OFFSET 0x04
+
+#define IOAT_CBVER_OFFSET 0x08
+
+#define IOAT_VER_3_0 0x30
+#define IOAT_VER_3_3 0x33
+
+#define IOAT_INTRDELAY_OFFSET 0x0C
+
+#define IOAT_CS_STATUS_OFFSET 0x0E
+
+#define IOAT_DMACAPABILITY_OFFSET 0x10
+
+/* DMA Channel Registers */
+#define IOAT_CHANCTRL_OFFSET 0x80
+#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000
+#define IOAT_CHANCTRL_COMPL_DCA_EN 0x0200
+#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100
+#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020
+#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
+#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN 0x0008
+#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
+#define IOAT_CHANCTRL_INT_REARM 0x0001
+#define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\
+ IOAT_CHANCTRL_ANY_ERR_ABORT_EN)
+
+#define IOAT_CHANCMD_OFFSET 0x84
+#define IOAT_CHANCMD_RESET 0x20
+#define IOAT_CHANCMD_SUSPEND 0x04
+
+#define IOAT_DMACOUNT_OFFSET 0x86
+
+#define IOAT_CHANSTS_OFFSET_LOW 0x88
+#define IOAT_CHANSTS_OFFSET_HIGH 0x8C
+#define IOAT_CHANSTS_OFFSET 0x88
+
+#define IOAT_CHANSTS_STATUS 0x7ULL
+#define IOAT_CHANSTS_ACTIVE 0x0
+#define IOAT_CHANSTS_IDLE 0x1
+#define IOAT_CHANSTS_SUSPENDED 0x2
+#define IOAT_CHANSTS_HALTED 0x3
+
+#define IOAT_CHANSTS_UNAFFILIATED_ERROR 0x8ULL
+#define IOAT_CHANSTS_SOFT_ERROR 0x10ULL
+
+#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK (~0x3FULL)
+
+#define IOAT_CHAINADDR_OFFSET_LOW 0x90
+#define IOAT_CHAINADDR_OFFSET_HIGH 0x94
+
+#define IOAT_CHANCMP_OFFSET_LOW 0x98
+#define IOAT_CHANCMP_OFFSET_HIGH 0x9C
+
+#define IOAT_CHANERR_OFFSET 0xA8
+
+#define IOAT_CFG_CHANERR_INT_OFFSET 0x180
+#define IOAT_CFG_CHANERRMASK_INT_OFFSET 0x184
+
+#define IOAT_MIN_ORDER 4
+#define IOAT_MAX_ORDER 16
+
+#endif /* __IOAT_HW_H__ */
Index: sys/dev/ioat/ioat_internal.h
===================================================================
--- sys/dev/ioat/ioat_internal.h
+++ sys/dev/ioat/ioat_internal.h
@@ -29,95 +29,24 @@
#define DEVICE2SOFTC(dev) ((struct ioat_softc *) device_get_softc(dev))
-#define IOAT_MAX_CHANNELS 32
-
-#define IOAT_CHANCNT_OFFSET 0x00
-
-#define ioat_read_chancnt(ioat) \
+#define ioat_read_chancnt(ioat) \
ioat_read_1((ioat), IOAT_CHANCNT_OFFSET)
-#define IOAT_XFERCAP_OFFSET 0x01
-
-#define ioat_read_xfercap(ioat) \
+#define ioat_read_xfercap(ioat) \
ioat_read_1((ioat), IOAT_XFERCAP_OFFSET)
-#define IOAT_GENCTRL_OFFSET 0x02
-
-#define IOAT_INTRCTRL_OFFSET 0x03
-#define IOAT_INTRCTRL_MASTER_INT_EN 0x01
-
-#define ioat_write_intrctrl(ioat, value) \
+#define ioat_write_intrctrl(ioat, value) \
ioat_write_1((ioat), IOAT_INTRCTRL_OFFSET, (value))
-#define IOAT_ATTNSTATUS_OFFSET 0x04
-
-#define IOAT_CBVER_OFFSET 0x08
-
-#define ioat_read_cbver(ioat) \
+#define ioat_read_cbver(ioat) \
(ioat_read_1((ioat), IOAT_CBVER_OFFSET) & 0xFF)
-#define IOAT_VER_3_0 0x30
-#define IOAT_VER_3_3 0x33
-
-#define IOAT_INTRDELAY_OFFSET 0x0C
-
-#define IOAT_CS_STATUS_OFFSET 0x0E
-
-#define IOAT_DMACAPABILITY_OFFSET 0x10
-#define ioat_read_dmacapability(ioat) \
+#define ioat_read_dmacapability(ioat) \
ioat_read_4((ioat), IOAT_DMACAPABILITY_OFFSET)
-/* DMA Channel Registers */
-#define IOAT_CHANCTRL_OFFSET 0x80
-#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000
-#define IOAT_CHANCTRL_COMPL_DCA_EN 0x0200
-#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100
-#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020
-#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
-#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN 0x0008
-#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
-#define IOAT_CHANCTRL_INT_REARM 0x0001
-#define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\
- IOAT_CHANCTRL_ANY_ERR_ABORT_EN)
-
-#define ioat_write_chanctrl(ioat, value) \
+#define ioat_write_chanctrl(ioat, value) \
ioat_write_2((ioat), IOAT_CHANCTRL_OFFSET, (value))
-#define IOAT_CHANCMD_OFFSET 0x84
-#define IOAT_CHANCMD_RESET 0x20
-#define IOAT_CHANCMD_SUSPEND 0x04
-
-#define IOAT_DMACOUNT_OFFSET 0x86
-
-#define IOAT_CHANSTS_OFFSET_LOW 0x88
-#define IOAT_CHANSTS_OFFSET_HIGH 0x8C
-#define IOAT_CHANSTS_OFFSET 0x88
-
-#define IOAT_CHANSTS_STATUS 0x7ULL
-#define IOAT_CHANSTS_ACTIVE 0x0
-#define IOAT_CHANSTS_IDLE 0x1
-#define IOAT_CHANSTS_SUSPENDED 0x2
-#define IOAT_CHANSTS_HALTED 0x3
-
-#define IOAT_CHANSTS_UNAFFILIATED_ERROR 0x8ULL
-#define IOAT_CHANSTS_SOFT_ERROR 0x10ULL
-
-#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK (~0x3FULL)
-
-#define IOAT_CHAINADDR_OFFSET_LOW 0x90
-#define IOAT_CHAINADDR_OFFSET_HIGH 0x94
-
-#define IOAT_CHANCMP_OFFSET_LOW 0x98
-#define IOAT_CHANCMP_OFFSET_HIGH 0x9C
-
-#define IOAT_CHANERR_OFFSET 0xA8
-
-#define IOAT_CFG_CHANERR_INT_OFFSET 0x180
-#define IOAT_CFG_CHANERRMASK_INT_OFFSET 0x184
-
-#define IOAT_MIN_ORDER 4
-#define IOAT_MAX_ORDER 16
-
static __inline uint64_t
ioat_bus_space_read_8_lower_first(bus_space_tag_t tag,
bus_space_handle_t handle, bus_size_t offset)
@@ -186,6 +115,10 @@
MALLOC_DECLARE(M_IOAT);
+SYSCTL_DECL(_hw_ioat);
+
+void ioat_log_message(int verbosity, char *fmt, ...);
+
struct ioat_dma_hw_descriptor {
uint32_t size;
union {
@@ -380,57 +313,57 @@
/* One of these per allocated PCI device. */
struct ioat_softc {
-
- bus_dmaengine_t dmaengine;
- int version;
-
- struct mtx submit_lock;
- int num_interrupts;
- device_t device;
- bus_space_tag_t pci_bus_tag;
- bus_space_handle_t pci_bus_handle;
- int pci_resource_id;
- struct resource *pci_resource;
- uint32_t max_xfer_size;
-
- struct resource * res;
- int rid;
- void * tag;
-
- bus_dma_tag_t hw_desc_tag;
- bus_dmamap_t hw_desc_map;
-
- bus_dma_tag_t comp_update_tag;
- bus_dmamap_t comp_update_map;
- uint64_t *comp_update;
- bus_addr_t comp_update_bus_addr;
-
- struct callout timer;
-
- boolean_t is_resize_pending;
- boolean_t is_completion_pending;
- boolean_t is_reset_pending;
- boolean_t is_channel_running;
- boolean_t is_waiting_for_ack;
-
- uint32_t xfercap_log;
- uint32_t head;
- uint32_t tail;
- uint16_t reserved;
- uint32_t ring_size_order;
- bus_addr_t last_seen;
-
- struct ioat_descriptor **ring;
-
- struct mtx cleanup_lock;
+ bus_dmaengine_t dmaengine;
+#define to_ioat_softc(_dmaeng) \
+({ \
+ bus_dmaengine_t *_p = (_dmaeng); \
+ (struct ioat_softc *)((char *)_p - \
+ offsetof(struct ioat_softc, dmaengine)); \
+})
+
+ int version;
+
+ struct mtx submit_lock;
+ int num_interrupts;
+ device_t device;
+ bus_space_tag_t pci_bus_tag;
+ bus_space_handle_t pci_bus_handle;
+ int pci_resource_id;
+ struct resource *pci_resource;
+ uint32_t max_xfer_size;
+
+ struct resource *res;
+ int rid;
+ void *tag;
+
+ bus_dma_tag_t hw_desc_tag;
+ bus_dmamap_t hw_desc_map;
+
+ bus_dma_tag_t comp_update_tag;
+ bus_dmamap_t comp_update_map;
+ uint64_t *comp_update;
+ bus_addr_t comp_update_bus_addr;
+
+ struct callout timer;
+
+ boolean_t is_resize_pending;
+ boolean_t is_completion_pending;
+ boolean_t is_reset_pending;
+ boolean_t is_channel_running;
+ boolean_t is_waiting_for_ack;
+
+ uint32_t xfercap_log;
+ uint32_t head;
+ uint32_t tail;
+ uint16_t reserved;
+ uint32_t ring_size_order;
+ bus_addr_t last_seen;
+
+ struct ioat_descriptor **ring;
+
+ struct mtx cleanup_lock;
};
-#ifdef IOAT_LOGGING
-void ioat_log_message(int verbosity, char *log_message, ...);
-#else
-#define ioat_log_message(x...)
-#endif
-
static inline uint64_t
ioat_get_chansts(struct ioat_softc *ioat)
{
@@ -438,10 +371,9 @@
if (ioat->version >= IOAT_VER_3_3)
status = ioat_read_8(ioat, IOAT_CHANSTS_OFFSET);
- else {
+ else
/* Must read lower 4 bytes before upper 4 bytes. */
status = ioat_read_double_4(ioat, IOAT_CHANSTS_OFFSET);
- }
return (status);
}
@@ -507,7 +439,7 @@
uint8_t cmd;
cmd = ioat_read_1(ioat, IOAT_CHANCMD_OFFSET);
- return ((cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET);
+ return ((cmd & IOAT_CHANCMD_RESET) != 0);
}
#endif /* __IOAT_INTERNAL_H__ */
Index: sys/dev/ioat/ioat_logger.h
===================================================================
--- sys/dev/ioat/ioat_logger.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*-
- * Copyright (C) 2012 Intel Corporation
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef __IOAT_LOGGER_H__
-#define __IOAT_LOGGER_H__
-
-#ifdef IOAT_LOGGING
-void ioat_log_message(int verbosity, char *log_message, ...);
-#else
-#define ioat_log_message(x...)
-#endif
-
-extern int g_ioat_debug_level;
-
-#endif /* __IOAT_LOGGER_H__ */
Index: sys/dev/ioat/ioat_logger.c
===================================================================
--- sys/dev/ioat/ioat_logger.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (C) 2012 Intel Corporation
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/time.h>
-#include <machine/stdarg.h>
-
-#include "ioat_logger.h"
-
-int g_ioat_debug_level = 0;
-
-#ifdef IOAT_LOGGING
-void
-ioat_log_message(int verbosity, char *log_message, ...)
-{
- va_list argp;
- char buffer[512];
- struct timeval tv;
-
- if (verbosity > g_ioat_debug_level)
- return;
-
- va_start (argp, log_message);
- vsnprintf(buffer, sizeof(buffer)-1, log_message, argp);
- va_end(argp);
- microuptime(&tv);
-
- printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
-}
-#endif
Index: sys/dev/ioat/ioat_test.h
===================================================================
--- sys/dev/ioat/ioat_test.h
+++ sys/dev/ioat/ioat_test.h
@@ -28,18 +28,17 @@
#define __IOAT_TEST_H__
struct ioat_test {
-
uint32_t channel_index;
uint32_t num_loops;
volatile uint32_t num_completions;
uint32_t status;
};
-#define IOAT_TEST_OK 0
-#define IOAT_TEST_NO_DMA_ENGINE 1
-#define IOAT_TEST_NO_MEMORY 2
-#define IOAT_TEST_MISCOMPARE 3
+#define IOAT_TEST_OK 0
+#define IOAT_TEST_NO_DMA_ENGINE 1
+#define IOAT_TEST_NO_MEMORY 2
+#define IOAT_TEST_MISCOMPARE 3
-#define IOAT_DMATEST _IOWR('i', 0, struct ioat_test)
+#define IOAT_DMATEST _IOWR('i', 0, struct ioat_test)
#endif /* __IOAT_TEST_H__ */
Index: sys/dev/ioat/ioat_test.c
===================================================================
--- sys/dev/ioat/ioat_test.c
+++ sys/dev/ioat/ioat_test.c
@@ -47,23 +47,25 @@
#include <vm/pmap.h>
#include "ioat.h"
+#include "ioat_hw.h"
+#include "ioat_internal.h"
#include "ioat_test.h"
-#include "ioat_logger.h"
MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
-static struct cdev *ioat_cdev = NULL;
-
-#define IOAT_TEST_SIZE 0x40000
-#define IOAT_MAX_BUFS 8
+#define IOAT_TEST_SIZE 0x40000
+#define IOAT_MAX_BUFS 8
struct test_transaction {
uint8_t num_buffers;
void *buf[IOAT_MAX_BUFS];
uint32_t length;
- struct ioat_test *test;
+ struct ioat_test *test;
};
+static int g_thread_index = 1;
+static struct cdev *g_ioat_cdev = NULL;
+
static void
ioat_test_transaction_destroy(struct test_transaction *tx)
{
@@ -83,11 +85,10 @@
test_transaction *ioat_test_transaction_create(uint8_t num_buffers,
uint32_t buffer_size)
{
- struct test_transaction *tx;
+ struct test_transaction *tx;
int i;
tx = malloc(sizeof(struct test_transaction), M_IOAT_TEST, M_NOWAIT | M_ZERO);
-
if (tx == NULL)
return (NULL);
@@ -95,7 +96,6 @@
tx->length = buffer_size;
for (i = 0; i < num_buffers; i++) {
-
tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT,
0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
@@ -104,15 +104,17 @@
return (NULL);
}
}
-
return (tx);
}
static void
ioat_dma_test_callback(void *arg)
{
- struct test_transaction *tx = arg;
- struct ioat_test *test = tx->test;
+ struct test_transaction *tx;
+ struct ioat_test *test;
+
+ tx = arg;
+ test = tx->test;
if (memcmp(tx->buf[0], tx->buf[1], tx->length) != 0) {
ioat_log_message(0, "miscompare found\n");
@@ -124,16 +126,17 @@
wakeup(test);
}
-int g_thread_index = 1;
-
static void
ioat_dma_test(void *arg)
{
- struct ioat_test *test = arg;
struct test_transaction *tx;
+ struct ioat_test *test;
bus_dmaengine_t dmaengine;
+ uint32_t loops;
int index, i;
- uint32_t loops = test->num_loops;
+
+ test = arg;
+ loops = test->num_loops;
test->status = IOAT_TEST_OK;
test->num_completions = 0;
@@ -148,18 +151,18 @@
}
ioat_log_message(0, "Thread %d: num_loops remaining: 0x%07x\n", index,
- loops);
+ test->num_loops);
- while (loops-- > 0) {
+ for (loops = 0; loops < test->num_loops; loops++) {
bus_addr_t src, dest;
if (loops % 0x10000 == 0) {
ioat_log_message(0, "Thread %d: "
- "num_loops remaining: 0x%07x\n", index, loops);
+ "num_loops remaining: 0x%07x\n", index,
+ test->num_loops - loops);
}
tx = ioat_test_transaction_create(2, IOAT_TEST_SIZE);
-
if (tx == NULL) {
ioat_log_message(0, "tx == NULL - memory exhausted\n");
atomic_add_32(&test->num_completions, 1);
@@ -186,8 +189,8 @@
ioat_release(dmaengine);
}
- while(test->num_completions < test->num_loops)
- tsleep(test, 0, "compl", 5*hz);
+ while (test->num_completions < test->num_loops)
+ tsleep(test, 0, "compl", 5 * hz);
}
@@ -207,15 +210,16 @@
static int
ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag,
- struct thread *td)
+ struct thread *td)
{
+
switch (cmd) {
case IOAT_DMATEST:
ioat_dma_test(arg);
break;
+ default:
+ return (EINVAL);
}
-
-
return (0);
}
@@ -228,22 +232,25 @@
.d_name = "ioat_test",
};
-static void
-ioat_test_init(void *arg)
-{
- ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_OPERATOR,
- 0600, "ioat_test");
-}
-
-static void
-ioat_test_uninit(void *arg)
+static int
+sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)
{
- if (ioat_cdev != NULL) {
- destroy_dev(ioat_cdev);
- ioat_cdev = NULL;
+ int error, enabled;
+
+ enabled = (g_ioat_cdev != NULL);
+ error = sysctl_handle_int(oidp, &enabled, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+
+ if (enabled != 0 && g_ioat_cdev == NULL) {
+ g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL,
+ 0600, "ioat_test");
+ } else if (enabled == 0 && g_ioat_cdev != NULL) {
+ destroy_dev(g_ioat_cdev);
+ g_ioat_cdev = NULL;
}
+ return (0);
}
-
-SYSINIT(ioat_test_load, SI_SUB_DRIVERS, SI_ORDER_FIRST, ioat_test_init, NULL);
-SYSUNINIT(ioat_test_unload, SI_SUB_DRIVERS, SI_ORDER_FIRST, ioat_test_uninit, NULL);
-
+SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_STRING | CTLFLAG_RW,
+ 0, 0, sysctl_enable_ioat_test, "A",
+ "Non-zero: Enable the /dev/ioat_test device");
Index: sys/modules/ioat/Makefile
===================================================================
--- sys/modules/ioat/Makefile
+++ sys/modules/ioat/Makefile
@@ -6,16 +6,10 @@
.PATH: ${IOAT_SRC_PATH}/dev/ioat
-KMOD = ioat
-SRCS = ioat.c ioat_logger.c
-#SRCS += ioat_test.c
+KMOD= ioat
+SRCS= ioat.c ioat_test.c
+SRCS+= device_if.h bus_if.h pci_if.h
-SRCS += device_if.h bus_if.h pci_if.h
+CFLAGS+= -I${IOAT_SRC_PATH}
.include <bsd.kmod.mk>
-
-DEBUG_FLAGS = -g
-#CFLAGS += -O0
-CFLAGS += -I$(IOAT_SRC_PATH)
-CFLAGS += -DIOAT_LOGGING
-#CFLAGS += -DINVARIANTS
Index: sys/modules/ioat/README
===================================================================
--- sys/modules/ioat/README
+++ sys/modules/ioat/README
@@ -5,30 +5,28 @@
to insert descriptors which only trigger an interrupt.
* Note that copy operation takes bus addresses as parameters, not virtual
addresses.
-* Copies larger than max transfer size (1MB) are not supported, and will
- panic the system explicitly. Future versions will likely support this by
- breaking up the transfer into smaller sizes.
-* There are 8 CB-DMA channels per CPU package. Each channel will be
- represented by a struct ioat_softc data structure.
-* There is an ioat_test.c module in the driver, and an ioatcontrol.c
- application that can be used to both test the driver as well as understand
- how to interface with the driver. App uses a /dev/ioat_test device node
- to send IOCTLs which will cause the driver to:
+* Copies larger than max transfer size (1MB) are not supported. Future
+ versions will likely support this by breaking up the transfer into smaller
+ sizes.
+* There are 8 CB-DMA channels per CPU package. Each channel is represented by
+ a struct ioat_softc.
+* There is an ioat_test.c module in the driver which exposes a simple device at
+ /dev/ioat_test. The presense of the test device may be controlled with the
+ hw.ioat.enable_ioat_test tunable. (The default is off.) There is an
+ ioatcontrol.c application in tools/tools/ioat
+ that can be used to test the driver. It sends IOCTLs which will cause the
+ driver to:
* run a loop a number of times as requested by the app
* each loop will allocate two chunks of memory, write data patterns
to each, submit a DMA request to copy one buffer to the other,
and compare the contents in the callback, reporting an error if one
is found
- The ioat_test.c module needs to be enabled in the driver Makefile to use it.
- To build the app from /sys/modules/ioat directory: make ioatcontrol
- To run the app: ./ioatcontrol <channel #> <num_loops>
-* Debug prints are enabled by defining IOAT_LOGGING, which is currently the
- default in the Makefile. Verbosity is 0 by default (only critical errors)
- but can be increased by doing one of the following:
+ To build the test program from tools/tools/ioat: make ioatcontrol
+ To run it: ./ioatcontrol <channel #> <num_loops>
+* Debug verbosity is 0 by default (only critical errors) but can be increased
+ by doing one of the following:
* from command line, before driver is loaded:
kenv hw.ioat.debug_level=3
* add the following to /boot/loader.conf
hw.ioat.debug_level=3
-
-
-
+ * at runtime: sysctl hw.ioat.debug_level=3
Index: tools/tools/ioat/Makefile
===================================================================
--- tools/tools/ioat/Makefile
+++ tools/tools/ioat/Makefile
@@ -1,6 +1,7 @@
-PROG= ioatcontrol
-SRCS= ioatcontrol.c
+PROG= ioatcontrol
+SRCS= ioatcontrol.c
NO_MAN=
-CFLAGS+= -I../../../sys/dev/ioat
+CFLAGS+= -I${.CURDIR:H:H:H}/sys/dev/ioat
+WARNS?= 6
.include <bsd.prog.mk>
Index: tools/tools/ioat/ioatcontrol.c
===================================================================
--- tools/tools/ioat/ioatcontrol.c
+++ tools/tools/ioat/ioatcontrol.c
@@ -25,15 +25,17 @@
*/
#include <sys/ioctl.h>
+
+#include <fcntl.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
-#include <fcntl.h>
+#include <sysexits.h>
#include <unistd.h>
#include "ioat_test.h"
-int
+int
main(int argc, char **argv)
{
struct ioat_test t;
@@ -41,26 +43,24 @@
if (argc < 3) {
printf("Usage: %s <channel #> <num_loops>\n", argv[0]);
- return (-1);
+ return (EX_USAGE);
}
t.channel_index = atoi(argv[1]);
-
if (t.channel_index > 8) {
printf("Channel number must be between 0 and 7.\n");
- return (-1);
+ return (EX_USAGE);
}
t.num_loops = atoi(argv[2]);
fd = open("/dev/ioat_test", O_RDWR);
-
if (fd < 0) {
printf("Cannot open /dev/ioat_test\n");
- return (-1);
+ return (EX_UNAVAILABLE);
}
- ioctl(fd, IOAT_DMATEST, &t);
+ (void)ioctl(fd, IOAT_DMATEST, &t);
close(fd);
return (t.status);

File Metadata

Mime Type
text/plain
Expires
Mon, Feb 23, 6:20 AM (19 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28952081
Default Alt Text
D3457.id8132.diff (50 KB)

Event Timeline