Index: sys/amd64/conf/NOTES
===================================================================
--- sys/amd64/conf/NOTES
+++ sys/amd64/conf/NOTES
@@ -468,6 +468,7 @@
 device		asmc
 #device		si
 device		tpm
+device		ioat		# Intel I/OAT DMA engine
 
 #
 # Laptop/Notebook options:
Index: sys/conf/files.amd64
===================================================================
--- sys/conf/files.amd64
+++ sys/conf/files.amd64
@@ -175,6 +175,8 @@
 dev/if_ndis/if_ndis_pci.c	optional	ndis cardbus | ndis pci
 dev/if_ndis/if_ndis_usb.c	optional	ndis usb
 dev/io/iodev.c			optional	io
+dev/ioat/ioat.c			optional	ioat pci
+dev/ioat/ioat_test.c		optional	ioat pci
 dev/ipmi/ipmi.c			optional	ipmi
 dev/ipmi/ipmi_acpi.c		optional	ipmi acpi
 dev/ipmi/ipmi_isa.c		optional	ipmi isa
Index: sys/dev/ioat/ioat.h
===================================================================
--- sys/dev/ioat/ioat.h
+++ sys/dev/ioat/ioat.h
@@ -34,7 +34,7 @@
  * This file defines the public interface to the IOAT driver.
  */
 
-/* 
+/*
  * Enables an interrupt for this operation. Typically, you would only enable
  * this on the last operation in a group
  */
@@ -44,12 +44,14 @@
 struct bus_dmadesc;
 typedef void (*bus_dmaengine_callback_t)(void *arg);
 
-/* Called first to acquire a reference to the DMA channel */
+/*
+ * Called first to acquire a reference to the DMA channel
+ */
 bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index);
 
-/* 
+/*
  * Acquire must be called before issuing an operation to perform. Release is
- * called after. Multiple operations can be issued within the context of one 
+ * called after. Multiple operations can be issued within the context of one
  * acquire and release
  */
 void ioat_acquire(bus_dmaengine_t dmaengine);
Index: sys/dev/ioat/ioat.c
===================================================================
--- sys/dev/ioat/ioat.c
+++ sys/dev/ioat/ioat.c
@@ -39,14 +39,16 @@
 #include <sys/mutex.h>
 #include <sys/rman.h>
 #include <sys/sysctl.h>
+#include <sys/time.h>
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 #include <machine/bus.h>
 #include <machine/resource.h>
+#include <machine/stdarg.h>
 
 #include "ioat.h"
+#include "ioat_hw.h"
 #include "ioat_internal.h"
-#include "ioat_logger.h"
 
 static int ioat_probe(device_t device);
 static int ioat_attach(device_t device);
@@ -55,7 +57,7 @@
 static int ioat_map_pci_bar(struct ioat_softc *ioat);
 static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
     int error);
-static void ioat_interrupt_setup(struct ioat_softc *ioat);
+static int ioat_interrupt_setup(struct ioat_softc *ioat);
 static void ioat_interrupt_handler(void *arg);
 static void ioat_process_events(struct ioat_softc *ioat);
 static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
@@ -76,8 +78,16 @@
 static void ioat_setup_sysctl(device_t device);
 
 MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
+SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
+
+static int g_force_legacy_interrupts;
+SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
+    &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
+
+static int g_ioat_debug_level = 0;
+SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
+    0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
 
-//#define IOAT_LOGGING
 /*
  * OS <-> Driver interface structures
  */
@@ -101,25 +111,80 @@
 /*
  * Private data structures
  */
-struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
-int ioat_channel_index = 0;
+static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
+static int ioat_channel_index = 0;
+SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
+    "Number of IOAT channels attached");
 
 static struct _pcsid
 {
 	u_int32_t   type;
 	const char  *desc;
 } pci_ids[] = {
-	{ 0x3c208086,   "SNB IOAT Ch0"  },
-	{ 0x3c218086,   "SNB IOAT Ch1"  },
-	{ 0x3c228086,   "SNB IOAT Ch2"  },
-	{ 0x3c238086,   "SNB IOAT Ch3"  },
-	{ 0x3c248086,   "SNB IOAT Ch4"  },
-	{ 0x3c258086,   "SNB IOAT Ch5"  },
-	{ 0x3c268086,   "SNB IOAT Ch6"  },
-	{ 0x3c278086,   "SNB IOAT Ch7"  },
-	{ 0x3c2e8086,   "SNB IOAT Ch0"  },
-	{ 0x3c2f8086,   "SNB IOAT Ch1"  },
-	{ 0x00000000,   NULL            }
+	{ 0x34308086, "TBG IOAT Ch0" },
+	{ 0x34318086, "TBG IOAT Ch1" },
+	{ 0x34328086, "TBG IOAT Ch2" },
+	{ 0x34338086, "TBG IOAT Ch3" },
+	{ 0x34298086, "TBG IOAT Ch4" },
+	{ 0x342a8086, "TBG IOAT Ch5" },
+	{ 0x342b8086, "TBG IOAT Ch6" },
+	{ 0x342c8086, "TBG IOAT Ch7" },
+
+	{ 0x37108086, "JSF IOAT Ch0" },
+	{ 0x37118086, "JSF IOAT Ch1" },
+	{ 0x37128086, "JSF IOAT Ch2" },
+	{ 0x37138086, "JSF IOAT Ch3" },
+	{ 0x37148086, "JSF IOAT Ch4" },
+	{ 0x37158086, "JSF IOAT Ch5" },
+	{ 0x37168086, "JSF IOAT Ch6" },
+	{ 0x37178086, "JSF IOAT Ch7" },
+	{ 0x37188086, "JSF IOAT Ch0 (RAID)" },
+	{ 0x37198086, "JSF IOAT Ch1 (RAID)" },
+
+	{ 0x3c208086, "SNB IOAT Ch0" },
+	{ 0x3c218086, "SNB IOAT Ch1" },
+	{ 0x3c228086, "SNB IOAT Ch2" },
+	{ 0x3c238086, "SNB IOAT Ch3" },
+	{ 0x3c248086, "SNB IOAT Ch4" },
+	{ 0x3c258086, "SNB IOAT Ch5" },
+	{ 0x3c268086, "SNB IOAT Ch6" },
+	{ 0x3c278086, "SNB IOAT Ch7" },
+	{ 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
+	{ 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
+
+	{ 0x0e208086, "IVB IOAT Ch0" },
+	{ 0x0e218086, "IVB IOAT Ch1" },
+	{ 0x0e228086, "IVB IOAT Ch2" },
+	{ 0x0e238086, "IVB IOAT Ch3" },
+	{ 0x0e248086, "IVB IOAT Ch4" },
+	{ 0x0e258086, "IVB IOAT Ch5" },
+	{ 0x0e268086, "IVB IOAT Ch6" },
+	{ 0x0e278086, "IVB IOAT Ch7" },
+	{ 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
+	{ 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
+
+	{ 0x2f208086, "HSW IOAT Ch0" },
+	{ 0x2f218086, "HSW IOAT Ch1" },
+	{ 0x2f228086, "HSW IOAT Ch2" },
+	{ 0x2f238086, "HSW IOAT Ch3" },
+	{ 0x2f248086, "HSW IOAT Ch4" },
+	{ 0x2f258086, "HSW IOAT Ch5" },
+	{ 0x2f268086, "HSW IOAT Ch6" },
+	{ 0x2f278086, "HSW IOAT Ch7" },
+	{ 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
+	{ 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
+
+	{ 0x0c508086, "BWD IOAT Ch0" },
+	{ 0x0c518086, "BWD IOAT Ch1" },
+	{ 0x0c528086, "BWD IOAT Ch2" },
+	{ 0x0c538086, "BWD IOAT Ch3" },
+
+	{ 0x6f508086, "BDXDE IOAT Ch0" },
+	{ 0x6f518086, "BDXDE IOAT Ch1" },
+	{ 0x6f528086, "BDXDE IOAT Ch2" },
+	{ 0x6f538086, "BDXDE IOAT Ch3" },
+
+	{ 0x00000000, NULL           }
 };
 
 /*
@@ -128,15 +193,15 @@
 static int
 ioat_probe(device_t device)
 {
-	u_int32_t type = pci_get_devid(device);
-	struct _pcsid *ep = pci_ids;
+	struct _pcsid *ep;
+	u_int32_t type;
 
-	while (ep->type) {
+	type = pci_get_devid(device);
+	for (ep = pci_ids; ep->type; ep++) {
 		if (ep->type == type) {
 			device_set_desc(device, ep->desc);
 			return (0);
 		}
-		++ep;
 	}
 	return (ENXIO);
 }
@@ -144,54 +209,62 @@
 static int
 ioat_attach(device_t device)
 {
-	int error = 0;
-	struct ioat_softc *ioat = DEVICE2SOFTC(device);
+	struct ioat_softc *ioat;
+	int error;
 
+	ioat = DEVICE2SOFTC(device);
 	ioat->device = device;
 
-	/*
-	 * TODO: this needs to be moved into a driver init function
-	 *  so that it only gets executed once overall, rather than
-	 *  once per channel.  But this works fine for now.
-	 */
-	TUNABLE_INT_FETCH("hw.ioat.debug_level", &g_ioat_debug_level);
+	error = ioat_map_pci_bar(ioat);
+	if (error != 0)
+		goto err;
 
-	ioat_map_pci_bar(ioat);
 	ioat->version = ioat_read_cbver(ioat);
 	ioat_interrupt_setup(ioat);
 
-	if (ioat->version >= IOAT_VER_3_0) {
-		error = ioat3_attach(device);
-	} else {
-		ioat_detach(device);
-		error = -1;
+	if (ioat->version < IOAT_VER_3_0) {
+		error = ENODEV;
+		goto err;
 	}
 
-	if (error == 0)
-		ioat_channel[ioat_channel_index++] = ioat;
+	error = ioat3_attach(device);
+	if (error != 0)
+		goto err;
+
+	error = pci_enable_busmaster(device);
+	if (error != 0)
+		goto err;
+
+	ioat_channel[ioat_channel_index++] = ioat;
 
+err:
+	if (error != 0)
+		ioat_detach(device);
 	return (error);
 }
 
 static int
 ioat_detach(device_t device)
 {
-	struct ioat_softc *ioat = DEVICE2SOFTC(device);
+	struct ioat_softc *ioat;
+	uint32_t i;
+
+	ioat = DEVICE2SOFTC(device);
+	callout_drain(&ioat->timer);
 
-	if (ioat->pci_resource != NULL) {
+	pci_disable_busmaster(device);
+
+	if (ioat->pci_resource != NULL)
 		bus_release_resource(device, SYS_RES_MEMORY,
 		    ioat->pci_resource_id, ioat->pci_resource);
-	}
 
-	if (ioat->ring) {
-		int i;
+	if (ioat->ring != NULL) {
 		for (i = 0; i < (1 << ioat->ring_size_order); i++)
 			ioat_free_ring_entry(ioat, ioat->ring[i]);
-
 		free(ioat->ring, M_IOAT);
 	}
 
-	if (ioat->comp_update) {
+	if (ioat->comp_update != NULL) {
 		bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
 		bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
 		    ioat->comp_update_map);
@@ -200,35 +273,59 @@
 
 	bus_dma_tag_destroy(ioat->hw_desc_tag);
 
-	if(ioat->tag != NULL)
+	if (ioat->tag != NULL)
 		bus_teardown_intr(device, ioat->res, ioat->tag);
 
-	if(ioat->res != NULL)
+	if (ioat->res != NULL)
 		bus_release_resource(device, SYS_RES_IRQ,
 		    rman_get_rid(ioat->res), ioat->res);
 
 	pci_release_msi(device);
-	callout_drain(&ioat->timer);
 
 	return (0);
 }
 
+static int
+ioat3_selftest(struct ioat_softc *ioat)
+{
+	uint64_t status;
+	uint32_t chanerr;
+	int i;
+
+	ioat_acquire(&ioat->dmaengine);
+	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
+	ioat_release(&ioat->dmaengine);
+
+	for (i = 0; i < 100; i++) {
+		DELAY(1);
+		status = ioat_get_chansts(ioat);
+		if (is_ioat_idle(status))
+			return (0);
+	}
+
+	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
+	ioat_log_message(0, "could not start channel: "
+	    "status = %#jx error = %x\n", (uintmax_t)status, chanerr);
+	return (ENXIO);
+}
+
 /*
  * Initialize Hardware
  */
 static int
 ioat3_attach(device_t device)
 {
-	struct ioat_softc *ioat = DEVICE2SOFTC(device);
+	struct ioat_softc *ioat;
 	struct ioat_descriptor **ring;
-	int error = 0;
-	uint8_t xfercap;
-	uint32_t capabilities, chanerr;
-	uint64_t status;
-	int i, num_descriptors;
 	struct ioat_descriptor *next;
 	struct ioat_dma_hw_descriptor *dma_hw_desc;
+	uint32_t capabilities;
+	int i, num_descriptors;
+	int error;
+	uint8_t xfercap;
 
+	error = 0;
+	ioat = DEVICE2SOFTC(device);
 	capabilities = ioat_read_dmacapability(ioat);
 
 	xfercap = ioat_read_xfercap(ioat);
@@ -238,7 +335,7 @@
 	ioat->max_xfer_size = 1 << xfercap;
 
 	/* TODO: need to check DCA here if we ever do XOR/PQ */
-	
+
 	mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
 	mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
 	callout_init(&ioat->timer, CALLOUT_MPSAFE);
@@ -256,15 +353,14 @@
 
 	error = bus_dmamem_alloc(ioat->comp_update_tag,
 	    (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
-
-	if (!ioat->comp_update) {
-		ioat_detach(ioat->device);
-		return (-1);
-	}
+	if (ioat->comp_update == NULL)
+		return (ENOMEM);
 
 	error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
-				ioat->comp_update, sizeof(uint64_t),
-				ioat_comp_update_map, ioat, 0);
+	    ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
+	    0);
+	if (error != 0)
+		return (error);
 
 	ioat->ring_size_order = IOAT_MIN_ORDER;
 
@@ -278,27 +374,20 @@
 
 	ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
 	    M_ZERO | M_NOWAIT);
-
-	if (!ioat->ring) {
-		ioat_detach(ioat->device);
-		return (-1);
-	}
+	if (ioat->ring == NULL)
+		return (ENOMEM);
 
 	ring = ioat->ring;
-
 	for (i = 0; i < num_descriptors; i++) {
 		ring[i] = ioat_alloc_ring_entry(ioat);
-
-		if (!ring[i]) {
-			ioat_detach(ioat->device);
-			return (-1);
-		}
+		if (ring[i] == NULL)
+			return (ENOMEM);
 
 		ring[i]->id = i;
 	}
 
-	for (i = 0; i < num_descriptors-1; i++) {
-		next = ring[i+1];
+	for (i = 0; i < num_descriptors - 1; i++) {
+		next = ring[i + 1];
 		dma_hw_desc = ring[i]->u.dma;
 
 		dma_hw_desc->next = next->hw_desc_bus_addr;
@@ -310,46 +399,20 @@
 	ioat->tail = 0;
 	ioat->last_seen = 0;
 
-	status = ioat_get_chansts(ioat);
-
-	ioat_reset_hw(ioat);
+	error = ioat_reset_hw(ioat);
+	if (error != 0)
+		return (error);
 
 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
 	ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
 	ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
 
-	ioat_acquire(&ioat->dmaengine);
-	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
-	ioat_release(&ioat->dmaengine);
-
-	i = 100;
-	while (i-- > 0) {
-		DELAY(1);
-		status = ioat_get_chansts(ioat);
-		if (is_ioat_idle(status))
-			break;
-	}
-
-	if (is_ioat_idle(status)) {
-		ioat_process_events(ioat);
-	} else {
-		chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
-
-		ioat_log_message(0, "could not start channel: "
-				 "status = %p\n error = %x\n",
-				 (void *)status, chanerr);
-
-		error = -1;
-	}
-
-	if (error)
-	{
-		ioat_detach(device);
+	error = ioat3_selftest(ioat);
+	if (error != 0)
 		return (error);
-	}
 
+	ioat_process_events(ioat);
 	ioat_setup_sysctl(device);
-	
 	return (0);
 }
 
@@ -359,16 +422,15 @@
 
 	ioat->pci_resource_id = PCIR_BAR(0);
 	ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY,
-						&ioat->pci_resource_id, 0, ~0,
-						1, RF_ACTIVE);
+	    &ioat->pci_resource_id, 0, ~0, 1, RF_ACTIVE);
 
-	if(ioat->pci_resource == NULL)
+	if (ioat->pci_resource == NULL) {
 		ioat_log_message(0, "unable to allocate pci resource\n");
-	else {
-		ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
-		ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
+		return (ENODEV);
 	}
 
+	ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
+	ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
 	return (0);
 }
 
@@ -383,58 +445,57 @@
 static void
 ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
+	bus_addr_t *baddr;
 
-	bus_addr_t *baddr = arg;
+	baddr = arg;
 	*baddr = segs->ds_addr;
 }
 
 /*
  * Interrupt setup and handlers
  */
-static void
+static int
 ioat_interrupt_setup(struct ioat_softc *ioat)
 {
-	boolean_t use_msix = 0;
-	boolean_t force_legacy_interrupts = 0;
-
-	TUNABLE_INT_FETCH("hw.ioat.force_legacy_interrupts",
-			  &force_legacy_interrupts);
+	uint32_t num_vectors;
+	int error;
+	boolean_t use_msix;
+	boolean_t force_legacy_interrupts;
 
-	if (!force_legacy_interrupts &&
-	    pci_msix_count(ioat->device) >= 1) {
-		uint32_t num_vectors = 1;
+	use_msix = FALSE;
+	force_legacy_interrupts = FALSE;
 
+	if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
+		num_vectors = 1;
 		pci_alloc_msix(ioat->device, &num_vectors);
 		if (num_vectors == 1)
 			use_msix = TRUE;
 	}
 
-	if (use_msix == TRUE) {
+	if (use_msix) {
 		ioat->rid = 1;
 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
 		    &ioat->rid, RF_ACTIVE);
-	}
-	else {
+	} else {
 		ioat->rid = 0;
 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
 		    &ioat->rid, RF_SHAREABLE | RF_ACTIVE);
 	}
-
 	if (ioat->res == NULL) {
 		ioat_log_message(0, "bus_alloc_resource failed\n");
-		return;
+		return (ENOMEM);
 	}
 
 	ioat->tag = NULL;
-
-	if (bus_setup_intr(ioat->device, ioat->res,
-	    INTR_MPSAFE | INTR_TYPE_MISC, NULL,
-	    ioat_interrupt_handler, ioat, &ioat->tag)) {
+	error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
+	    INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
+	if (error != 0) {
 		ioat_log_message(0, "bus_setup_intr failed\n");
-		return;
+		return (error);
 	}
 
 	ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
+	return (0);
 }
 
 static void
@@ -449,9 +510,9 @@
 ioat_process_events(struct ioat_softc *ioat)
 {
 	struct ioat_descriptor *desc;
+	struct bus_dmadesc *dmadesc;
 	uint64_t comp_update, status;
 	uint32_t completed;
-	struct bus_dmadesc *dmadesc;
 
 	mtx_lock(&ioat->cleanup_lock);
 
@@ -469,30 +530,24 @@
 	while (1) {
 		desc = ioat_get_ring_entry(ioat, ioat->tail);
 		dmadesc = &desc->bus_dmadesc;
-
 		ioat_log_message(3, "completing desc %d\n", ioat->tail);
 
-		if (dmadesc->callback_fn) {
+		if (dmadesc->callback_fn)
 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
-		}
 
 		ioat->tail++;
-
 		if (desc->hw_desc_bus_addr == status)
 			break;
-
 	}
 
 	ioat->last_seen = desc->hw_desc_bus_addr;
 
 	if (ioat->head == ioat->tail) {
 		ioat->is_completion_pending = FALSE;
-		callout_reset(&ioat->timer, 5000*hz/1000, ioat_timer_callback,
-		    ioat);
+		callout_reset(&ioat->timer, 5 * hz, ioat_timer_callback, ioat);
 	}
 
 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
-
 	mtx_unlock(&ioat->cleanup_lock);
 }
 
@@ -505,7 +560,6 @@
 
 	if (index < ioat_channel_index)
 		return (&ioat_channel[index]->dmaengine);
-
 	return (NULL);
 }
 
@@ -514,7 +568,7 @@
 {
 	struct ioat_softc *ioat;
 
-	ioat = (struct ioat_softc *)dmaengine;
+	ioat = to_ioat_softc(dmaengine);
 	mtx_lock(&ioat->submit_lock);
 	ioat_log_message(3, "%s\n", __func__);
 }
@@ -525,7 +579,7 @@
 	struct ioat_softc *ioat;
 
 	ioat_log_message(3, "%s\n", __func__);
-	ioat = (struct ioat_softc *)dmaengine;
+	ioat = to_ioat_softc(dmaengine);
 	ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
 	mtx_unlock(&ioat->submit_lock);
 }
@@ -538,7 +592,7 @@
 	struct ioat_descriptor *desc;
 	struct ioat_dma_hw_descriptor *hw_desc;
 
-	ioat = (struct ioat_softc *)dmaengine;
+	ioat = to_ioat_softc(dmaengine);
 
 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
 		return (NULL);
@@ -552,7 +606,7 @@
 	hw_desc->u.control.null = 1;
 	hw_desc->u.control.completion_update = 1;
 
-	if (flags & DMA_INT_EN)
+	if ((flags & DMA_INT_EN) != 0)
 		hw_desc->u.control.int_enable = 1;
 
 	hw_desc->size = 8;
@@ -563,7 +617,6 @@
 	desc->bus_dmadesc.callback_arg = callback_arg;
 
 	ioat_submit_single(ioat);
-
 	return (&desc->bus_dmadesc);
 }
 
@@ -576,11 +629,13 @@
 	struct ioat_dma_hw_descriptor *hw_desc;
 	struct ioat_softc *ioat;
 
-	ioat = (struct ioat_softc *)dmaengine;
+	ioat = to_ioat_softc(dmaengine);
 
-	if (len > ioat->max_xfer_size)
-		panic("ioat_copy: max_xfer_size = %d, requested = %d\n",
-		    ioat->max_xfer_size, (int)len);
+	if (len > ioat->max_xfer_size) {
+		ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
+		    __func__, ioat->max_xfer_size, (int)len);
+		return (NULL);
+	}
 
 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
 		return (NULL);
@@ -593,23 +648,20 @@
 	hw_desc->u.control_raw = 0;
 	hw_desc->u.control.completion_update = 1;
 
-	if (flags & DMA_INT_EN)
+	if ((flags & DMA_INT_EN) != 0)
 		hw_desc->u.control.int_enable = 1;
 
 	hw_desc->size = len;
 	hw_desc->src_addr = src;
 	hw_desc->dest_addr = dst;
 
-#ifdef IOAT_LOGGING
 	if (g_ioat_debug_level >= 3)
 		dump_descriptor(hw_desc);
-#endif
 
 	desc->bus_dmadesc.callback_fn = callback_fn;
 	desc->bus_dmadesc.callback_arg = callback_arg;
 
 	ioat_submit_single(ioat);
-
 	return (&desc->bus_dmadesc);
 }
 
@@ -633,17 +685,15 @@
 static struct ioat_descriptor *
 ioat_alloc_ring_entry(struct ioat_softc *ioat)
 {
-	struct ioat_descriptor          *desc;
-	struct ioat_dma_hw_descriptor	*hw_desc;
+	struct ioat_dma_hw_descriptor *hw_desc;
+	struct ioat_descriptor *desc;
 
 	desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT);
-
 	if (desc == NULL)
 		return (NULL);
 
 	bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO,
 	    &ioat->hw_desc_map);
-
 	if (hw_desc == NULL) {
 		free(desc, M_IOAT);
 		return (NULL);
@@ -653,7 +703,6 @@
 	    sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0);
 
 	desc->u.dma = hw_desc;
-
 	return (desc);
 }
 
@@ -661,34 +710,31 @@
 ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
 {
 
-	if (desc != NULL) {
-		if (desc->u.dma)
-			bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
-			    ioat->hw_desc_map);
-		free(desc, M_IOAT);
-	}
+	if (desc == NULL)
+		return;
+
+	if (desc->u.dma)
+		bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
+		    ioat->hw_desc_map);
+	free(desc, M_IOAT);
 }
 
 static int
 ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
 {
-	int retry;
+	boolean_t retry;
 
-	do {
-		if (ioat_get_ring_space(ioat) >= num_descs) {
+	while (1) {
+		if (ioat_get_ring_space(ioat) >= num_descs)
 			return (0);
-		}
 
 		mtx_lock(&ioat->cleanup_lock);
 		retry = resize_ring(ioat, ioat->ring_size_order + 1);
 		mtx_unlock(&ioat->cleanup_lock);
 
-		if (retry)
-			continue;
-		else {
+		if (!retry)
 			return (ENOMEM);
-		}
-	} while (1);
+	}
 }
 
 static struct ioat_descriptor *
@@ -724,19 +770,18 @@
 
 	/* allocate the array to hold the software ring */
 	ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
-
-	if (!ring)
+	if (ring == NULL)
 		return (FALSE);
 
-	ioat_log_message(1, "ring resize: new: %d old: %d\n",
+	ioat_log_message(2, "ring resize: new: %d old: %d\n",
 	    new_size, current_size);
 
 	/* allocate/trim descriptors as needed */
 	if (new_size > current_size) {
 		/* copy current descriptors to the new ring */
 		for (i = 0; i < current_size; i++) {
-			current_idx = (ioat->tail+i) & (current_size-1);
-			new_idx = (ioat->tail+i) & (new_size-1);
+			current_idx = (ioat->tail + i) & (current_size - 1);
+			new_idx = (ioat->tail + i) & (new_size - 1);
 
 			ring[new_idx] = ioat->ring[current_idx];
 			ring[new_idx]->id = new_idx;
@@ -744,15 +789,15 @@
 
 		/* add new descriptors to the ring */
 		for (i = current_size; i < new_size; i++) {
-			new_idx = (ioat->tail+i) & (new_size-1);
+			new_idx = (ioat->tail + i) & (new_size - 1);
 
 			ring[new_idx] = ioat_alloc_ring_entry(ioat);
 			if (!ring[new_idx]) {
 				while (i--) {
-					new_idx2 = (ioat->tail+i) &
-					    (new_size-1);
+					new_idx2 = (ioat->tail + i) &
+					    (new_size - 1);
 
-					ioat_free_ring_entry(ioat, 
+					ioat_free_ring_entry(ioat,
 					    ring[new_idx2]);
 				}
 				free(ring, M_IOAT);
@@ -761,9 +806,9 @@
 			ring[new_idx]->id = new_idx;
 		}
 
-		for (i = current_size-1; i < new_size; i++) {
-			new_idx = (ioat->tail+i) & (new_size-1);
-			next = ring[(new_idx+1) & (new_size-1)];
+		for (i = current_size - 1; i < new_size; i++) {
+			new_idx = (ioat->tail + i) & (new_size - 1);
+			next = ring[(new_idx + 1) & (new_size - 1)];
 			hw = ring[new_idx]->u.dma;
 
 			hw->next = next->hw_desc_bus_addr;
@@ -774,22 +819,22 @@
 		 * removed descriptors
 		 */
 		for (i = 0; i < new_size; i++) {
-			uint32_t curr_idx = (ioat->tail+i) & (current_size-1);
-			uint32_t new_idx = (ioat->tail+i) & (new_size-1);
+			current_idx = (ioat->tail + i) & (current_size - 1);
+			new_idx = (ioat->tail + i) & (new_size - 1);
 
-			ring[new_idx] = ioat->ring[curr_idx];
+			ring[new_idx] = ioat->ring[current_idx];
 			ring[new_idx]->id = new_idx;
 		}
 
 		/* free deleted descriptors */
 		for (i = new_size; i < current_size; i++) {
-			ent = ioat_get_ring_entry(ioat, ioat->tail+i);
+			ent = ioat_get_ring_entry(ioat, ioat->tail + i);
 			ioat_free_ring_entry(ioat, ent);
 		}
 
 		/* fix up hardware ring */
-		hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->u.dma;
-		next = ring[(ioat->tail+new_size) & (new_size-1)];
+		hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
+		next = ring[(ioat->tail + new_size) & (new_size - 1)];
 		hw->next = next->hw_desc_bus_addr;
 	}
 
@@ -803,28 +848,29 @@
 static void
 ioat_timer_callback(void *arg)
 {
-	struct ioat_softc *ioat = arg;
-	uint64_t status;
 	struct ioat_descriptor *desc;
+	struct ioat_softc *ioat;
+	uint64_t status;
 	uint32_t chanerr;
 
+	ioat = arg;
 	ioat_log_message(2, "%s\n", __func__);
 
 	if (ioat->is_completion_pending) {
 		status = ioat_get_chansts(ioat);
 
 		/*
-		 * when halted due to errors check for channel
-		 * programming errors before advancing the completion state
+		 * When halted due to errors, check for channel programming
+		 * errors before advancing the completion state.
 		 */
 		if (is_ioat_halted(status)) {
 			chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
 			ioat_log_message(0, "Channel halted (%x)\n", chanerr);
 
-			desc = ioat_get_ring_entry(ioat, ioat->tail+0);
+			desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
 			dump_descriptor(desc->u.raw);
 
-			desc = ioat_get_ring_entry(ioat, ioat->tail+1);
+			desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
 			dump_descriptor(desc->u.raw);
 		}
 		ioat_process_events(ioat);
@@ -834,17 +880,17 @@
 
 		if (ioat_get_active(ioat) == 0 &&
 		    ioat->ring_size_order > IOAT_MIN_ORDER)
-			resize_ring(ioat, ioat->ring_size_order-1);
+			resize_ring(ioat, ioat->ring_size_order - 1);
 
 		mtx_unlock(&ioat->cleanup_lock);
 		mtx_unlock(&ioat->submit_lock);
 
-		if (ioat->ring_size_order > IOAT_MIN_ORDER) {
-			callout_reset(&ioat->timer, 5000*hz/1000,
+		if (ioat->ring_size_order > IOAT_MIN_ORDER)
+			callout_reset(&ioat->timer, 5 * hz,
 			    ioat_timer_callback, ioat);
-		}
 	}
 }
+
 /*
  * Support Functions
  */
@@ -856,28 +902,30 @@
 
 	if (!ioat->is_completion_pending) {
 		ioat->is_completion_pending = TRUE;
-		callout_reset(&ioat->timer, 10000*hz/1000, ioat_timer_callback,
+		callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback,
 		    ioat);
 	}
 }
 
-static int ioat_reset_hw(struct ioat_softc *ioat)
+static int
+ioat_reset_hw(struct ioat_softc *ioat)
 {
-	int timeout = 20; /* in milliseconds */
 	uint64_t status;
 	uint32_t chanerr;
+	int timeout;
 
 	status = ioat_get_chansts(ioat);
 	if (is_ioat_active(status) || is_ioat_idle(status))
 		ioat_suspend(ioat);
 
-	while (is_ioat_active(status) || is_ioat_idle(status)) {
+	/* Wait at most 20 ms */
+	for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
+	    timeout < 20; timeout++) {
 		DELAY(1000);
-		timeout--;
-		if (timeout == 0)
-			return (-1);
 		status = ioat_get_chansts(ioat);
 	}
+	if (timeout == 20)
+		return (ETIMEDOUT);
 
 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
 	ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
@@ -893,14 +941,11 @@
 
 	ioat_reset(ioat);
 
-	timeout = 20;
-
-	while (ioat_reset_pending(ioat)) {
+	/* Wait at most 20 ms */
+	for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
 		DELAY(1000);
-		timeout--;
-		if (timeout == 0)
-			return (-1);
-	}
+	if (timeout == 20)
+		return (ETIMEDOUT);
 
 	return (0);
 }
@@ -912,7 +957,7 @@
 
 	for (i = 0; i < 2; i++) {
 		for (j = 0; j < 8; j++)
-			printf("%08x ", ((uint32_t *)hw_desc)[i*8+j]);
+			printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
 		printf("\n");
 	}
 }
@@ -920,19 +965,39 @@
 static void
 ioat_setup_sysctl(device_t device)
 {
-	struct ioat_softc *ioat = DEVICE2SOFTC(device);
-	struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(device);
-	struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(device);
+	struct sysctl_ctx_list *sysctl_ctx;
+	struct sysctl_oid *sysctl_tree;
+	struct ioat_softc *ioat;
+
+	ioat = DEVICE2SOFTC(device);
+	sysctl_ctx = device_get_sysctl_ctx(device);
+	sysctl_tree = device_get_sysctl_tree(device);
 
 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
 	    "ring_size_order", CTLFLAG_RD, &ioat->ring_size_order,
 	    0, "HW descriptor ring size order");
-
 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
 	    "head", CTLFLAG_RD, &ioat->head,
 	    0, "HW descriptor head pointer index");
-
 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
 	    "tail", CTLFLAG_RD, &ioat->tail,
 	    0, "HW descriptor tail pointer index");
 }
+
+void
+ioat_log_message(int verbosity, char *fmt, ...)
+{
+	va_list argp;
+	char buffer[512];
+	struct timeval tv;
+
+	if (verbosity > g_ioat_debug_level)
+		return;
+
+	va_start(argp, fmt);
+	vsnprintf(buffer, sizeof(buffer) - 1, fmt, argp);
+	va_end(argp);
+	microuptime(&tv);
+
+	printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
+}
Index: sys/dev/ioat/ioat_hw.h
===================================================================
--- /dev/null
+++ sys/dev/ioat/ioat_hw.h
@@ -0,0 +1,102 @@
+/*-
+ * Copyright (C) 2012 Intel Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __IOAT_HW_H__
+#define __IOAT_HW_H__
+
+#define	IOAT_MAX_CHANNELS		32
+
+#define	IOAT_CHANCNT_OFFSET		0x00
+
+#define	IOAT_XFERCAP_OFFSET		0x01
+
+#define	IOAT_GENCTRL_OFFSET		0x02
+
+#define	IOAT_INTRCTRL_OFFSET		0x03
+#define	IOAT_INTRCTRL_MASTER_INT_EN	0x01
+
+#define	IOAT_ATTNSTATUS_OFFSET		0x04
+
+#define	IOAT_CBVER_OFFSET		0x08
+
+#define	IOAT_VER_3_0			0x30
+#define	IOAT_VER_3_3			0x33
+
+#define	IOAT_INTRDELAY_OFFSET		0x0C
+
+#define	IOAT_CS_STATUS_OFFSET		0x0E
+
+#define	IOAT_DMACAPABILITY_OFFSET	0x10
+
+/* DMA Channel Registers */
+#define	IOAT_CHANCTRL_OFFSET			0x80
+#define	IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK	0xF000
+#define	IOAT_CHANCTRL_COMPL_DCA_EN		0x0200
+#define	IOAT_CHANCTRL_CHANNEL_IN_USE		0x0100
+#define	IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL	0x0020
+#define	IOAT_CHANCTRL_ERR_INT_EN		0x0010
+#define	IOAT_CHANCTRL_ANY_ERR_ABORT_EN		0x0008
+#define	IOAT_CHANCTRL_ERR_COMPLETION_EN		0x0004
+#define	IOAT_CHANCTRL_INT_REARM			0x0001
+#define	IOAT_CHANCTRL_RUN			(IOAT_CHANCTRL_INT_REARM |\
+						 IOAT_CHANCTRL_ANY_ERR_ABORT_EN)
+
+#define	IOAT_CHANCMD_OFFSET		0x84
+#define	IOAT_CHANCMD_RESET		0x20
+#define	IOAT_CHANCMD_SUSPEND		0x04
+
+#define	IOAT_DMACOUNT_OFFSET		0x86
+
+#define	IOAT_CHANSTS_OFFSET_LOW		0x88
+#define	IOAT_CHANSTS_OFFSET_HIGH	0x8C
+#define	IOAT_CHANSTS_OFFSET		0x88
+
+#define	IOAT_CHANSTS_STATUS		0x7ULL
+#define	IOAT_CHANSTS_ACTIVE		0x0
+#define	IOAT_CHANSTS_IDLE		0x1
+#define	IOAT_CHANSTS_SUSPENDED		0x2
+#define	IOAT_CHANSTS_HALTED		0x3
+
+#define	IOAT_CHANSTS_UNAFFILIATED_ERROR	0x8ULL
+#define	IOAT_CHANSTS_SOFT_ERROR		0x10ULL
+
+#define	IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK	(~0x3FULL)
+
+#define	IOAT_CHAINADDR_OFFSET_LOW	0x90
+#define	IOAT_CHAINADDR_OFFSET_HIGH	0x94
+
+#define	IOAT_CHANCMP_OFFSET_LOW		0x98
+#define	IOAT_CHANCMP_OFFSET_HIGH	0x9C
+
+#define	IOAT_CHANERR_OFFSET		0xA8
+
+#define	IOAT_CFG_CHANERR_INT_OFFSET		0x180
+#define	IOAT_CFG_CHANERRMASK_INT_OFFSET		0x184
+
+#define	IOAT_MIN_ORDER			4
+#define	IOAT_MAX_ORDER			16
+
+#endif /* __IOAT_HW_H__ */
Index: sys/dev/ioat/ioat_internal.h
===================================================================
--- sys/dev/ioat/ioat_internal.h
+++ sys/dev/ioat/ioat_internal.h
@@ -29,95 +29,24 @@
 
 #define DEVICE2SOFTC(dev) ((struct ioat_softc *) device_get_softc(dev))
 
-#define IOAT_MAX_CHANNELS	32
-
-#define IOAT_CHANCNT_OFFSET		0x00
-
-#define ioat_read_chancnt(ioat) \
+#define	ioat_read_chancnt(ioat) \
 	ioat_read_1((ioat), IOAT_CHANCNT_OFFSET)
 
-#define IOAT_XFERCAP_OFFSET		0x01
-
-#define ioat_read_xfercap(ioat) \
+#define	ioat_read_xfercap(ioat) \
 	ioat_read_1((ioat), IOAT_XFERCAP_OFFSET)
 
-#define IOAT_GENCTRL_OFFSET		0x02
-
-#define IOAT_INTRCTRL_OFFSET		0x03
-#define IOAT_INTRCTRL_MASTER_INT_EN	0x01
-
-#define ioat_write_intrctrl(ioat, value) \
+#define	ioat_write_intrctrl(ioat, value) \
 	ioat_write_1((ioat), IOAT_INTRCTRL_OFFSET, (value))
 
-#define IOAT_ATTNSTATUS_OFFSET		0x04
-
-#define IOAT_CBVER_OFFSET		0x08
-
-#define ioat_read_cbver(ioat) \
+#define	ioat_read_cbver(ioat) \
 	(ioat_read_1((ioat), IOAT_CBVER_OFFSET) & 0xFF)
-#define IOAT_VER_3_0                0x30
-#define IOAT_VER_3_3                0x33
-
-#define IOAT_INTRDELAY_OFFSET		0x0C
-
-#define IOAT_CS_STATUS_OFFSET		0x0E
-
-#define IOAT_DMACAPABILITY_OFFSET	0x10
 
-#define ioat_read_dmacapability(ioat) \
+#define	ioat_read_dmacapability(ioat) \
 	ioat_read_4((ioat), IOAT_DMACAPABILITY_OFFSET)
 
-/* DMA Channel Registers */
-#define IOAT_CHANCTRL_OFFSET		0x80
-#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK	0xF000
-#define IOAT_CHANCTRL_COMPL_DCA_EN		0x0200
-#define IOAT_CHANCTRL_CHANNEL_IN_USE		0x0100
-#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL	0x0020
-#define IOAT_CHANCTRL_ERR_INT_EN		0x0010
-#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN		0x0008
-#define IOAT_CHANCTRL_ERR_COMPLETION_EN		0x0004
-#define IOAT_CHANCTRL_INT_REARM			0x0001
-#define IOAT_CHANCTRL_RUN			(IOAT_CHANCTRL_INT_REARM |\
-						 IOAT_CHANCTRL_ANY_ERR_ABORT_EN)
-
-#define ioat_write_chanctrl(ioat, value) \
+#define	ioat_write_chanctrl(ioat, value) \
 	ioat_write_2((ioat), IOAT_CHANCTRL_OFFSET, (value))
 
-#define IOAT_CHANCMD_OFFSET		0x84
-#define IOAT_CHANCMD_RESET		0x20
-#define IOAT_CHANCMD_SUSPEND		0x04
-
-#define IOAT_DMACOUNT_OFFSET		0x86
-
-#define IOAT_CHANSTS_OFFSET_LOW		0x88
-#define IOAT_CHANSTS_OFFSET_HIGH	0x8C
-#define IOAT_CHANSTS_OFFSET	        0x88
-
-#define IOAT_CHANSTS_STATUS		0x7ULL
-#define IOAT_CHANSTS_ACTIVE		0x0
-#define IOAT_CHANSTS_IDLE		0x1
-#define IOAT_CHANSTS_SUSPENDED		0x2
-#define IOAT_CHANSTS_HALTED		0x3
-
-#define IOAT_CHANSTS_UNAFFILIATED_ERROR	0x8ULL
-#define IOAT_CHANSTS_SOFT_ERROR		0x10ULL
-
-#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK	(~0x3FULL)
-
-#define IOAT_CHAINADDR_OFFSET_LOW	0x90
-#define IOAT_CHAINADDR_OFFSET_HIGH	0x94
-
-#define IOAT_CHANCMP_OFFSET_LOW		0x98
-#define IOAT_CHANCMP_OFFSET_HIGH	0x9C
-
-#define IOAT_CHANERR_OFFSET		0xA8
-
-#define IOAT_CFG_CHANERR_INT_OFFSET		0x180
-#define IOAT_CFG_CHANERRMASK_INT_OFFSET		0x184
-
-#define IOAT_MIN_ORDER			4
-#define IOAT_MAX_ORDER			16
-
 static __inline uint64_t
 ioat_bus_space_read_8_lower_first(bus_space_tag_t tag,
     bus_space_handle_t handle, bus_size_t offset)
@@ -186,6 +115,10 @@
 
 MALLOC_DECLARE(M_IOAT);
 
+SYSCTL_DECL(_hw_ioat);
+
+void ioat_log_message(int verbosity, char *fmt, ...);
+
 struct ioat_dma_hw_descriptor {
 	uint32_t size;
 	union {
@@ -380,57 +313,57 @@
 
 /* One of these per allocated PCI device. */
 struct ioat_softc {
-
-	bus_dmaengine_t     dmaengine;
-	int                 version;
-
-	struct mtx          submit_lock;
-	int                 num_interrupts;
-	device_t            device;
-	bus_space_tag_t     pci_bus_tag;
-	bus_space_handle_t  pci_bus_handle;
-	int                 pci_resource_id;
-	struct resource     *pci_resource;
-	uint32_t            max_xfer_size;
-
-	struct resource *   res;
-	int                 rid;
-	void *              tag;
-
-	bus_dma_tag_t       hw_desc_tag;
-	bus_dmamap_t        hw_desc_map;
-
-	bus_dma_tag_t       comp_update_tag;
-	bus_dmamap_t        comp_update_map;
-	uint64_t            *comp_update;
-	bus_addr_t          comp_update_bus_addr;
-
-	struct callout      timer;
-
-	boolean_t           is_resize_pending;
-	boolean_t           is_completion_pending;
-	boolean_t           is_reset_pending;
-	boolean_t           is_channel_running;
-	boolean_t           is_waiting_for_ack;
-
-	uint32_t            xfercap_log;
-	uint32_t            head;
-	uint32_t            tail;
-	uint16_t            reserved;
-	uint32_t            ring_size_order;
-	bus_addr_t          last_seen;
-
-	struct ioat_descriptor **ring;
-
-	struct mtx          cleanup_lock;
+	bus_dmaengine_t		dmaengine;
+#define	to_ioat_softc(_dmaeng)						\
+({									\
+	bus_dmaengine_t *_p = (_dmaeng);				\
+	(struct ioat_softc *)((char *)_p -				\
+	    offsetof(struct ioat_softc, dmaengine));			\
+})
+
+	int			version;
+
+	struct mtx		submit_lock;
+	int			num_interrupts;
+	device_t		device;
+	bus_space_tag_t		pci_bus_tag;
+	bus_space_handle_t	pci_bus_handle;
+	int			pci_resource_id;
+	struct resource		*pci_resource;
+	uint32_t		max_xfer_size;
+
+	struct resource		*res;
+	int			rid;
+	void			*tag;
+
+	bus_dma_tag_t		hw_desc_tag;
+	bus_dmamap_t		hw_desc_map;
+
+	bus_dma_tag_t		comp_update_tag;
+	bus_dmamap_t		comp_update_map;
+	uint64_t		*comp_update;
+	bus_addr_t		comp_update_bus_addr;
+
+	struct callout		timer;
+
+	boolean_t		is_resize_pending;
+	boolean_t		is_completion_pending;
+	boolean_t		is_reset_pending;
+	boolean_t		is_channel_running;
+	boolean_t		is_waiting_for_ack;
+
+	uint32_t		xfercap_log;
+	uint32_t		head;
+	uint32_t		tail;
+	uint16_t		reserved;
+	uint32_t		ring_size_order;
+	bus_addr_t		last_seen;
+
+	struct ioat_descriptor	**ring;
+
+	struct mtx		cleanup_lock;
 };
 
-#ifdef IOAT_LOGGING
-void ioat_log_message(int verbosity, char *log_message, ...);
-#else
-#define ioat_log_message(x...)
-#endif
-
 static inline uint64_t
 ioat_get_chansts(struct ioat_softc *ioat)
 {
@@ -438,10 +371,9 @@
 
 	if (ioat->version >= IOAT_VER_3_3)
 		status = ioat_read_8(ioat, IOAT_CHANSTS_OFFSET);
-	else {
+	else
 		/* Must read lower 4 bytes before upper 4 bytes. */
 		status = ioat_read_double_4(ioat, IOAT_CHANSTS_OFFSET);
-	}
 	return (status);
 }
 
@@ -507,7 +439,7 @@
 	uint8_t cmd;
 
 	cmd = ioat_read_1(ioat, IOAT_CHANCMD_OFFSET);
-	return ((cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET);
+	return ((cmd & IOAT_CHANCMD_RESET) != 0);
 }
 
 #endif /* __IOAT_INTERNAL_H__ */
Index: sys/dev/ioat/ioat_logger.h
===================================================================
--- sys/dev/ioat/ioat_logger.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*-
- * Copyright (C) 2012 Intel Corporation
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef __IOAT_LOGGER_H__
-#define __IOAT_LOGGER_H__
-
-#ifdef IOAT_LOGGING
-void ioat_log_message(int verbosity, char *log_message, ...);
-#else
-#define ioat_log_message(x...)
-#endif
-
-extern int g_ioat_debug_level;
-
-#endif /* __IOAT_LOGGER_H__ */
Index: sys/dev/ioat/ioat_logger.c
===================================================================
--- sys/dev/ioat/ioat_logger.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (C) 2012 Intel Corporation
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/time.h>
-#include <machine/stdarg.h>
-
-#include "ioat_logger.h"
-
-int g_ioat_debug_level = 0;
-
-#ifdef IOAT_LOGGING
-void
-ioat_log_message(int verbosity, char *log_message, ...)
-{
-	va_list argp;
-	char buffer[512];
-	struct timeval tv;
-
-	if (verbosity > g_ioat_debug_level)
-		return;
-
-	va_start (argp, log_message);
-	vsnprintf(buffer, sizeof(buffer)-1, log_message, argp);
-	va_end(argp);
-	microuptime(&tv);
-
-	printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
-}
-#endif
Index: sys/dev/ioat/ioat_test.h
===================================================================
--- sys/dev/ioat/ioat_test.h
+++ sys/dev/ioat/ioat_test.h
@@ -28,18 +28,17 @@
 #define __IOAT_TEST_H__
 
 struct ioat_test {
-
 	uint32_t channel_index;
 	uint32_t num_loops;
 	volatile uint32_t num_completions;
 	uint32_t status;
 };
 
-#define IOAT_TEST_OK		0
-#define IOAT_TEST_NO_DMA_ENGINE	1
-#define IOAT_TEST_NO_MEMORY	2
-#define IOAT_TEST_MISCOMPARE	3
+#define	IOAT_TEST_OK		0
+#define	IOAT_TEST_NO_DMA_ENGINE	1
+#define	IOAT_TEST_NO_MEMORY	2
+#define	IOAT_TEST_MISCOMPARE	3
 
-#define IOAT_DMATEST	_IOWR('i', 0, struct ioat_test)
+#define	IOAT_DMATEST	_IOWR('i', 0, struct ioat_test)
 
 #endif /* __IOAT_TEST_H__ */
Index: sys/dev/ioat/ioat_test.c
===================================================================
--- sys/dev/ioat/ioat_test.c
+++ sys/dev/ioat/ioat_test.c
@@ -47,23 +47,25 @@
 #include <vm/pmap.h>
 
 #include "ioat.h"
+#include "ioat_hw.h"
+#include "ioat_internal.h"
 #include "ioat_test.h"
-#include "ioat_logger.h"
 
 MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
 
-static struct cdev *ioat_cdev = NULL;
-
-#define IOAT_TEST_SIZE	0x40000
-#define IOAT_MAX_BUFS	8
+#define	IOAT_TEST_SIZE	0x40000
+#define	IOAT_MAX_BUFS	8
 
 struct test_transaction {
 	uint8_t			num_buffers;
 	void			*buf[IOAT_MAX_BUFS];
 	uint32_t		length;
-	struct ioat_test        *test;
+	struct ioat_test	*test;
 };
 
+static int g_thread_index = 1;
+static struct cdev *g_ioat_cdev = NULL;
+
 static void
 ioat_test_transaction_destroy(struct test_transaction *tx)
 {
@@ -83,11 +85,10 @@
 test_transaction *ioat_test_transaction_create(uint8_t num_buffers,
     uint32_t buffer_size)
 {
-	struct test_transaction	*tx;
+	struct test_transaction *tx;
 	int i;
 
 	tx = malloc(sizeof(struct test_transaction), M_IOAT_TEST, M_NOWAIT | M_ZERO);
-
 	if (tx == NULL)
 		return (NULL);
 
@@ -95,7 +96,6 @@
 	tx->length = buffer_size;
 
 	for (i = 0; i < num_buffers; i++) {
-
 		tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT,
 		    0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
 
@@ -104,15 +104,17 @@
 			return (NULL);
 		}
 	}
-
 	return (tx);
 }
 
 static void
 ioat_dma_test_callback(void *arg)
 {
-	struct test_transaction *tx = arg;
-	struct ioat_test *test = tx->test;
+	struct test_transaction *tx;
+	struct ioat_test *test;
+
+	tx = arg;
+	test = tx->test;
 
 	if (memcmp(tx->buf[0], tx->buf[1], tx->length) != 0) {
 		ioat_log_message(0, "miscompare found\n");
@@ -124,16 +126,17 @@
 		wakeup(test);
 }
 
-int g_thread_index = 1;
-
 static void
 ioat_dma_test(void *arg)
 {
-	struct ioat_test *test = arg;
 	struct test_transaction *tx;
+	struct ioat_test *test;
 	bus_dmaengine_t dmaengine;
+	uint32_t loops;
 	int index, i;
-	uint32_t loops = test->num_loops;
+
+	test = arg;
+	loops = test->num_loops;
 
 	test->status = IOAT_TEST_OK;
 	test->num_completions = 0;
@@ -148,18 +151,18 @@
 	}
 
 	ioat_log_message(0, "Thread %d: num_loops remaining: 0x%07x\n", index,
-			 loops);
+	    test->num_loops);
 
-	while (loops-- > 0) {
+	for (loops = 0; loops < test->num_loops; loops++) {
 		bus_addr_t src, dest;
 
 		if (loops % 0x10000 == 0) {
 			ioat_log_message(0, "Thread %d: "
-			    "num_loops remaining: 0x%07x\n", index, loops);
+			    "num_loops remaining: 0x%07x\n", index,
+			    test->num_loops - loops);
 		}
 
 		tx = ioat_test_transaction_create(2, IOAT_TEST_SIZE);
-
 		if (tx == NULL) {
 			ioat_log_message(0, "tx == NULL - memory exhausted\n");
 			atomic_add_32(&test->num_completions, 1);
@@ -186,8 +189,8 @@
 		ioat_release(dmaengine);
 	}
 
-	while(test->num_completions < test->num_loops)
-		tsleep(test, 0, "compl", 5*hz);
+	while (test->num_completions < test->num_loops)
+		tsleep(test, 0, "compl", 5 * hz);
 
 }
 
@@ -207,15 +210,16 @@
 
 static int
 ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag,
-		struct thread *td)
+    struct thread *td)
 {
+
 	switch (cmd) {
 	case IOAT_DMATEST:
 		ioat_dma_test(arg);
 		break;
+	default:
+		return (EINVAL);
 	}
-
-
 	return (0);
 }
 
@@ -228,22 +232,25 @@
 	.d_name =	"ioat_test",
 };
 
-static void 
-ioat_test_init(void *arg)
-{
-	ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_OPERATOR,
-	    0600, "ioat_test");
-}
-
-static void 
-ioat_test_uninit(void *arg)
+static int
+sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)
 {
-	if (ioat_cdev != NULL) {
-		destroy_dev(ioat_cdev);
-		ioat_cdev = NULL;
+	int error, enabled;
+
+	enabled = (g_ioat_cdev != NULL);
+	error = sysctl_handle_int(oidp, &enabled, 0, req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+
+	if (enabled != 0 && g_ioat_cdev == NULL) {
+		g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL,
+		    0600, "ioat_test");
+	} else if (enabled == 0 && g_ioat_cdev != NULL) {
+		destroy_dev(g_ioat_cdev);
+		g_ioat_cdev = NULL;
 	}
+	return (0);
 }
-
-SYSINIT(ioat_test_load, SI_SUB_DRIVERS, SI_ORDER_FIRST, ioat_test_init, NULL);
-SYSUNINIT(ioat_test_unload, SI_SUB_DRIVERS, SI_ORDER_FIRST, ioat_test_uninit, NULL);
-
+SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_STRING | CTLFLAG_RW,
+    0, 0, sysctl_enable_ioat_test, "A",
+    "Non-zero: Enable the /dev/ioat_test device");
Index: sys/modules/Makefile
===================================================================
--- sys/modules/Makefile
+++ sys/modules/Makefile
@@ -145,6 +145,7 @@
 	${_igb} \
 	${_iir} \
 	${_io} \
+	${_ioat} \
 	${_ipdivert} \
 	${_ipfilter} \
 	${_ipfw} \
@@ -661,6 +662,7 @@
 _igb=		igb
 _iir=		iir
 _io=		io
+_ioat=		ioat
 _ipmi=		ipmi
 _ips=		ips
 _ipw=		ipw
Index: sys/modules/ioat/Makefile
===================================================================
--- sys/modules/ioat/Makefile
+++ sys/modules/ioat/Makefile
@@ -6,16 +6,10 @@
 
 .PATH:  ${IOAT_SRC_PATH}/dev/ioat
 
-KMOD    = ioat
-SRCS    = ioat.c ioat_logger.c
-#SRCS += ioat_test.c
+KMOD=	ioat
+SRCS=	ioat.c ioat_test.c
+SRCS+=	device_if.h bus_if.h pci_if.h
 
-SRCS += device_if.h bus_if.h pci_if.h
+CFLAGS+=	-I${IOAT_SRC_PATH}
 
 .include <bsd.kmod.mk>
-
-DEBUG_FLAGS = -g
-#CFLAGS += -O0
-CFLAGS += -I$(IOAT_SRC_PATH)
-CFLAGS += -DIOAT_LOGGING
-#CFLAGS += -DINVARIANTS
Index: sys/modules/ioat/README
===================================================================
--- sys/modules/ioat/README
+++ sys/modules/ioat/README
@@ -5,30 +5,28 @@
    to insert descriptors which only trigger an interrupt.
 * Note that copy operation takes bus addresses as parameters, not virtual
    addresses.
-* Copies larger than max transfer size (1MB) are not supported, and will
-   panic the system explicitly.  Future versions will likely support this by
-   breaking up the transfer into smaller sizes.
-* There are 8 CB-DMA channels per CPU package.  Each channel will be
-   represented by a struct ioat_softc data structure.  
-* There is an ioat_test.c module in the driver, and an ioatcontrol.c
-   application that can be used to both test the driver as well as understand
-   how to interface with the driver.  App uses a /dev/ioat_test device node
-   to send IOCTLs which will cause the driver to:
+* Copies larger than max transfer size (1MB) are not supported.  Future
+   versions will likely support this by breaking up the transfer into smaller
+   sizes.
+* There are 8 CB-DMA channels per CPU package.  Each channel is represented by
+   a struct ioat_softc.
+* There is an ioat_test.c module in the driver which exposes a simple device at
+   /dev/ioat_test.  The presense of the test device may be controlled with the
+   hw.ioat.enable_ioat_test tunable.  (The default is off.)  There is an
+   ioatcontrol.c application in tools/tools/ioat
+   that can be used to test the driver.  It sends IOCTLs which will cause the
+   driver to:
 	* run a loop a number of times as requested by the app
 	* each loop will allocate two chunks of memory, write data patterns
 	   to each, submit a DMA request to copy one buffer to the other,
 	   and compare the contents in the callback, reporting an error if one
 	   is found
-   The ioat_test.c module needs to be enabled in the driver Makefile to use it.
-   To build the app from /sys/modules/ioat directory:  make ioatcontrol
-   To run the app:  ./ioatcontrol <channel #> <num_loops>
-* Debug prints are enabled by defining IOAT_LOGGING, which is currently the
-   default in the Makefile.  Verbosity is 0 by default (only critical errors)
-   but can be increased by doing one of the following:
+   To build the test program from tools/tools/ioat:  make ioatcontrol
+   To run it:  ./ioatcontrol <channel #> <num_loops>
+* Debug verbosity is 0 by default (only critical errors) but can be increased
+   by doing one of the following:
 	* from command line, before driver is loaded:
 		kenv hw.ioat.debug_level=3
 	* add the following to /boot/loader.conf
 		hw.ioat.debug_level=3
-
-
-
+	* at runtime: sysctl hw.ioat.debug_level=3
Index: tools/tools/ioat/Makefile
===================================================================
--- tools/tools/ioat/Makefile
+++ tools/tools/ioat/Makefile
@@ -1,6 +1,7 @@
-PROG= ioatcontrol
-SRCS= ioatcontrol.c
+PROG=	ioatcontrol
+SRCS=	ioatcontrol.c
 NO_MAN=
-CFLAGS+= -I../../../sys/dev/ioat
+CFLAGS+=	-I${.CURDIR:H:H:H}/sys/dev/ioat
+WARNS?=	6
 
 .include <bsd.prog.mk>
Index: tools/tools/ioat/ioatcontrol.c
===================================================================
--- tools/tools/ioat/ioatcontrol.c
+++ tools/tools/ioat/ioatcontrol.c
@@ -25,15 +25,17 @@
  */
 
 #include <sys/ioctl.h>
+
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include <fcntl.h>
+#include <sysexits.h>
 #include <unistd.h>
 
 #include "ioat_test.h"
 
-int 
+int
 main(int argc, char **argv)
 {
 	struct ioat_test t;
@@ -41,26 +43,24 @@
 
 	if (argc < 3) {
 		printf("Usage: %s <channel #> <num_loops>\n", argv[0]);
-		return (-1);
+		return (EX_USAGE);
 	}
 
 	t.channel_index = atoi(argv[1]);
-
 	if (t.channel_index > 8) {
 		printf("Channel number must be between 0 and 7.\n");
-		return (-1);
+		return (EX_USAGE);
 	}
 
 	t.num_loops = atoi(argv[2]);
 
 	fd = open("/dev/ioat_test", O_RDWR);
-
 	if (fd < 0) {
 		printf("Cannot open /dev/ioat_test\n");
-		return (-1);
+		return (EX_UNAVAILABLE);
 	}
 
-	ioctl(fd, IOAT_DMATEST, &t);
+	(void)ioctl(fd, IOAT_DMATEST, &t);
 	close(fd);
 
 	return (t.status);