Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ioat/ioat.c
Show All 39 Lines | |||||
#include <sys/rman.h> | #include <sys/rman.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/resource.h> | #include <machine/resource.h> | ||||
#include "ioat.h" | #include "ioat.h" | ||||
#include "ioat_hw.h" | |||||
#include "ioat_internal.h" | #include "ioat_internal.h" | ||||
#include "ioat_logger.h" | #include "ioat_logger.h" | ||||
#include "ioat_sysctl.h" | |||||
static int ioat_probe(device_t device); | static int ioat_probe(device_t device); | ||||
static int ioat_attach(device_t device); | static int ioat_attach(device_t device); | ||||
static int ioat_detach(device_t device); | static int ioat_detach(device_t device); | ||||
static int ioat3_attach(device_t device); | static int ioat3_attach(device_t device); | ||||
static int ioat_map_pci_bar(struct ioat_softc *ioat); | static int ioat_map_pci_bar(struct ioat_softc *ioat); | ||||
static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, | static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, | ||||
int error); | int error); | ||||
static void ioat_interrupt_setup(struct ioat_softc *ioat); | static int ioat_interrupt_setup(struct ioat_softc *ioat); | ||||
static void ioat_interrupt_handler(void *arg); | static void ioat_interrupt_handler(void *arg); | ||||
static void ioat_process_events(struct ioat_softc *ioat); | static void ioat_process_events(struct ioat_softc *ioat); | ||||
static inline uint32_t ioat_get_active(struct ioat_softc *ioat); | static inline uint32_t ioat_get_active(struct ioat_softc *ioat); | ||||
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); | static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); | ||||
static void ioat_free_ring_entry(struct ioat_softc *ioat, | static void ioat_free_ring_entry(struct ioat_softc *ioat, | ||||
struct ioat_descriptor *desc); | struct ioat_descriptor *desc); | ||||
static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat); | static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat); | ||||
static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs); | static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs); | ||||
static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat, | static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat, | ||||
uint32_t index); | uint32_t index); | ||||
static boolean_t resize_ring(struct ioat_softc *ioat, int order); | static boolean_t resize_ring(struct ioat_softc *ioat, int order); | ||||
static void ioat_timer_callback(void *arg); | static void ioat_timer_callback(void *arg); | ||||
static void dump_descriptor(void *hw_desc); | static void dump_descriptor(void *hw_desc); | ||||
static void ioat_submit_single(struct ioat_softc *ioat); | static void ioat_submit_single(struct ioat_softc *ioat); | ||||
static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | ||||
int error); | int error); | ||||
static int ioat_reset_hw(struct ioat_softc *ioat); | static int ioat_reset_hw(struct ioat_softc *ioat); | ||||
static void ioat_setup_sysctl(device_t device); | static void ioat_setup_sysctl(device_t device); | ||||
MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations"); | MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations"); | ||||
SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node"); | |||||
//#define IOAT_LOGGING | static int g_force_legacy_interrupts; | ||||
SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RWTUN, | |||||
&g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled"); | |||||
jimharris: I don't think this works as a sysctl. This value is needed when the driver loads, to determine… | |||||
cemAuthorUnsubmitted Not Done Inline ActionsSure. I will change it to RDTUN. cem: Sure. I will change it to RDTUN. | |||||
/* | /* | ||||
* OS <-> Driver interface structures | * OS <-> Driver interface structures | ||||
*/ | */ | ||||
static device_method_t ioat_pci_methods[] = { | static device_method_t ioat_pci_methods[] = { | ||||
/* Device interface */ | /* Device interface */ | ||||
DEVMETHOD(device_probe, ioat_probe), | DEVMETHOD(device_probe, ioat_probe), | ||||
DEVMETHOD(device_attach, ioat_attach), | DEVMETHOD(device_attach, ioat_attach), | ||||
DEVMETHOD(device_detach, ioat_detach), | DEVMETHOD(device_detach, ioat_detach), | ||||
{ 0, 0 } | { 0, 0 } | ||||
}; | }; | ||||
static driver_t ioat_pci_driver = { | static driver_t ioat_pci_driver = { | ||||
"ioat", | "ioat", | ||||
ioat_pci_methods, | ioat_pci_methods, | ||||
sizeof(struct ioat_softc), | sizeof(struct ioat_softc), | ||||
}; | }; | ||||
static devclass_t ioat_devclass; | static devclass_t ioat_devclass; | ||||
DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0); | DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0); | ||||
/* | /* | ||||
* Private data structures | * Private data structures | ||||
*/ | */ | ||||
struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; | static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; | ||||
int ioat_channel_index = 0; | static int ioat_channel_index = 0; | ||||
SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, | |||||
"Number of IOAT channels attached"); | |||||
static struct _pcsid | static struct _pcsid | ||||
{ | { | ||||
u_int32_t type; | u_int16_t type; | ||||
const char *desc; | const char *desc; | ||||
} pci_ids[] = { | } pci_ids[] = { | ||||
{ 0x3c208086, "SNB IOAT Ch0" }, | { 0x3430, "TBG IOAT Ch0" }, | ||||
{ 0x3c218086, "SNB IOAT Ch1" }, | { 0x3431, "TBG IOAT Ch1" }, | ||||
{ 0x3c228086, "SNB IOAT Ch2" }, | { 0x3432, "TBG IOAT Ch2" }, | ||||
{ 0x3c238086, "SNB IOAT Ch3" }, | { 0x3433, "TBG IOAT Ch3" }, | ||||
{ 0x3c248086, "SNB IOAT Ch4" }, | { 0x3429, "TBG IOAT Ch4" }, | ||||
{ 0x3c258086, "SNB IOAT Ch5" }, | { 0x342a, "TBG IOAT Ch5" }, | ||||
{ 0x3c268086, "SNB IOAT Ch6" }, | { 0x342b, "TBG IOAT Ch6" }, | ||||
{ 0x3c278086, "SNB IOAT Ch7" }, | { 0x342c, "TBG IOAT Ch7" }, | ||||
{ 0x3c2e8086, "SNB IOAT Ch0" }, | |||||
{ 0x3c2f8086, "SNB IOAT Ch1" }, | { 0x3710, "JSF IOAT Ch0" }, | ||||
{ 0x00000000, NULL } | { 0x3711, "JSF IOAT Ch1" }, | ||||
{ 0x3712, "JSF IOAT Ch2" }, | |||||
{ 0x3713, "JSF IOAT Ch3" }, | |||||
{ 0x3714, "JSF IOAT Ch4" }, | |||||
{ 0x3715, "JSF IOAT Ch5" }, | |||||
{ 0x3716, "JSF IOAT Ch6" }, | |||||
{ 0x3717, "JSF IOAT Ch7" }, | |||||
{ 0x3718, "JSF IOAT Ch0 (RAID)" }, | |||||
{ 0x3719, "JSF IOAT Ch1 (RAID)" }, | |||||
{ 0x3c20, "SNB IOAT Ch0" }, | |||||
{ 0x3c21, "SNB IOAT Ch1" }, | |||||
{ 0x3c22, "SNB IOAT Ch2" }, | |||||
{ 0x3c23, "SNB IOAT Ch3" }, | |||||
{ 0x3c24, "SNB IOAT Ch4" }, | |||||
{ 0x3c25, "SNB IOAT Ch5" }, | |||||
{ 0x3c26, "SNB IOAT Ch6" }, | |||||
{ 0x3c27, "SNB IOAT Ch7" }, | |||||
{ 0x3c2e, "SNB IOAT Ch0 (RAID)" }, | |||||
{ 0x3c2f, "SNB IOAT Ch1 (RAID)" }, | |||||
{ 0x0e20, "IVB IOAT Ch0" }, | |||||
{ 0x0e21, "IVB IOAT Ch1" }, | |||||
{ 0x0e22, "IVB IOAT Ch2" }, | |||||
{ 0x0e23, "IVB IOAT Ch3" }, | |||||
{ 0x0e24, "IVB IOAT Ch4" }, | |||||
{ 0x0e25, "IVB IOAT Ch5" }, | |||||
{ 0x0e26, "IVB IOAT Ch6" }, | |||||
{ 0x0e27, "IVB IOAT Ch7" }, | |||||
{ 0x0e2e, "IVB IOAT Ch0 (RAID)" }, | |||||
{ 0x0e2f, "IVB IOAT Ch1 (RAID)" }, | |||||
{ 0x2f20, "HSW IOAT Ch0" }, | |||||
{ 0x2f21, "HSW IOAT Ch1" }, | |||||
{ 0x2f22, "HSW IOAT Ch2" }, | |||||
{ 0x2f23, "HSW IOAT Ch3" }, | |||||
{ 0x2f24, "HSW IOAT Ch4" }, | |||||
{ 0x2f25, "HSW IOAT Ch5" }, | |||||
{ 0x2f26, "HSW IOAT Ch6" }, | |||||
{ 0x2f27, "HSW IOAT Ch7" }, | |||||
{ 0x2f2e, "HSW IOAT Ch0 (RAID)" }, | |||||
{ 0x2f2f, "HSW IOAT Ch1 (RAID)" }, | |||||
{ 0x0c50, "BWD IOAT Ch0" }, | |||||
{ 0x0c51, "BWD IOAT Ch1" }, | |||||
{ 0x0c52, "BWD IOAT Ch2" }, | |||||
{ 0x0c53, "BWD IOAT Ch3" }, | |||||
{ 0x6f50, "BDXDE IOAT Ch0" }, | |||||
{ 0x6f51, "BDXDE IOAT Ch1" }, | |||||
{ 0x6f52, "BDXDE IOAT Ch2" }, | |||||
{ 0x6f53, "BDXDE IOAT Ch3" }, | |||||
{ 0x0000, NULL } | |||||
jimharrisUnsubmitted Done Inline ActionsI agree with imp@ that this should go back to including both device and vendor ID in the table. jimharris: I agree with imp@ that this should go back to including both device and vendor ID in the table. | |||||
}; | }; | ||||
/* | /* | ||||
* OS <-> Driver linkage functions | * OS <-> Driver linkage functions | ||||
*/ | */ | ||||
static int | static int | ||||
ioat_probe(device_t device) | ioat_probe(device_t device) | ||||
{ | { | ||||
u_int32_t type = pci_get_devid(device); | struct _pcsid *ep; | ||||
struct _pcsid *ep = pci_ids; | u_int32_t type; | ||||
while (ep->type) { | type = pci_get_devid(device); | ||||
if ((type & 0xffff) != 0x8086) | |||||
return (ENXIO); | |||||
type >>= 16; | |||||
for (ep = pci_ids; ep->type; ep++) { | |||||
if (ep->type == type) { | if (ep->type == type) { | ||||
device_set_desc(device, ep->desc); | device_set_desc(device, ep->desc); | ||||
return (0); | return (0); | ||||
} | } | ||||
++ep; | |||||
} | } | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
static int | static int | ||||
ioat_attach(device_t device) | ioat_attach(device_t device) | ||||
{ | { | ||||
int error = 0; | struct ioat_softc *ioat; | ||||
struct ioat_softc *ioat = DEVICE2SOFTC(device); | int error; | ||||
ioat = DEVICE2SOFTC(device); | |||||
ioat->device = device; | ioat->device = device; | ||||
/* | error = ioat_map_pci_bar(ioat); | ||||
* TODO: this needs to be moved into a driver init function | if (error != 0) | ||||
* so that it only gets executed once overall, rather than | goto err; | ||||
* once per channel. But this works fine for now. | |||||
*/ | |||||
TUNABLE_INT_FETCH("hw.ioat.debug_level", &g_ioat_debug_level); | |||||
ioat_map_pci_bar(ioat); | |||||
ioat->version = ioat_read_cbver(ioat); | ioat->version = ioat_read_cbver(ioat); | ||||
ioat_interrupt_setup(ioat); | ioat_interrupt_setup(ioat); | ||||
if (ioat->version >= IOAT_VER_3_0) { | if (ioat->version < IOAT_VER_3_0) { | ||||
error = ioat3_attach(device); | error = ENODEV; | ||||
} else { | goto err; | ||||
ioat_detach(device); | |||||
error = -1; | |||||
} | } | ||||
if (error == 0) | error = ioat3_attach(device); | ||||
if (error != 0) | |||||
goto err; | |||||
ioat_channel[ioat_channel_index++] = ioat; | ioat_channel[ioat_channel_index++] = ioat; | ||||
err: | |||||
if (error != 0) | |||||
ioat_detach(device); | |||||
return (error); | return (error); | ||||
jimharrisUnsubmitted Done Inline ActionsI missed an explicit pci_enable_busmaster() call here (with matching pci_disable_busmaster() in detach). jimharris: I missed an explicit pci_enable_busmaster() call here (with matching pci_disable_busmaster() in… | |||||
} | } | ||||
static int | static int | ||||
ioat_detach(device_t device) | ioat_detach(device_t device) | ||||
{ | { | ||||
struct ioat_softc *ioat = DEVICE2SOFTC(device); | struct ioat_softc *ioat; | ||||
uint32_t i; | |||||
if (ioat->pci_resource != NULL) { | ioat = DEVICE2SOFTC(device); | ||||
callout_drain(&ioat->timer); | |||||
if (ioat->pci_resource != NULL) | |||||
bus_release_resource(device, SYS_RES_MEMORY, | bus_release_resource(device, SYS_RES_MEMORY, | ||||
ioat->pci_resource_id, ioat->pci_resource); | ioat->pci_resource_id, ioat->pci_resource); | ||||
} | |||||
if (ioat->ring) { | if (ioat->ring != NULL) { | ||||
int i; | |||||
for (i = 0; i < (1 << ioat->ring_size_order); i++) | for (i = 0; i < (1 << ioat->ring_size_order); i++) | ||||
ioat_free_ring_entry(ioat, ioat->ring[i]); | ioat_free_ring_entry(ioat, ioat->ring[i]); | ||||
free(ioat->ring, M_IOAT); | free(ioat->ring, M_IOAT); | ||||
} | } | ||||
if (ioat->comp_update) { | if (ioat->comp_update != NULL) { | ||||
bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map); | bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map); | ||||
bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update, | bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update, | ||||
ioat->comp_update_map); | ioat->comp_update_map); | ||||
bus_dma_tag_destroy(ioat->comp_update_tag); | bus_dma_tag_destroy(ioat->comp_update_tag); | ||||
} | } | ||||
bus_dma_tag_destroy(ioat->hw_desc_tag); | bus_dma_tag_destroy(ioat->hw_desc_tag); | ||||
if(ioat->tag != NULL) | if (ioat->tag != NULL) | ||||
bus_teardown_intr(device, ioat->res, ioat->tag); | bus_teardown_intr(device, ioat->res, ioat->tag); | ||||
if(ioat->res != NULL) | if (ioat->res != NULL) | ||||
bus_release_resource(device, SYS_RES_IRQ, | bus_release_resource(device, SYS_RES_IRQ, | ||||
rman_get_rid(ioat->res), ioat->res); | rman_get_rid(ioat->res), ioat->res); | ||||
pci_release_msi(device); | pci_release_msi(device); | ||||
callout_drain(&ioat->timer); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | |||||
ioat3_selftest(struct ioat_softc *ioat) | |||||
{ | |||||
uint64_t status; | |||||
int i; | |||||
ioat_acquire(&ioat->dmaengine); | |||||
ioat_null(&ioat->dmaengine, NULL, NULL, 0); | |||||
ioat_release(&ioat->dmaengine); | |||||
for (i = 0; i < 100; i++) { | |||||
DELAY(1); | |||||
status = ioat_get_chansts(ioat); | |||||
if (is_ioat_idle(status)) | |||||
return (0); | |||||
} | |||||
return (ENXIO); | |||||
} | |||||
/* | /* | ||||
* Initialize Hardware | * Initialize Hardware | ||||
*/ | */ | ||||
static int | static int | ||||
ioat3_attach(device_t device) | ioat3_attach(device_t device) | ||||
{ | { | ||||
struct ioat_softc *ioat = DEVICE2SOFTC(device); | struct ioat_softc *ioat; | ||||
struct ioat_descriptor **ring; | struct ioat_descriptor **ring; | ||||
int error = 0; | |||||
uint8_t xfercap; | |||||
uint32_t capabilities, chanerr; | |||||
uint64_t status; | |||||
int i, num_descriptors; | |||||
struct ioat_descriptor *next; | struct ioat_descriptor *next; | ||||
struct ioat_dma_hw_descriptor *dma_hw_desc; | struct ioat_dma_hw_descriptor *dma_hw_desc; | ||||
uint64_t status; | |||||
uint32_t capabilities, chanerr; | |||||
int i, num_descriptors; | |||||
int error; | |||||
uint8_t xfercap; | |||||
error = 0; | |||||
ioat = DEVICE2SOFTC(device); | |||||
capabilities = ioat_read_dmacapability(ioat); | capabilities = ioat_read_dmacapability(ioat); | ||||
xfercap = ioat_read_xfercap(ioat); | xfercap = ioat_read_xfercap(ioat); | ||||
/* Only bits [4:0] are valid. */ | /* Only bits [4:0] are valid. */ | ||||
xfercap &= 0x1f; | xfercap &= 0x1f; | ||||
ioat->max_xfer_size = 1 << xfercap; | ioat->max_xfer_size = 1 << xfercap; | ||||
/* TODO: need to check DCA here if we ever do XOR/PQ */ | /* TODO: need to check DCA here if we ever do XOR/PQ */ | ||||
mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | ||||
mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF); | mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF); | ||||
callout_init(&ioat->timer, CALLOUT_MPSAFE); | callout_init(&ioat->timer, CALLOUT_MPSAFE); | ||||
ioat->is_resize_pending = FALSE; | ioat->is_resize_pending = FALSE; | ||||
ioat->is_completion_pending = FALSE; | ioat->is_completion_pending = FALSE; | ||||
ioat->is_reset_pending = FALSE; | ioat->is_reset_pending = FALSE; | ||||
ioat->is_channel_running = FALSE; | ioat->is_channel_running = FALSE; | ||||
ioat->is_waiting_for_ack = FALSE; | ioat->is_waiting_for_ack = FALSE; | ||||
bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, | bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, | ||||
BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | ||||
sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, | sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, | ||||
&ioat->comp_update_tag); | &ioat->comp_update_tag); | ||||
error = bus_dmamem_alloc(ioat->comp_update_tag, | error = bus_dmamem_alloc(ioat->comp_update_tag, | ||||
(void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); | (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); | ||||
if (ioat->comp_update == NULL) | |||||
return (ENOMEM); | |||||
if (!ioat->comp_update) { | |||||
ioat_detach(ioat->device); | |||||
return (-1); | |||||
} | |||||
error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map, | error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map, | ||||
ioat->comp_update, sizeof(uint64_t), | ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat, | ||||
ioat_comp_update_map, ioat, 0); | 0); | ||||
if (error != 0) | |||||
return (error); | |||||
ioat->ring_size_order = IOAT_MIN_ORDER; | ioat->ring_size_order = IOAT_MIN_ORDER; | ||||
num_descriptors = 1 << ioat->ring_size_order; | num_descriptors = 1 << ioat->ring_size_order; | ||||
bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, | bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, | ||||
BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | ||||
sizeof(struct ioat_dma_hw_descriptor), 1, | sizeof(struct ioat_dma_hw_descriptor), 1, | ||||
sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, | sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, | ||||
&ioat->hw_desc_tag); | &ioat->hw_desc_tag); | ||||
ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, | ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, | ||||
M_ZERO | M_NOWAIT); | M_ZERO | M_NOWAIT); | ||||
if (ioat->ring == NULL) | |||||
return (ENOMEM); | |||||
if (!ioat->ring) { | |||||
ioat_detach(ioat->device); | |||||
return (-1); | |||||
} | |||||
ring = ioat->ring; | ring = ioat->ring; | ||||
for (i = 0; i < num_descriptors; i++) { | for (i = 0; i < num_descriptors; i++) { | ||||
ring[i] = ioat_alloc_ring_entry(ioat); | ring[i] = ioat_alloc_ring_entry(ioat); | ||||
if (ring[i] == NULL) | |||||
return (ENOMEM); | |||||
if (!ring[i]) { | |||||
ioat_detach(ioat->device); | |||||
return (-1); | |||||
} | |||||
ring[i]->id = i; | ring[i]->id = i; | ||||
} | } | ||||
for (i = 0; i < num_descriptors-1; i++) { | for (i = 0; i < num_descriptors - 1; i++) { | ||||
next = ring[i+1]; | next = ring[i + 1]; | ||||
dma_hw_desc = ring[i]->u.dma; | dma_hw_desc = ring[i]->u.dma; | ||||
dma_hw_desc->next = next->hw_desc_bus_addr; | dma_hw_desc->next = next->hw_desc_bus_addr; | ||||
} | } | ||||
ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; | ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; | ||||
ioat->head = 0; | ioat->head = 0; | ||||
ioat->tail = 0; | ioat->tail = 0; | ||||
ioat->last_seen = 0; | ioat->last_seen = 0; | ||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
ioat_reset_hw(ioat); | error = ioat_reset_hw(ioat); | ||||
if (error) | |||||
return (error); | |||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ||||
ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr); | ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr); | ||||
ioat_acquire(&ioat->dmaengine); | error = ioat3_selftest(ioat); | ||||
ioat_null(&ioat->dmaengine, NULL, NULL, 0); | if (error != 0) { | ||||
ioat_release(&ioat->dmaengine); | |||||
i = 100; | |||||
while (i-- > 0) { | |||||
DELAY(1); | |||||
status = ioat_get_chansts(ioat); | |||||
if (is_ioat_idle(status)) | |||||
break; | |||||
} | |||||
if (is_ioat_idle(status)) { | |||||
ioat_process_events(ioat); | |||||
} else { | |||||
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | ||||
ioat_log_message(0, "could not start channel: " | ioat_log_message(0, "could not start channel: " | ||||
"status = %p\n error = %x\n", | "status = %p\n error = %x\n", (void *)status, chanerr); | ||||
jimharrisUnsubmitted Done Inline ActionsSuggest moving the ioat_read_4 and ioat_log_message be moved to your new ioat3_selftest function. jimharris: Suggest moving the ioat_read_4 and ioat_log_message be moved to your new ioat3_selftest… | |||||
cemAuthorUnsubmitted Not Done Inline ActionsSure. Will do. cem: Sure. Will do. | |||||
(void *)status, chanerr); | return (ENXIO); | ||||
error = -1; | |||||
} | } | ||||
if (error) | ioat_process_events(ioat); | ||||
{ | |||||
ioat_detach(device); | |||||
return (error); | |||||
} | |||||
ioat_setup_sysctl(device); | ioat_setup_sysctl(device); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ioat_map_pci_bar(struct ioat_softc *ioat) | ioat_map_pci_bar(struct ioat_softc *ioat) | ||||
{ | { | ||||
ioat->pci_resource_id = PCIR_BAR(0); | ioat->pci_resource_id = PCIR_BAR(0); | ||||
ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY, | ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY, | ||||
&ioat->pci_resource_id, 0, ~0, | &ioat->pci_resource_id, 0, ~0, 1, RF_ACTIVE); | ||||
1, RF_ACTIVE); | |||||
if(ioat->pci_resource == NULL) | if (ioat->pci_resource == NULL) { | ||||
ioat_log_message(0, "unable to allocate pci resource\n"); | ioat_log_message(0, "unable to allocate pci resource\n"); | ||||
else { | return (ENODEV); | ||||
ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource); | |||||
ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource); | |||||
} | } | ||||
ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource); | |||||
ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource); | |||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) | ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) | ||||
{ | { | ||||
struct ioat_softc *ioat = arg; | struct ioat_softc *ioat = arg; | ||||
ioat->comp_update_bus_addr = seg[0].ds_addr; | ioat->comp_update_bus_addr = seg[0].ds_addr; | ||||
} | } | ||||
static void | static void | ||||
ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) | ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) | ||||
jimharrisUnsubmitted Done Inline ActionsThis function needs style updates too. jimharris: This function needs style updates too. | |||||
{ | { | ||||
bus_addr_t *baddr = arg; | bus_addr_t *baddr = arg; | ||||
*baddr = segs->ds_addr; | *baddr = segs->ds_addr; | ||||
} | } | ||||
/* | /* | ||||
* Interrupt setup and handlers | * Interrupt setup and handlers | ||||
*/ | */ | ||||
static void | static int | ||||
ioat_interrupt_setup(struct ioat_softc *ioat) | ioat_interrupt_setup(struct ioat_softc *ioat) | ||||
{ | { | ||||
boolean_t use_msix = 0; | uint32_t num_vectors; | ||||
boolean_t force_legacy_interrupts = 0; | int error; | ||||
boolean_t use_msix; | |||||
boolean_t force_legacy_interrupts; | |||||
TUNABLE_INT_FETCH("hw.ioat.force_legacy_interrupts", | use_msix = FALSE; | ||||
&force_legacy_interrupts); | force_legacy_interrupts = FALSE; | ||||
if (!force_legacy_interrupts && | if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) { | ||||
pci_msix_count(ioat->device) >= 1) { | num_vectors = 1; | ||||
uint32_t num_vectors = 1; | |||||
pci_alloc_msix(ioat->device, &num_vectors); | pci_alloc_msix(ioat->device, &num_vectors); | ||||
if (num_vectors == 1) | if (num_vectors == 1) | ||||
use_msix = TRUE; | use_msix = TRUE; | ||||
} | } | ||||
if (use_msix == TRUE) { | if (use_msix) { | ||||
ioat->rid = 1; | ioat->rid = 1; | ||||
ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, | ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, | ||||
&ioat->rid, RF_ACTIVE); | &ioat->rid, RF_ACTIVE); | ||||
} | } else { | ||||
else { | |||||
ioat->rid = 0; | ioat->rid = 0; | ||||
ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, | ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, | ||||
&ioat->rid, RF_SHAREABLE | RF_ACTIVE); | &ioat->rid, RF_SHAREABLE | RF_ACTIVE); | ||||
} | } | ||||
if (ioat->res == NULL) { | if (ioat->res == NULL) { | ||||
ioat_log_message(0, "bus_alloc_resource failed\n"); | ioat_log_message(0, "bus_alloc_resource failed\n"); | ||||
return; | return (ENOMEM); | ||||
} | } | ||||
ioat->tag = NULL; | ioat->tag = NULL; | ||||
error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE | | |||||
if (bus_setup_intr(ioat->device, ioat->res, | INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag); | ||||
INTR_MPSAFE | INTR_TYPE_MISC, NULL, | if (error != 0) { | ||||
ioat_interrupt_handler, ioat, &ioat->tag)) { | |||||
ioat_log_message(0, "bus_setup_intr failed\n"); | ioat_log_message(0, "bus_setup_intr failed\n"); | ||||
return; | return (error); | ||||
} | } | ||||
ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN); | ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN); | ||||
return (0); | |||||
} | } | ||||
static void | static void | ||||
ioat_interrupt_handler(void *arg) | ioat_interrupt_handler(void *arg) | ||||
{ | { | ||||
struct ioat_softc *ioat = arg; | struct ioat_softc *ioat = arg; | ||||
ioat_process_events(ioat); | ioat_process_events(ioat); | ||||
} | } | ||||
static void | static void | ||||
ioat_process_events(struct ioat_softc *ioat) | ioat_process_events(struct ioat_softc *ioat) | ||||
{ | { | ||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct bus_dmadesc *dmadesc; | |||||
uint64_t comp_update, status; | uint64_t comp_update, status; | ||||
uint32_t completed; | uint32_t completed; | ||||
struct bus_dmadesc *dmadesc; | |||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
completed = 0; | completed = 0; | ||||
comp_update = *ioat->comp_update; | comp_update = *ioat->comp_update; | ||||
status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; | status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
if (status == ioat->last_seen) { | if (status == ioat->last_seen) { | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
return; | return; | ||||
} | } | ||||
while (1) { | while (1) { | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail); | desc = ioat_get_ring_entry(ioat, ioat->tail); | ||||
dmadesc = &desc->bus_dmadesc; | dmadesc = &desc->bus_dmadesc; | ||||
ioat_log_message(3, "completing desc %d\n", ioat->tail); | ioat_log_message(3, "completing desc %d\n", ioat->tail); | ||||
if (dmadesc->callback_fn) { | if (dmadesc->callback_fn) | ||||
(*dmadesc->callback_fn)(dmadesc->callback_arg); | (*dmadesc->callback_fn)(dmadesc->callback_arg); | ||||
} | |||||
ioat->tail++; | ioat->tail++; | ||||
if (desc->hw_desc_bus_addr == status) | if (desc->hw_desc_bus_addr == status) | ||||
break; | break; | ||||
} | } | ||||
ioat->last_seen = desc->hw_desc_bus_addr; | ioat->last_seen = desc->hw_desc_bus_addr; | ||||
if (ioat->head == ioat->tail) { | if (ioat->head == ioat->tail) { | ||||
ioat->is_completion_pending = FALSE; | ioat->is_completion_pending = FALSE; | ||||
callout_reset(&ioat->timer, 5000*hz/1000, ioat_timer_callback, | callout_reset(&ioat->timer, 5 * hz, ioat_timer_callback, ioat); | ||||
ioat); | |||||
} | } | ||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
} | } | ||||
/* | /* | ||||
* User API functions | * User API functions | ||||
*/ | */ | ||||
bus_dmaengine_t | bus_dmaengine_t | ||||
ioat_get_dmaengine(uint32_t index) | ioat_get_dmaengine(uint32_t index) | ||||
{ | { | ||||
if (index < ioat_channel_index) | if (index < ioat_channel_index) | ||||
return (&ioat_channel[index]->dmaengine); | return (&ioat_channel[index]->dmaengine); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
void | void | ||||
ioat_acquire(bus_dmaengine_t dmaengine) | ioat_acquire(bus_dmaengine_t dmaengine) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = (struct ioat_softc *)dmaengine; | ioat = to_ioat_softc(dmaengine); | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
} | } | ||||
void | void | ||||
ioat_release(bus_dmaengine_t dmaengine) | ioat_release(bus_dmaengine_t dmaengine) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
ioat = (struct ioat_softc *)dmaengine; | ioat = to_ioat_softc(dmaengine); | ||||
ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head); | ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head); | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
} | } | ||||
struct bus_dmadesc * | struct bus_dmadesc * | ||||
ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, | ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, | ||||
void *callback_arg, uint32_t flags) | void *callback_arg, uint32_t flags) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct ioat_dma_hw_descriptor *hw_desc; | struct ioat_dma_hw_descriptor *hw_desc; | ||||
ioat = (struct ioat_softc *)dmaengine; | ioat = to_ioat_softc(dmaengine); | ||||
if (ioat_reserve_space_and_lock(ioat, 1) != 0) | if (ioat_reserve_space_and_lock(ioat, 1) != 0) | ||||
return (NULL); | return (NULL); | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
desc = ioat_get_ring_entry(ioat, ioat->head); | desc = ioat_get_ring_entry(ioat, ioat->head); | ||||
hw_desc = desc->u.dma; | hw_desc = desc->u.dma; | ||||
hw_desc->u.control_raw = 0; | hw_desc->u.control_raw = 0; | ||||
hw_desc->u.control.null = 1; | hw_desc->u.control.null = 1; | ||||
hw_desc->u.control.completion_update = 1; | hw_desc->u.control.completion_update = 1; | ||||
if (flags & DMA_INT_EN) | if ((flags & DMA_INT_EN) != 0) | ||||
hw_desc->u.control.int_enable = 1; | hw_desc->u.control.int_enable = 1; | ||||
hw_desc->size = 8; | hw_desc->size = 8; | ||||
hw_desc->src_addr = 0; | hw_desc->src_addr = 0; | ||||
hw_desc->dest_addr = 0; | hw_desc->dest_addr = 0; | ||||
desc->bus_dmadesc.callback_fn = callback_fn; | desc->bus_dmadesc.callback_fn = callback_fn; | ||||
desc->bus_dmadesc.callback_arg = callback_arg; | desc->bus_dmadesc.callback_arg = callback_arg; | ||||
ioat_submit_single(ioat); | ioat_submit_single(ioat); | ||||
return (&desc->bus_dmadesc); | return (&desc->bus_dmadesc); | ||||
} | } | ||||
struct bus_dmadesc * | struct bus_dmadesc * | ||||
ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, | ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, | ||||
bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, | bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, | ||||
void *callback_arg, uint32_t flags) | void *callback_arg, uint32_t flags) | ||||
{ | { | ||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct ioat_dma_hw_descriptor *hw_desc; | struct ioat_dma_hw_descriptor *hw_desc; | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = (struct ioat_softc *)dmaengine; | ioat = to_ioat_softc(dmaengine); | ||||
if (len > ioat->max_xfer_size) | if (len > ioat->max_xfer_size) { | ||||
panic("ioat_copy: max_xfer_size = %d, requested = %d\n", | ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n", | ||||
ioat->max_xfer_size, (int)len); | __func__, ioat->max_xfer_size, (int)len); | ||||
return (NULL); | |||||
} | |||||
if (ioat_reserve_space_and_lock(ioat, 1) != 0) | if (ioat_reserve_space_and_lock(ioat, 1) != 0) | ||||
return (NULL); | return (NULL); | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
desc = ioat_get_ring_entry(ioat, ioat->head); | desc = ioat_get_ring_entry(ioat, ioat->head); | ||||
hw_desc = desc->u.dma; | hw_desc = desc->u.dma; | ||||
hw_desc->u.control_raw = 0; | hw_desc->u.control_raw = 0; | ||||
hw_desc->u.control.completion_update = 1; | hw_desc->u.control.completion_update = 1; | ||||
if (flags & DMA_INT_EN) | if ((flags & DMA_INT_EN) != 0) | ||||
hw_desc->u.control.int_enable = 1; | hw_desc->u.control.int_enable = 1; | ||||
hw_desc->size = len; | hw_desc->size = len; | ||||
hw_desc->src_addr = src; | hw_desc->src_addr = src; | ||||
hw_desc->dest_addr = dst; | hw_desc->dest_addr = dst; | ||||
#ifdef IOAT_LOGGING | |||||
if (g_ioat_debug_level >= 3) | if (g_ioat_debug_level >= 3) | ||||
dump_descriptor(hw_desc); | dump_descriptor(hw_desc); | ||||
#endif | |||||
desc->bus_dmadesc.callback_fn = callback_fn; | desc->bus_dmadesc.callback_fn = callback_fn; | ||||
desc->bus_dmadesc.callback_arg = callback_arg; | desc->bus_dmadesc.callback_arg = callback_arg; | ||||
ioat_submit_single(ioat); | ioat_submit_single(ioat); | ||||
return (&desc->bus_dmadesc); | return (&desc->bus_dmadesc); | ||||
} | } | ||||
/* | /* | ||||
* Ring Management | * Ring Management | ||||
*/ | */ | ||||
static inline uint32_t | static inline uint32_t | ||||
ioat_get_active(struct ioat_softc *ioat) | ioat_get_active(struct ioat_softc *ioat) | ||||
{ | { | ||||
return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1)); | return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1)); | ||||
} | } | ||||
static inline uint32_t | static inline uint32_t | ||||
ioat_get_ring_space(struct ioat_softc *ioat) | ioat_get_ring_space(struct ioat_softc *ioat) | ||||
{ | { | ||||
return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); | return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); | ||||
} | } | ||||
static struct ioat_descriptor * | static struct ioat_descriptor * | ||||
ioat_alloc_ring_entry(struct ioat_softc *ioat) | ioat_alloc_ring_entry(struct ioat_softc *ioat) | ||||
{ | { | ||||
struct ioat_descriptor *desc; | |||||
struct ioat_dma_hw_descriptor *hw_desc; | struct ioat_dma_hw_descriptor *hw_desc; | ||||
struct ioat_descriptor *desc; | |||||
desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT); | desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT); | ||||
if (desc == NULL) | if (desc == NULL) | ||||
return (NULL); | return (NULL); | ||||
bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO, | bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO, | ||||
&ioat->hw_desc_map); | &ioat->hw_desc_map); | ||||
if (hw_desc == NULL) { | if (hw_desc == NULL) { | ||||
free(desc, M_IOAT); | free(desc, M_IOAT); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, | bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, | ||||
sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0); | sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0); | ||||
desc->u.dma = hw_desc; | desc->u.dma = hw_desc; | ||||
return (desc); | return (desc); | ||||
} | } | ||||
static void | static void | ||||
ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) | ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) | ||||
{ | { | ||||
if (desc != NULL) { | if (desc == NULL) | ||||
return; | |||||
if (desc->u.dma) | if (desc->u.dma) | ||||
bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma, | bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma, | ||||
ioat->hw_desc_map); | ioat->hw_desc_map); | ||||
free(desc, M_IOAT); | free(desc, M_IOAT); | ||||
} | } | ||||
} | |||||
static int | static int | ||||
ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs) | ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs) | ||||
{ | { | ||||
int retry; | boolean_t retry; | ||||
do { | while (1) { | ||||
if (ioat_get_ring_space(ioat) >= num_descs) { | if (ioat_get_ring_space(ioat) >= num_descs) | ||||
return (0); | return (0); | ||||
} | |||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
retry = resize_ring(ioat, ioat->ring_size_order + 1); | retry = resize_ring(ioat, ioat->ring_size_order + 1); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
if (retry) | if (!retry) | ||||
continue; | |||||
else { | |||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
} while (1); | |||||
} | } | ||||
static struct ioat_descriptor * | static struct ioat_descriptor * | ||||
ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) | ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) | ||||
{ | { | ||||
return (ioat->ring[index % (1 << ioat->ring_size_order)]); | return (ioat->ring[index % (1 << ioat->ring_size_order)]); | ||||
} | } | ||||
Show All 19 Lines | resize_ring(struct ioat_softc *ioat, int order) | ||||
* when shrinking, verify that we can hold the current active | * when shrinking, verify that we can hold the current active | ||||
* set in the new ring | * set in the new ring | ||||
*/ | */ | ||||
if (active >= new_size) | if (active >= new_size) | ||||
return (FALSE); | return (FALSE); | ||||
/* allocate the array to hold the software ring */ | /* allocate the array to hold the software ring */ | ||||
ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT); | ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT); | ||||
if (ring == NULL) | |||||
if (!ring) | |||||
return (FALSE); | return (FALSE); | ||||
ioat_log_message(1, "ring resize: new: %d old: %d\n", | ioat_log_message(2, "ring resize: new: %d old: %d\n", | ||||
new_size, current_size); | new_size, current_size); | ||||
/* allocate/trim descriptors as needed */ | /* allocate/trim descriptors as needed */ | ||||
if (new_size > current_size) { | if (new_size > current_size) { | ||||
/* copy current descriptors to the new ring */ | /* copy current descriptors to the new ring */ | ||||
for (i = 0; i < current_size; i++) { | for (i = 0; i < current_size; i++) { | ||||
current_idx = (ioat->tail+i) & (current_size-1); | current_idx = (ioat->tail + i) & (current_size - 1); | ||||
new_idx = (ioat->tail+i) & (new_size-1); | new_idx = (ioat->tail + i) & (new_size - 1); | ||||
ring[new_idx] = ioat->ring[current_idx]; | ring[new_idx] = ioat->ring[current_idx]; | ||||
ring[new_idx]->id = new_idx; | ring[new_idx]->id = new_idx; | ||||
} | } | ||||
/* add new descriptors to the ring */ | /* add new descriptors to the ring */ | ||||
for (i = current_size; i < new_size; i++) { | for (i = current_size; i < new_size; i++) { | ||||
new_idx = (ioat->tail+i) & (new_size-1); | new_idx = (ioat->tail + i) & (new_size - 1); | ||||
ring[new_idx] = ioat_alloc_ring_entry(ioat); | ring[new_idx] = ioat_alloc_ring_entry(ioat); | ||||
if (!ring[new_idx]) { | if (!ring[new_idx]) { | ||||
while (i--) { | while (i--) { | ||||
new_idx2 = (ioat->tail+i) & | new_idx2 = (ioat->tail + i) & | ||||
(new_size-1); | (new_size - 1); | ||||
ioat_free_ring_entry(ioat, | ioat_free_ring_entry(ioat, | ||||
ring[new_idx2]); | ring[new_idx2]); | ||||
} | } | ||||
free(ring, M_IOAT); | free(ring, M_IOAT); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
ring[new_idx]->id = new_idx; | ring[new_idx]->id = new_idx; | ||||
} | } | ||||
for (i = current_size-1; i < new_size; i++) { | for (i = current_size - 1; i < new_size; i++) { | ||||
new_idx = (ioat->tail+i) & (new_size-1); | new_idx = (ioat->tail + i) & (new_size - 1); | ||||
next = ring[(new_idx+1) & (new_size-1)]; | next = ring[(new_idx + 1) & (new_size - 1)]; | ||||
hw = ring[new_idx]->u.dma; | hw = ring[new_idx]->u.dma; | ||||
hw->next = next->hw_desc_bus_addr; | hw->next = next->hw_desc_bus_addr; | ||||
} | } | ||||
} else { | } else { | ||||
/* | /* | ||||
* copy current descriptors to the new ring, dropping the | * copy current descriptors to the new ring, dropping the | ||||
* removed descriptors | * removed descriptors | ||||
*/ | */ | ||||
for (i = 0; i < new_size; i++) { | for (i = 0; i < new_size; i++) { | ||||
uint32_t curr_idx = (ioat->tail+i) & (current_size-1); | current_idx = (ioat->tail + i) & (current_size - 1); | ||||
uint32_t new_idx = (ioat->tail+i) & (new_size-1); | new_idx = (ioat->tail + i) & (new_size - 1); | ||||
ring[new_idx] = ioat->ring[curr_idx]; | ring[new_idx] = ioat->ring[current_idx]; | ||||
ring[new_idx]->id = new_idx; | ring[new_idx]->id = new_idx; | ||||
} | } | ||||
/* free deleted descriptors */ | /* free deleted descriptors */ | ||||
for (i = new_size; i < current_size; i++) { | for (i = new_size; i < current_size; i++) { | ||||
ent = ioat_get_ring_entry(ioat, ioat->tail+i); | ent = ioat_get_ring_entry(ioat, ioat->tail + i); | ||||
ioat_free_ring_entry(ioat, ent); | ioat_free_ring_entry(ioat, ent); | ||||
} | } | ||||
/* fix up hardware ring */ | /* fix up hardware ring */ | ||||
hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->u.dma; | hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma; | ||||
next = ring[(ioat->tail+new_size) & (new_size-1)]; | next = ring[(ioat->tail + new_size) & (new_size - 1)]; | ||||
hw->next = next->hw_desc_bus_addr; | hw->next = next->hw_desc_bus_addr; | ||||
} | } | ||||
free(ioat->ring, M_IOAT); | free(ioat->ring, M_IOAT); | ||||
ioat->ring = ring; | ioat->ring = ring; | ||||
ioat->ring_size_order = order; | ioat->ring_size_order = order; | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
static void | static void | ||||
ioat_timer_callback(void *arg) | ioat_timer_callback(void *arg) | ||||
{ | { | ||||
struct ioat_softc *ioat = arg; | |||||
uint64_t status; | |||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct ioat_softc *ioat; | |||||
uint64_t status; | |||||
uint32_t chanerr; | uint32_t chanerr; | ||||
ioat = arg; | |||||
ioat_log_message(2, "%s\n", __func__); | ioat_log_message(2, "%s\n", __func__); | ||||
if (ioat->is_completion_pending) { | if (ioat->is_completion_pending) { | ||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
/* | /* | ||||
* when halted due to errors check for channel | * When halted due to errors, check for channel programming | ||||
* programming errors before advancing the completion state | * errors before advancing the completion state. | ||||
*/ | */ | ||||
if (is_ioat_halted(status)) { | if (is_ioat_halted(status)) { | ||||
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | ||||
ioat_log_message(0, "Channel halted (%x)\n", chanerr); | ioat_log_message(0, "Channel halted (%x)\n", chanerr); | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail+0); | desc = ioat_get_ring_entry(ioat, ioat->tail + 0); | ||||
dump_descriptor(desc->u.raw); | dump_descriptor(desc->u.raw); | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail+1); | desc = ioat_get_ring_entry(ioat, ioat->tail + 1); | ||||
dump_descriptor(desc->u.raw); | dump_descriptor(desc->u.raw); | ||||
} | } | ||||
ioat_process_events(ioat); | ioat_process_events(ioat); | ||||
} else { | } else { | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
if (ioat_get_active(ioat) == 0 && | if (ioat_get_active(ioat) == 0 && | ||||
ioat->ring_size_order > IOAT_MIN_ORDER) | ioat->ring_size_order > IOAT_MIN_ORDER) | ||||
resize_ring(ioat, ioat->ring_size_order-1); | resize_ring(ioat, ioat->ring_size_order - 1); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
if (ioat->ring_size_order > IOAT_MIN_ORDER) { | if (ioat->ring_size_order > IOAT_MIN_ORDER) | ||||
callout_reset(&ioat->timer, 5000*hz/1000, | callout_reset(&ioat->timer, 5 * hz, | ||||
ioat_timer_callback, ioat); | ioat_timer_callback, ioat); | ||||
} | } | ||||
} | } | ||||
} | |||||
/* | /* | ||||
* Support Functions | * Support Functions | ||||
*/ | */ | ||||
static void | static void | ||||
ioat_submit_single(struct ioat_softc *ioat) | ioat_submit_single(struct ioat_softc *ioat) | ||||
{ | { | ||||
atomic_add_rel_int(&ioat->head, 1); | atomic_add_rel_int(&ioat->head, 1); | ||||
if (!ioat->is_completion_pending) { | if (!ioat->is_completion_pending) { | ||||
ioat->is_completion_pending = TRUE; | ioat->is_completion_pending = TRUE; | ||||
callout_reset(&ioat->timer, 10000*hz/1000, ioat_timer_callback, | callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback, | ||||
ioat); | ioat); | ||||
} | } | ||||
} | } | ||||
static int ioat_reset_hw(struct ioat_softc *ioat) | static int | ||||
ioat_reset_hw(struct ioat_softc *ioat) | |||||
{ | { | ||||
int timeout = 20; /* in milliseconds */ | |||||
uint64_t status; | uint64_t status; | ||||
uint32_t chanerr; | uint32_t chanerr; | ||||
int timeout; | |||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
if (is_ioat_active(status) || is_ioat_idle(status)) | if (is_ioat_active(status) || is_ioat_idle(status)) | ||||
ioat_suspend(ioat); | ioat_suspend(ioat); | ||||
while (is_ioat_active(status) || is_ioat_idle(status)) { | /* Wait at most 20 ms */ | ||||
for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) && | |||||
timeout < 20; timeout++) { | |||||
DELAY(1000); | DELAY(1000); | ||||
timeout--; | |||||
if (timeout == 0) | |||||
return (-1); | |||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
} | } | ||||
if (timeout == 20) | |||||
return (ETIMEDOUT); | |||||
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | ||||
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); | ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); | ||||
/* | /* | ||||
* IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors | * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors | ||||
* that can cause stability issues for IOAT v3. | * that can cause stability issues for IOAT v3. | ||||
*/ | */ | ||||
pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07, | pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07, | ||||
4); | 4); | ||||
chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4); | chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4); | ||||
pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4); | pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4); | ||||
ioat_reset(ioat); | ioat_reset(ioat); | ||||
timeout = 20; | /* Wait at most 20 ms */ | ||||
for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++) | |||||
while (ioat_reset_pending(ioat)) { | |||||
DELAY(1000); | DELAY(1000); | ||||
timeout--; | if (timeout == 20) | ||||
if (timeout == 0) | return (ETIMEDOUT); | ||||
return (-1); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
dump_descriptor(void *hw_desc) | dump_descriptor(void *hw_desc) | ||||
{ | { | ||||
int i, j; | int i, j; | ||||
for (i = 0; i < 2; i++) { | for (i = 0; i < 2; i++) { | ||||
for (j = 0; j < 8; j++) | for (j = 0; j < 8; j++) | ||||
printf("%08x ", ((uint32_t *)hw_desc)[i*8+j]); | printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]); | ||||
printf("\n"); | printf("\n"); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
ioat_setup_sysctl(device_t device) | ioat_setup_sysctl(device_t device) | ||||
{ | { | ||||
struct ioat_softc *ioat = DEVICE2SOFTC(device); | struct sysctl_ctx_list *sysctl_ctx; | ||||
struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(device); | struct sysctl_oid *sysctl_tree; | ||||
struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(device); | struct ioat_softc *ioat; | ||||
ioat = DEVICE2SOFTC(device); | |||||
sysctl_ctx = device_get_sysctl_ctx(device); | |||||
sysctl_tree = device_get_sysctl_tree(device); | |||||
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | ||||
"ring_size_order", CTLFLAG_RD, &ioat->ring_size_order, | "ring_size_order", CTLFLAG_RD, &ioat->ring_size_order, | ||||
0, "HW descriptor ring size order"); | 0, "HW descriptor ring size order"); | ||||
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | ||||
"head", CTLFLAG_RD, &ioat->head, | "head", CTLFLAG_RD, &ioat->head, | ||||
0, "HW descriptor head pointer index"); | 0, "HW descriptor head pointer index"); | ||||
SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, | ||||
"tail", CTLFLAG_RD, &ioat->tail, | "tail", CTLFLAG_RD, &ioat->tail, | ||||
0, "HW descriptor tail pointer index"); | 0, "HW descriptor tail pointer index"); | ||||
} | } |
I don't think this works as a sysctl. This value is needed when the driver loads, to determine whether or not to force legacy interrupt usage. By time the sysctl is available to force legacy interrupt usage, it will be too late.