Changeset View
Standalone View
sys/dev/ioat/ioat.c
Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | |||||
#include "ioat.h" | #include "ioat.h" | ||||
#include "ioat_hw.h" | #include "ioat_hw.h" | ||||
#include "ioat_internal.h" | #include "ioat_internal.h" | ||||
#ifndef BUS_SPACE_MAXADDR_40BIT | #ifndef BUS_SPACE_MAXADDR_40BIT | ||||
#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL | #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL | ||||
#endif | #endif | ||||
#define IOAT_INTR_TIMO (hz / 10) | |||||
#define IOAT_REFLK (&ioat->submit_lock) | #define IOAT_REFLK (&ioat->submit_lock) | ||||
#define IOAT_SHRINK_PERIOD (10 * hz) | |||||
static int ioat_probe(device_t device); | static int ioat_probe(device_t device); | ||||
static int ioat_attach(device_t device); | static int ioat_attach(device_t device); | ||||
static int ioat_detach(device_t device); | static int ioat_detach(device_t device); | ||||
static int ioat_setup_intr(struct ioat_softc *ioat); | static int ioat_setup_intr(struct ioat_softc *ioat); | ||||
static int ioat_teardown_intr(struct ioat_softc *ioat); | static int ioat_teardown_intr(struct ioat_softc *ioat); | ||||
static int ioat3_attach(device_t device); | static int ioat3_attach(device_t device); | ||||
static int ioat_start_channel(struct ioat_softc *ioat); | static int ioat_start_channel(struct ioat_softc *ioat); | ||||
Show All 17 Lines | static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat, | ||||
uint32_t index); | uint32_t index); | ||||
static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *, | static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *, | ||||
uint32_t size, boolean_t need_dscr, int mflags); | uint32_t size, boolean_t need_dscr, int mflags); | ||||
static int ring_grow(struct ioat_softc *, uint32_t oldorder, | static int ring_grow(struct ioat_softc *, uint32_t oldorder, | ||||
struct ioat_descriptor **); | struct ioat_descriptor **); | ||||
static int ring_shrink(struct ioat_softc *, uint32_t oldorder, | static int ring_shrink(struct ioat_softc *, uint32_t oldorder, | ||||
struct ioat_descriptor **); | struct ioat_descriptor **); | ||||
static void ioat_halted_debug(struct ioat_softc *, uint32_t); | static void ioat_halted_debug(struct ioat_softc *, uint32_t); | ||||
static void ioat_timer_callback(void *arg); | static void ioat_poll_timer_callback(void *arg); | ||||
static void ioat_shrink_timer_callback(void *arg); | |||||
static void dump_descriptor(void *hw_desc); | static void dump_descriptor(void *hw_desc); | ||||
static void ioat_submit_single(struct ioat_softc *ioat); | static void ioat_submit_single(struct ioat_softc *ioat); | ||||
static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | ||||
int error); | int error); | ||||
static int ioat_reset_hw(struct ioat_softc *ioat); | static int ioat_reset_hw(struct ioat_softc *ioat); | ||||
static void ioat_reset_hw_task(void *, int); | static void ioat_reset_hw_task(void *, int); | ||||
static void ioat_setup_sysctl(device_t device); | static void ioat_setup_sysctl(device_t device); | ||||
static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); | ||||
▲ Show 20 Lines • Show All 212 Lines • ▼ Show 20 Lines | ioat_detach(device_t device) | ||||
ioat_test_detach(); | ioat_test_detach(); | ||||
taskqueue_drain(taskqueue_thread, &ioat->reset_task); | taskqueue_drain(taskqueue_thread, &ioat->reset_task); | ||||
mtx_lock(IOAT_REFLK); | mtx_lock(IOAT_REFLK); | ||||
ioat->quiescing = TRUE; | ioat->quiescing = TRUE; | ||||
ioat->destroying = TRUE; | ioat->destroying = TRUE; | ||||
wakeup(&ioat->quiescing); | wakeup(&ioat->quiescing); | ||||
wakeup(&ioat->resetting); | |||||
ioat_channel[ioat->chan_idx] = NULL; | ioat_channel[ioat->chan_idx] = NULL; | ||||
ioat_drain_locked(ioat); | ioat_drain_locked(ioat); | ||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(IOAT_REFLK); | ||||
ioat_teardown_intr(ioat); | ioat_teardown_intr(ioat); | ||||
callout_drain(&ioat->timer); | callout_drain(&ioat->poll_timer); | ||||
callout_drain(&ioat->shrink_timer); | |||||
pci_disable_busmaster(device); | pci_disable_busmaster(device); | ||||
if (ioat->pci_resource != NULL) | if (ioat->pci_resource != NULL) | ||||
bus_release_resource(device, SYS_RES_MEMORY, | bus_release_resource(device, SYS_RES_MEMORY, | ||||
ioat->pci_resource_id, ioat->pci_resource); | ioat->pci_resource_id, ioat->pci_resource); | ||||
if (ioat->ring != NULL) | if (ioat->ring != NULL) | ||||
Show All 24 Lines | ioat_teardown_intr(struct ioat_softc *ioat) | ||||
pci_release_msi(ioat->device); | pci_release_msi(ioat->device); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ioat_start_channel(struct ioat_softc *ioat) | ioat_start_channel(struct ioat_softc *ioat) | ||||
{ | { | ||||
struct ioat_dma_hw_descriptor *hw_desc; | |||||
struct ioat_descriptor *desc; | |||||
struct bus_dmadesc *dmadesc; | |||||
uint64_t status; | uint64_t status; | ||||
uint32_t chanerr; | uint32_t chanerr; | ||||
int i; | int i; | ||||
ioat_acquire(&ioat->dmaengine); | ioat_acquire(&ioat->dmaengine); | ||||
ioat_null(&ioat->dmaengine, NULL, NULL, 0); | |||||
markj: ioat_null() doesn't seem to be used anymore, FWIW. | |||||
Not Done Inline ActionsIt's a public KPI/KBI. I suppose it's pretty useless and we could probably yank it despite the KBI freeze. cem: It's a public KPI/KBI. I suppose it's pretty useless and we could probably yank it despite the… | |||||
Not Done Inline ActionsYeah. I wouldn't use the KBI freeze as a justification for keeping it, but it's up to you. markj: Yeah. I wouldn't use the KBI freeze as a justification for keeping it, but it's up to you. | |||||
/* Submit 'NULL' operation manually to avoid quiescing flag */ | |||||
desc = ioat_get_ring_entry(ioat, ioat->head); | |||||
Done Inline ActionsThis should be ->head. The ring should be empty here and they should be identical, but semantically we submit at the head. cem: This should be `->head`. The ring should be empty here and they should be identical, but… | |||||
dmadesc = &desc->bus_dmadesc; | |||||
hw_desc = desc->u.dma; | |||||
dmadesc->callback_fn = NULL; | |||||
dmadesc->callback_arg = NULL; | |||||
hw_desc->u.control_raw = 0; | |||||
hw_desc->u.control_generic.op = IOAT_OP_COPY; | |||||
hw_desc->u.control_generic.completion_update = 1; | |||||
hw_desc->size = 8; | |||||
hw_desc->src_addr = 0; | |||||
hw_desc->dest_addr = 0; | |||||
hw_desc->u.control.null = 1; | |||||
ioat_submit_single(ioat); | |||||
ioat_release(&ioat->dmaengine); | ioat_release(&ioat->dmaengine); | ||||
for (i = 0; i < 100; i++) { | for (i = 0; i < 100; i++) { | ||||
DELAY(1); | DELAY(1); | ||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
if (is_ioat_idle(status)) | if (is_ioat_idle(status)) | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 33 Lines | ioat->intrdelay_supported = (ioat_read_2(ioat, IOAT_INTRDELAY_OFFSET) & | ||||
IOAT_INTRDELAY_SUPPORTED) != 0; | IOAT_INTRDELAY_SUPPORTED) != 0; | ||||
if (ioat->intrdelay_supported) | if (ioat->intrdelay_supported) | ||||
ioat->intrdelay_max = IOAT_INTRDELAY_US_MASK; | ioat->intrdelay_max = IOAT_INTRDELAY_US_MASK; | ||||
/* TODO: need to check DCA here if we ever do XOR/PQ */ | /* TODO: need to check DCA here if we ever do XOR/PQ */ | ||||
mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | ||||
mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); | mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); | ||||
callout_init(&ioat->timer, 1); | callout_init(&ioat->poll_timer, 1); | ||||
callout_init(&ioat->shrink_timer, 1); | |||||
TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); | TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); | ||||
/* Establish lock order for Witness */ | /* Establish lock order for Witness */ | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | for (i = 0; i < num_descriptors - 1; i++) { | ||||
dma_hw_desc->next = next->hw_desc_bus_addr; | dma_hw_desc->next = next->hw_desc_bus_addr; | ||||
} | } | ||||
ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; | ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; | ||||
ioat->head = ioat->hw_head = 0; | ioat->head = ioat->hw_head = 0; | ||||
ioat->tail = 0; | ioat->tail = 0; | ||||
ioat->last_seen = 0; | ioat->last_seen = 0; | ||||
*ioat->comp_update = 0; | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ioat_map_pci_bar(struct ioat_softc *ioat) | ioat_map_pci_bar(struct ioat_softc *ioat) | ||||
{ | { | ||||
ioat->pci_resource_id = PCIR_BAR(0); | ioat->pci_resource_id = PCIR_BAR(0); | ||||
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
ioat_process_events(struct ioat_softc *ioat) | ioat_process_events(struct ioat_softc *ioat) | ||||
{ | { | ||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct bus_dmadesc *dmadesc; | struct bus_dmadesc *dmadesc; | ||||
uint64_t comp_update, status; | uint64_t comp_update, status; | ||||
uint32_t completed, chanerr; | uint32_t completed, chanerr; | ||||
boolean_t pending; | |||||
int error; | int error; | ||||
CTR0(KTR_IOAT, __func__); | |||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
/* | |||||
* Don't run while the hardware is being reset. Reset is responsible | |||||
* for blocking new work and draining & completing existing work, so | |||||
* there is nothing to do until new work is queued after reset anyway. | |||||
*/ | |||||
if (ioat->resetting_cleanup) { | |||||
Not Done Inline ActionsI don't really see why we have the resetting and resetting_cleanup flags. resetting_cleanup is only ever set when resetting is true, and the resetting and quiescing flags seem to be very similar. markj: I don't really see why we have the resetting and resetting_cleanup flags. resetting_cleanup is… | |||||
Not Done Inline ActionsI didn't want to overload quiescing more than it already is. Also, we need to quiesce submissions before we block cleanup. The two resetting variables are protected by different locks. It means we can avoid taking the head lock in the cleanup (tail) thread. cem: I didn't want to overload quiescing more than it already is. Also, we need to quiesce… | |||||
Not Done Inline ActionsHm. I just noticed that ioat_process_events() is called from an interrupt filter, which means that it runs in hard interrupt context and thus isn't allowed to acquire regular mutexes at all - only spin mutexes. I thought witness would catch that though, so I might be missing something. markj: Hm. I just noticed that ioat_process_events() is called from an interrupt filter, which means… | |||||
Not Done Inline ActionsOh, never mind. I mixed the ithread and filter arguments to bus_setup_intr(). markj: Oh, never mind. I mixed the ithread and filter arguments to bus_setup_intr(). | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
return; | |||||
} | |||||
completed = 0; | completed = 0; | ||||
comp_update = *ioat->comp_update; | comp_update = *ioat->comp_update; | ||||
status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; | status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; | ||||
CTR0(KTR_IOAT, __func__); | |||||
if (status == ioat->last_seen) { | if (status == ioat->last_seen) { | ||||
/* | /* | ||||
* If we landed in process_events and nothing has been | * If we landed in process_events and nothing has been | ||||
* completed, check for a timeout due to channel halt. | * completed, check for a timeout due to channel halt. | ||||
*/ | */ | ||||
comp_update = ioat_get_chansts(ioat); | comp_update = ioat_get_chansts(ioat); | ||||
goto out; | goto out; | ||||
} | } | ||||
while (1) { | while (1) { | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail); | desc = ioat_get_ring_entry(ioat, ioat->tail); | ||||
dmadesc = &desc->bus_dmadesc; | dmadesc = &desc->bus_dmadesc; | ||||
CTR1(KTR_IOAT, "completing desc %d", ioat->tail); | CTR1(KTR_IOAT, "completing desc %d", ioat->tail); | ||||
if (dmadesc->callback_fn != NULL) | if (dmadesc->callback_fn != NULL) | ||||
dmadesc->callback_fn(dmadesc->callback_arg, 0); | dmadesc->callback_fn(dmadesc->callback_arg, 0); | ||||
completed++; | completed++; | ||||
ioat->tail++; | ioat->tail++; | ||||
if (desc->hw_desc_bus_addr == status) | if (desc->hw_desc_bus_addr == status) | ||||
break; | break; | ||||
} | } | ||||
ioat->last_seen = desc->hw_desc_bus_addr; | ioat->last_seen = desc->hw_desc_bus_addr; | ||||
if (ioat->head == ioat->tail) { | |||||
ioat->is_completion_pending = FALSE; | |||||
callout_reset(&ioat->timer, IOAT_INTR_TIMO, | |||||
ioat_timer_callback, ioat); | |||||
} | |||||
ioat->stats.descriptors_processed += completed; | ioat->stats.descriptors_processed += completed; | ||||
out: | out: | ||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
/* Perform a racy check first; only take the locks if it passes. */ | |||||
pending = (ioat_get_active(ioat) != 0); | |||||
if (!pending && ioat->is_completion_pending) { | |||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
mtx_lock(&ioat->submit_lock); | |||||
mtx_lock(&ioat->cleanup_lock); | |||||
pending = (ioat_get_active(ioat) != 0); | |||||
if (!pending && ioat->is_completion_pending) { | |||||
ioat->is_completion_pending = FALSE; | |||||
callout_reset(&ioat->shrink_timer, IOAT_SHRINK_PERIOD, | |||||
ioat_shrink_timer_callback, ioat); | |||||
callout_stop(&ioat->poll_timer); | |||||
} | |||||
mtx_unlock(&ioat->submit_lock); | |||||
} | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
if (pending) | |||||
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, | |||||
ioat); | |||||
if (completed != 0) { | if (completed != 0) { | ||||
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); | ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); | ||||
wakeup(&ioat->tail); | wakeup(&ioat->tail); | ||||
} | } | ||||
if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) | if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 905 Lines • ▼ Show 20 Lines | ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr) | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail + 0); | desc = ioat_get_ring_entry(ioat, ioat->tail + 0); | ||||
dump_descriptor(desc->u.raw); | dump_descriptor(desc->u.raw); | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail + 1); | desc = ioat_get_ring_entry(ioat, ioat->tail + 1); | ||||
dump_descriptor(desc->u.raw); | dump_descriptor(desc->u.raw); | ||||
} | } | ||||
static void | static void | ||||
ioat_timer_callback(void *arg) | ioat_poll_timer_callback(void *arg) | ||||
{ | { | ||||
struct ioat_softc *ioat; | |||||
ioat = arg; | |||||
ioat_log_message(3, "%s\n", __func__); | |||||
ioat_process_events(ioat); | |||||
} | |||||
static void | |||||
ioat_shrink_timer_callback(void *arg) | |||||
{ | |||||
struct ioat_descriptor **newring; | struct ioat_descriptor **newring; | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
uint32_t order; | uint32_t order; | ||||
ioat = arg; | ioat = arg; | ||||
ioat_log_message(1, "%s\n", __func__); | ioat_log_message(1, "%s\n", __func__); | ||||
if (ioat->is_completion_pending) { | /* Slowly scale the ring down if idle. */ | ||||
ioat_process_events(ioat); | mtx_lock(&ioat->submit_lock); | ||||
/* Don't run while the hardware is being reset. */ | |||||
if (ioat->resetting) { | |||||
mtx_unlock(&ioat->submit_lock); | |||||
return; | return; | ||||
} | } | ||||
/* Slowly scale the ring down if idle. */ | |||||
mtx_lock(&ioat->submit_lock); | |||||
order = ioat->ring_size_order; | order = ioat->ring_size_order; | ||||
if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) { | if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) { | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
goto out; | goto out; | ||||
} | } | ||||
ioat->is_resize_pending = TRUE; | ioat->is_resize_pending = TRUE; | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE, | newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE, | ||||
M_NOWAIT); | M_NOWAIT); | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
KASSERT(ioat->ring_size_order == order, | KASSERT(ioat->ring_size_order == order, | ||||
("resize_pending protects order")); | ("resize_pending protects order")); | ||||
if (newring != NULL) | if (newring != NULL) | ||||
ring_shrink(ioat, order, newring); | ring_shrink(ioat, order, newring); | ||||
ioat->is_resize_pending = FALSE; | ioat->is_resize_pending = FALSE; | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
out: | out: | ||||
if (ioat->ring_size_order > IOAT_MIN_ORDER) | if (ioat->ring_size_order > IOAT_MIN_ORDER) | ||||
callout_reset(&ioat->timer, 10 * hz, | callout_reset(&ioat->poll_timer, IOAT_SHRINK_PERIOD, | ||||
ioat_timer_callback, ioat); | ioat_shrink_timer_callback, ioat); | ||||
} | } | ||||
/* | /* | ||||
* Support Functions | * Support Functions | ||||
*/ | */ | ||||
static void | static void | ||||
ioat_submit_single(struct ioat_softc *ioat) | ioat_submit_single(struct ioat_softc *ioat) | ||||
{ | { | ||||
ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); | ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); | ||||
atomic_add_rel_int(&ioat->head, 1); | atomic_add_rel_int(&ioat->head, 1); | ||||
atomic_add_rel_int(&ioat->hw_head, 1); | atomic_add_rel_int(&ioat->hw_head, 1); | ||||
if (!ioat->is_completion_pending) { | if (!ioat->is_completion_pending) { | ||||
ioat->is_completion_pending = TRUE; | ioat->is_completion_pending = TRUE; | ||||
callout_reset(&ioat->timer, IOAT_INTR_TIMO, | callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, | ||||
ioat_timer_callback, ioat); | ioat); | ||||
callout_stop(&ioat->shrink_timer); | |||||
} | } | ||||
ioat->stats.descriptors_submitted++; | ioat->stats.descriptors_submitted++; | ||||
} | } | ||||
static int | static int | ||||
ioat_reset_hw(struct ioat_softc *ioat) | ioat_reset_hw(struct ioat_softc *ioat) | ||||
{ | { | ||||
uint64_t status; | uint64_t status; | ||||
uint32_t chanerr; | uint32_t chanerr; | ||||
unsigned timeout; | unsigned timeout; | ||||
int error; | int error; | ||||
mtx_lock(IOAT_REFLK); | mtx_lock(IOAT_REFLK); | ||||
while (ioat->resetting && !ioat->destroying) | |||||
msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0); | |||||
if (ioat->destroying) { | |||||
mtx_unlock(IOAT_REFLK); | |||||
return (ENXIO); | |||||
} | |||||
ioat->resetting = TRUE; | |||||
ioat->quiescing = TRUE; | ioat->quiescing = TRUE; | ||||
ioat_drain_locked(ioat); | ioat_drain_locked(ioat); | ||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(IOAT_REFLK); | ||||
/* | |||||
* Suspend ioat_process_events while the hardware and softc are in an | |||||
* indeterminate state. | |||||
*/ | |||||
mtx_lock(&ioat->cleanup_lock); | |||||
ioat->resetting_cleanup = TRUE; | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
if (is_ioat_active(status) || is_ioat_idle(status)) | if (is_ioat_active(status) || is_ioat_idle(status)) | ||||
ioat_suspend(ioat); | ioat_suspend(ioat); | ||||
/* Wait at most 20 ms */ | /* Wait at most 20 ms */ | ||||
for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) && | for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) && | ||||
timeout < 20; timeout++) { | timeout < 20; timeout++) { | ||||
DELAY(1000); | DELAY(1000); | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | ioat_reset_hw(struct ioat_softc *ioat) | ||||
* Bring device back online after reset. Writing CHAINADDR brings the | * Bring device back online after reset. Writing CHAINADDR brings the | ||||
* device back to active. | * device back to active. | ||||
* | * | ||||
* The internal ring counter resets to zero, so we have to start over | * The internal ring counter resets to zero, so we have to start over | ||||
* at zero as well. | * at zero as well. | ||||
*/ | */ | ||||
ioat->tail = ioat->head = ioat->hw_head = 0; | ioat->tail = ioat->head = ioat->hw_head = 0; | ||||
ioat->last_seen = 0; | ioat->last_seen = 0; | ||||
*ioat->comp_update = 0; | |||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ||||
ioat_write_chainaddr(ioat, ioat->ring[0]->hw_desc_bus_addr); | ioat_write_chainaddr(ioat, ioat->ring[0]->hw_desc_bus_addr); | ||||
error = 0; | error = 0; | ||||
out: | out: | ||||
/* | |||||
* Resume completions now that ring state is consistent. | |||||
* ioat_start_channel will add a pending completion and if we are still | |||||
* blocking completions, we may livelock. | |||||
*/ | |||||
mtx_lock(&ioat->cleanup_lock); | |||||
Done Inline ActionsI don't see anything that sleeps on this? markj: I don't see anything that sleeps on this? | |||||
Not Done Inline ActionsNothing does. But a spurious wakeup call doesn't hurt anything and is sort of consistent with our other state flags. cem: Nothing does. But a spurious wakeup call doesn't hurt anything and is sort of consistent with… | |||||
Not Done Inline ActionsOk. markj: Ok. | |||||
ioat->resetting_cleanup = FALSE; | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
/* Enqueues a null operation and ensures it completes. */ | |||||
if (error == 0) | |||||
error = ioat_start_channel(ioat); | |||||
/* Unblock submission of new work */ | |||||
mtx_lock(IOAT_REFLK); | mtx_lock(IOAT_REFLK); | ||||
ioat->quiescing = FALSE; | ioat->quiescing = FALSE; | ||||
wakeup(&ioat->quiescing); | wakeup(&ioat->quiescing); | ||||
ioat->resetting = FALSE; | |||||
wakeup(&ioat->resetting); | |||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(IOAT_REFLK); | ||||
if (error == 0) | |||||
error = ioat_start_channel(ioat); | |||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
sysctl_handle_chansts(SYSCTL_HANDLER_ARGS) | sysctl_handle_chansts(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
struct sbuf sb; | struct sbuf sb; | ||||
▲ Show 20 Lines • Show All 336 Lines • ▼ Show 20 Lines | DB_SHOW_COMMAND(ioat, db_show_ioat) | ||||
db_printf(" submit_lock: "); | db_printf(" submit_lock: "); | ||||
db_show_lock(&sc->submit_lock); | db_show_lock(&sc->submit_lock); | ||||
db_printf(" capabilities: %b\n", (int)sc->capabilities, | db_printf(" capabilities: %b\n", (int)sc->capabilities, | ||||
IOAT_DMACAP_STR); | IOAT_DMACAP_STR); | ||||
db_printf(" cached_intrdelay: %u\n", sc->cached_intrdelay); | db_printf(" cached_intrdelay: %u\n", sc->cached_intrdelay); | ||||
db_printf(" *comp_update: 0x%jx\n", (uintmax_t)*sc->comp_update); | db_printf(" *comp_update: 0x%jx\n", (uintmax_t)*sc->comp_update); | ||||
db_printf(" timer:\n"); | db_printf(" poll_timer:\n"); | ||||
db_printf(" c_time: %ju\n", (uintmax_t)sc->timer.c_time); | db_printf(" c_time: %ju\n", (uintmax_t)sc->poll_timer.c_time); | ||||
db_printf(" c_arg: %p\n", sc->timer.c_arg); | db_printf(" c_arg: %p\n", sc->poll_timer.c_arg); | ||||
db_printf(" c_func: %p\n", sc->timer.c_func); | db_printf(" c_func: %p\n", sc->poll_timer.c_func); | ||||
db_printf(" c_lock: %p\n", sc->timer.c_lock); | db_printf(" c_lock: %p\n", sc->poll_timer.c_lock); | ||||
db_printf(" c_flags: 0x%x\n", (unsigned)sc->timer.c_flags); | db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags); | ||||
db_printf(" shrink_timer:\n"); | |||||
db_printf(" c_time: %ju\n", (uintmax_t)sc->shrink_timer.c_time); | |||||
db_printf(" c_arg: %p\n", sc->shrink_timer.c_arg); | |||||
db_printf(" c_func: %p\n", sc->shrink_timer.c_func); | |||||
db_printf(" c_lock: %p\n", sc->shrink_timer.c_lock); | |||||
db_printf(" c_flags: 0x%x\n", (unsigned)sc->shrink_timer.c_flags); | |||||
db_printf(" quiescing: %d\n", (int)sc->quiescing); | db_printf(" quiescing: %d\n", (int)sc->quiescing); | ||||
db_printf(" destroying: %d\n", (int)sc->destroying); | db_printf(" destroying: %d\n", (int)sc->destroying); | ||||
db_printf(" is_resize_pending: %d\n", (int)sc->is_resize_pending); | db_printf(" is_resize_pending: %d\n", (int)sc->is_resize_pending); | ||||
db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending); | db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending); | ||||
db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending); | db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending); | ||||
db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running); | db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running); | ||||
db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported); | db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported); | ||||
db_printf(" resetting: %d\n", (int)sc->resetting); | |||||
db_printf(" head: %u\n", sc->head); | db_printf(" head: %u\n", sc->head); | ||||
db_printf(" tail: %u\n", sc->tail); | db_printf(" tail: %u\n", sc->tail); | ||||
db_printf(" hw_head: %u\n", sc->hw_head); | db_printf(" hw_head: %u\n", sc->hw_head); | ||||
db_printf(" ring_size_order: %u\n", sc->ring_size_order); | db_printf(" ring_size_order: %u\n", sc->ring_size_order); | ||||
db_printf(" last_seen: 0x%lx\n", sc->last_seen); | db_printf(" last_seen: 0x%lx\n", sc->last_seen); | ||||
db_printf(" ring: %p\n", sc->ring); | db_printf(" ring: %p\n", sc->ring); | ||||
▲ Show 20 Lines • Show All 41 Lines • Show Last 20 Lines |
ioat_null() doesn't seem to be used anymore, FWIW.