Changeset View
Standalone View
sys/dev/ioat/ioat.c
/*- | /*- | ||||
* Copyright (C) 2012 Intel Corporation | * Copyright (C) 2012 Intel Corporation | ||||
* All rights reserved. | * All rights reserved. | ||||
* Copyright (C) 2018 Alexander Motin <mav@FreeBSD.org> | |||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
* notice, this list of conditions and the following disclaimer. | * notice, this list of conditions and the following disclaimer. | ||||
* 2. Redistributions in binary form must reproduce the above copyright | * 2. Redistributions in binary form must reproduce the above copyright | ||||
* notice, this list of conditions and the following disclaimer in the | * notice, this list of conditions and the following disclaimer in the | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#include "ioat.h" | #include "ioat.h" | ||||
#include "ioat_hw.h" | #include "ioat_hw.h" | ||||
#include "ioat_internal.h" | #include "ioat_internal.h" | ||||
#ifndef BUS_SPACE_MAXADDR_40BIT | #ifndef BUS_SPACE_MAXADDR_40BIT | ||||
#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL | #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL | ||||
#endif | #endif | ||||
#define IOAT_REFLK (&ioat->submit_lock) | |||||
static int ioat_probe(device_t device); | static int ioat_probe(device_t device); | ||||
static int ioat_attach(device_t device); | static int ioat_attach(device_t device); | ||||
static int ioat_detach(device_t device); | static int ioat_detach(device_t device); | ||||
static int ioat_setup_intr(struct ioat_softc *ioat); | static int ioat_setup_intr(struct ioat_softc *ioat); | ||||
static int ioat_teardown_intr(struct ioat_softc *ioat); | static int ioat_teardown_intr(struct ioat_softc *ioat); | ||||
static int ioat3_attach(device_t device); | static int ioat3_attach(device_t device); | ||||
static int ioat_start_channel(struct ioat_softc *ioat); | static int ioat_start_channel(struct ioat_softc *ioat); | ||||
static int ioat_map_pci_bar(struct ioat_softc *ioat); | static int ioat_map_pci_bar(struct ioat_softc *ioat); | ||||
static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, | static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, | ||||
int error); | int error); | ||||
static void ioat_interrupt_handler(void *arg); | static void ioat_interrupt_handler(void *arg); | ||||
static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat); | static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat); | ||||
static int chanerr_to_errno(uint32_t); | static int chanerr_to_errno(uint32_t); | ||||
static void ioat_process_events(struct ioat_softc *ioat); | static void ioat_process_events(struct ioat_softc *ioat, boolean_t intr); | ||||
static inline uint32_t ioat_get_active(struct ioat_softc *ioat); | static inline uint32_t ioat_get_active(struct ioat_softc *ioat); | ||||
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); | static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); | ||||
static void ioat_free_ring(struct ioat_softc *, uint32_t size, | static void ioat_free_ring(struct ioat_softc *, uint32_t size, | ||||
struct ioat_descriptor *); | struct ioat_descriptor *); | ||||
static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags); | static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags); | ||||
static union ioat_hw_descriptor *ioat_get_descriptor(struct ioat_softc *, | static union ioat_hw_descriptor *ioat_get_descriptor(struct ioat_softc *, | ||||
uint32_t index); | uint32_t index); | ||||
static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *, | static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *, | ||||
uint32_t index); | uint32_t index); | ||||
static void ioat_halted_debug(struct ioat_softc *, uint32_t); | static void ioat_halted_debug(struct ioat_softc *, uint32_t); | ||||
static void ioat_poll_timer_callback(void *arg); | static void ioat_poll_timer_callback(void *arg); | ||||
static void dump_descriptor(void *hw_desc); | static void dump_descriptor(void *hw_desc); | ||||
static void ioat_submit_single(struct ioat_softc *ioat); | static void ioat_submit_single(struct ioat_softc *ioat); | ||||
static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, | ||||
int error); | int error); | ||||
static int ioat_reset_hw(struct ioat_softc *ioat); | static int ioat_reset_hw(struct ioat_softc *ioat); | ||||
static void ioat_reset_hw_task(void *, int); | static void ioat_reset_hw_task(void *, int); | ||||
static void ioat_setup_sysctl(device_t device); | static void ioat_setup_sysctl(device_t device); | ||||
static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); | ||||
static inline struct ioat_softc *ioat_get(struct ioat_softc *, | static void ioat_get(struct ioat_softc *); | ||||
enum ioat_ref_kind); | static void ioat_put(struct ioat_softc *); | ||||
static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind); | |||||
static inline void _ioat_putn(struct ioat_softc *, uint32_t, | |||||
enum ioat_ref_kind, boolean_t); | |||||
static inline void ioat_putn(struct ioat_softc *, uint32_t, | |||||
enum ioat_ref_kind); | |||||
static inline void ioat_putn_locked(struct ioat_softc *, uint32_t, | |||||
enum ioat_ref_kind); | |||||
static void ioat_drain_locked(struct ioat_softc *); | static void ioat_drain_locked(struct ioat_softc *); | ||||
#define ioat_log_message(v, ...) do { \ | #define ioat_log_message(v, ...) do { \ | ||||
if ((v) <= g_ioat_debug_level) { \ | if ((v) <= g_ioat_debug_level) { \ | ||||
device_printf(ioat->device, __VA_ARGS__); \ | device_printf(ioat->device, __VA_ARGS__); \ | ||||
} \ | } \ | ||||
} while (0) | } while (0) | ||||
Show All 35 Lines | |||||
/* | /* | ||||
* Private data structures | * Private data structures | ||||
*/ | */ | ||||
static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; | static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; | ||||
static unsigned ioat_channel_index = 0; | static unsigned ioat_channel_index = 0; | ||||
SYSCTL_UINT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, | SYSCTL_UINT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, | ||||
"Number of IOAT channels attached"); | "Number of IOAT channels attached"); | ||||
static struct mtx ioat_list_mtx; | |||||
MTX_SYSINIT(ioat_list_mtx, &ioat_list_mtx, "ioat list mtx", MTX_DEF); | |||||
static struct _pcsid | static struct _pcsid | ||||
{ | { | ||||
u_int32_t type; | u_int32_t type; | ||||
const char *desc; | const char *desc; | ||||
} pci_ids[] = { | } pci_ids[] = { | ||||
{ 0x34308086, "TBG IOAT Ch0" }, | { 0x34308086, "TBG IOAT Ch0" }, | ||||
{ 0x34318086, "TBG IOAT Ch1" }, | { 0x34318086, "TBG IOAT Ch1" }, | ||||
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines | ioat_probe(device_t device) | ||||
} | } | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
static int | static int | ||||
ioat_attach(device_t device) | ioat_attach(device_t device) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
int error; | int error, i; | ||||
ioat = DEVICE2SOFTC(device); | ioat = DEVICE2SOFTC(device); | ||||
ioat->device = device; | ioat->device = device; | ||||
error = ioat_map_pci_bar(ioat); | error = ioat_map_pci_bar(ioat); | ||||
if (error != 0) | if (error != 0) | ||||
goto err; | goto err; | ||||
Show All 14 Lines | ioat_attach(device_t device) | ||||
error = ioat_setup_intr(ioat); | error = ioat_setup_intr(ioat); | ||||
if (error != 0) | if (error != 0) | ||||
goto err; | goto err; | ||||
error = ioat_reset_hw(ioat); | error = ioat_reset_hw(ioat); | ||||
if (error != 0) | if (error != 0) | ||||
goto err; | goto err; | ||||
ioat_process_events(ioat); | ioat_process_events(ioat, FALSE); | ||||
ioat_setup_sysctl(device); | ioat_setup_sysctl(device); | ||||
ioat->chan_idx = ioat_channel_index; | mtx_lock(&ioat_list_mtx); | ||||
ioat_channel[ioat_channel_index++] = ioat; | for (i = 0; i < IOAT_MAX_CHANNELS; i++) { | ||||
if (ioat_channel[i] == NULL) | |||||
break; | |||||
} | |||||
if (i >= IOAT_MAX_CHANNELS) { | |||||
mtx_unlock(&ioat_list_mtx); | |||||
error = ENXIO; | |||||
cem: Maybe `ENFILE`? Or print something to console/log if we ever encounter this condition… | |||||
Done Inline ActionsI thought about it, but word "file" in error message confused me. mav: I thought about it, but word "file" in error message confused me. | |||||
Not Done Inline ActionsENXIO is the proper, traditional error for a unit out of the supported range IMHO. ENFILE means something else. This is an attach routine, not an open routine. imp: ENXIO is the proper, traditional error for a unit out of the supported range IMHO. ENFILE means… | |||||
Not Done Inline Actions*shrug* Adding a printf instead is fine. cem: *shrug*
Adding a printf instead is fine. | |||||
goto err; | |||||
} | |||||
ioat->chan_idx = i; | |||||
ioat_channel[i] = ioat; | |||||
if (i >= ioat_channel_index) | |||||
ioat_channel_index = i + 1; | |||||
Not Done Inline ActionsWith this change, I think ioat_channel_index can go away; it doesn't serve much purpose anymore. cem: With this change, I think `ioat_channel_index` can go away; it doesn't serve much purpose… | |||||
Done Inline ActionsYes, it can. I'll do it. mav: Yes, it can. I'll do it. | |||||
Done Inline ActionsAh, I recalled why I kept it -- to not bother writing SYSCTL_PROC handler. mav: Ah, I recalled why I kept it -- to not bother writing SYSCTL_PROC handler. | |||||
Not Done Inline ActionsSure, but why not just remove that SYSCTL entirely? cem: Sure, but why not just remove that SYSCTL entirely? | |||||
mtx_unlock(&ioat_list_mtx); | |||||
ioat_test_attach(); | ioat_test_attach(); | ||||
err: | err: | ||||
if (error != 0) | if (error != 0) | ||||
ioat_detach(device); | ioat_detach(device); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
ioat_detach(device_t device) | ioat_detach(device_t device) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = DEVICE2SOFTC(device); | ioat = DEVICE2SOFTC(device); | ||||
mtx_lock(&ioat_list_mtx); | |||||
ioat_channel[ioat->chan_idx] = NULL; | |||||
while (ioat_channel_index > 0 && | |||||
ioat_channel[ioat_channel_index - 1] == NULL) | |||||
ioat_channel_index--; | |||||
mtx_unlock(&ioat_list_mtx); | |||||
ioat_test_detach(); | ioat_test_detach(); | ||||
taskqueue_drain(taskqueue_thread, &ioat->reset_task); | taskqueue_drain(taskqueue_thread, &ioat->reset_task); | ||||
mtx_lock(IOAT_REFLK); | mtx_lock(&ioat->submit_lock); | ||||
ioat->quiescing = TRUE; | ioat->quiescing = TRUE; | ||||
ioat->destroying = TRUE; | ioat->destroying = TRUE; | ||||
Not Done Inline ActionsIt seems like we should take submit_lock under list_mtx in order to set destroying in a way that will not race with the logic below in ioat_get_dmaengine(). We can drop it and pick it up again around test_detach/taskqueue_drain if we need to. cem: It seems like we should take submit_lock under list_mtx in order to set `destroying` in a way… | |||||
Done Inline ActionsI don't think it is important here. Device can not disappear here, since we are is what destroying it. And just below we wait for last reference to disappear. I just think taskqueue_drain could be moved later, if you say. mav: I don't think it is important here. Device can not disappear here, since we are is what… | |||||
Not Done Inline ActionsI think you're right. cem: I think you're right. | |||||
wakeup(&ioat->quiescing); | wakeup(&ioat->quiescing); | ||||
wakeup(&ioat->resetting); | wakeup(&ioat->resetting); | ||||
ioat_channel[ioat->chan_idx] = NULL; | |||||
ioat_drain_locked(ioat); | ioat_drain_locked(ioat); | ||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(&ioat->submit_lock); | ||||
mtx_lock(&ioat->cleanup_lock); | |||||
while (ioat_get_active(ioat) > 0) | |||||
msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1); | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
ioat_teardown_intr(ioat); | ioat_teardown_intr(ioat); | ||||
callout_drain(&ioat->poll_timer); | callout_drain(&ioat->poll_timer); | ||||
pci_disable_busmaster(device); | pci_disable_busmaster(device); | ||||
if (ioat->pci_resource != NULL) | if (ioat->pci_resource != NULL) | ||||
bus_release_resource(device, SYS_RES_MEMORY, | bus_release_resource(device, SYS_RES_MEMORY, | ||||
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines | ioat3_attach(device_t device) | ||||
/* TODO: need to check DCA here if we ever do XOR/PQ */ | /* TODO: need to check DCA here if we ever do XOR/PQ */ | ||||
mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); | ||||
mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); | mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); | ||||
callout_init(&ioat->poll_timer, 1); | callout_init(&ioat->poll_timer, 1); | ||||
TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); | TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); | ||||
/* Establish lock order for Witness */ | /* Establish lock order for Witness */ | ||||
mtx_lock(&ioat->submit_lock); | |||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_lock(&ioat->submit_lock); | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
mtx_unlock(&ioat->cleanup_lock); | |||||
ioat->is_submitter_processing = FALSE; | ioat->is_submitter_processing = FALSE; | ||||
ioat->is_completion_pending = FALSE; | |||||
ioat->is_reset_pending = FALSE; | |||||
ioat->is_channel_running = FALSE; | |||||
bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, | bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, | ||||
BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, | ||||
sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, | sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, | ||||
&ioat->comp_update_tag); | &ioat->comp_update_tag); | ||||
error = bus_dmamem_alloc(ioat->comp_update_tag, | error = bus_dmamem_alloc(ioat->comp_update_tag, | ||||
(void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); | (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); | ||||
Show All 38 Lines | for (i = 0; i < num_descriptors; i++) { | ||||
ring[i].id = i; | ring[i].id = i; | ||||
} | } | ||||
for (i = 0; i < num_descriptors; i++) { | for (i = 0; i < num_descriptors; i++) { | ||||
dma_hw_desc = &ioat->hw_desc_ring[i].dma; | dma_hw_desc = &ioat->hw_desc_ring[i].dma; | ||||
dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1); | dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1); | ||||
} | } | ||||
ioat->head = ioat->hw_head = 0; | ioat->head = 0; | ||||
ioat->tail = 0; | ioat->tail = 0; | ||||
ioat->last_seen = 0; | ioat->last_seen = 0; | ||||
*ioat->comp_update = 0; | *ioat->comp_update = 0; | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ioat_map_pci_bar(struct ioat_softc *ioat) | ioat_map_pci_bar(struct ioat_softc *ioat) | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static void | static void | ||||
ioat_interrupt_handler(void *arg) | ioat_interrupt_handler(void *arg) | ||||
{ | { | ||||
struct ioat_softc *ioat = arg; | struct ioat_softc *ioat = arg; | ||||
ioat->stats.interrupts++; | ioat->stats.interrupts++; | ||||
ioat_process_events(ioat); | ioat_process_events(ioat, TRUE); | ||||
} | } | ||||
static int | static int | ||||
chanerr_to_errno(uint32_t chanerr) | chanerr_to_errno(uint32_t chanerr) | ||||
{ | { | ||||
if (chanerr == 0) | if (chanerr == 0) | ||||
return (0); | return (0); | ||||
if ((chanerr & (IOAT_CHANERR_XSADDERR | IOAT_CHANERR_XDADDERR)) != 0) | if ((chanerr & (IOAT_CHANERR_XSADDERR | IOAT_CHANERR_XDADDERR)) != 0) | ||||
return (EFAULT); | return (EFAULT); | ||||
if ((chanerr & (IOAT_CHANERR_RDERR | IOAT_CHANERR_WDERR)) != 0) | if ((chanerr & (IOAT_CHANERR_RDERR | IOAT_CHANERR_WDERR)) != 0) | ||||
return (EIO); | return (EIO); | ||||
/* This one is probably our fault: */ | /* This one is probably our fault: */ | ||||
if ((chanerr & IOAT_CHANERR_NDADDERR) != 0) | if ((chanerr & IOAT_CHANERR_NDADDERR) != 0) | ||||
return (EIO); | return (EIO); | ||||
return (EIO); | return (EIO); | ||||
} | } | ||||
static void | static void | ||||
ioat_process_events(struct ioat_softc *ioat) | ioat_process_events(struct ioat_softc *ioat, boolean_t intr) | ||||
{ | { | ||||
struct ioat_descriptor *desc; | struct ioat_descriptor *desc; | ||||
struct bus_dmadesc *dmadesc; | struct bus_dmadesc *dmadesc; | ||||
uint64_t comp_update, status; | uint64_t comp_update, status; | ||||
uint32_t completed, chanerr; | uint32_t completed, chanerr; | ||||
boolean_t pending; | |||||
int error; | int error; | ||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
/* | /* | ||||
* Don't run while the hardware is being reset. Reset is responsible | * Don't run while the hardware is being reset. Reset is responsible | ||||
* for blocking new work and draining & completing existing work, so | * for blocking new work and draining & completing existing work, so | ||||
* there is nothing to do until new work is queued after reset anyway. | * there is nothing to do until new work is queued after reset anyway. | ||||
Show All 37 Lines | while (RING_PHYS_ADDR(ioat, ioat->tail - 1) != status) { | ||||
ioat->tail++; | ioat->tail++; | ||||
} | } | ||||
CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, | CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, | ||||
ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); | ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); | ||||
if (completed != 0) { | if (completed != 0) { | ||||
ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1); | ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1); | ||||
ioat->stats.descriptors_processed += completed; | ioat->stats.descriptors_processed += completed; | ||||
wakeup(&ioat->tail); | |||||
Not Done Inline ActionsThis can be expensive to do all the time. Maybe waiters can set a flag that wakeup is requested, and we can wake conditionally? cem: This can be expensive to do all the time. Maybe waiters can set a flag that wakeup is… | |||||
Done Inline ActionsI thought about it, but first I don't see it in my CPU profiles, and second, it was there, just dozen lines later out of the lock, that opened possible race conditions. mav: I thought about it, but first I don't see it in my CPU profiles, and second, it was there, just… | |||||
Not Done Inline ActionsOk cem: Ok | |||||
} | } | ||||
out: | out: | ||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
/* Perform a racy check first; only take the locks if it passes. */ | |||||
pending = (ioat_get_active(ioat) != 0); | |||||
if (!pending && ioat->is_completion_pending) { | |||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
mtx_lock(&ioat->submit_lock); | |||||
mtx_lock(&ioat->cleanup_lock); | |||||
pending = (ioat_get_active(ioat) != 0); | |||||
if (!pending && ioat->is_completion_pending) { | |||||
ioat->is_completion_pending = FALSE; | |||||
callout_stop(&ioat->poll_timer); | |||||
} | |||||
mtx_unlock(&ioat->submit_lock); | |||||
} | |||||
mtx_unlock(&ioat->cleanup_lock); | |||||
if (pending) | |||||
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, | |||||
ioat); | |||||
Not Done Inline ActionsI'm not sure this is safe to remove. Edit: I see you moved it outside of this routine to only reset when the callout fires (or !pending and new work is added). That's probably ok. cem: I'm not sure this is safe to remove. Edit: I see you moved it outside of this routine to only… | |||||
if (completed != 0) { | |||||
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); | |||||
wakeup(&ioat->tail); | |||||
} | |||||
/* | /* | ||||
* The device doesn't seem to reliably push suspend/halt statuses to | * The device doesn't seem to reliably push suspend/halt statuses to | ||||
* the channel completion memory address, so poll the device register | * the channel completion memory address, so poll the device register | ||||
* here. | * here. For performance reasons skip it on interrupts, do it only | ||||
* on much more rare polling events. | |||||
*/ | */ | ||||
if (!intr) | |||||
comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS; | comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS; | ||||
if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) | if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) | ||||
Not Done Inline ActionsThe motivation here is to avoid the MMIO read during the interrupt handler? If so, please add a sentence in the comment mentioning that. cem: The motivation here is to avoid the MMIO read during the interrupt handler? If so, please add… | |||||
Done Inline ActionsYes, I'll add the comment. mav: Yes, I'll add the comment. | |||||
Not Done Inline ActionsThanks! cem: Thanks! | |||||
return; | return; | ||||
ioat->stats.channel_halts++; | ioat->stats.channel_halts++; | ||||
/* | /* | ||||
* Fatal programming error on this DMA channel. Flush any outstanding | * Fatal programming error on this DMA channel. Flush any outstanding | ||||
* work with error status and restart the engine. | * work with error status and restart the engine. | ||||
*/ | */ | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
mtx_lock(&ioat->cleanup_lock); | |||||
ioat->quiescing = TRUE; | ioat->quiescing = TRUE; | ||||
mtx_unlock(&ioat->submit_lock); | |||||
/* | /* | ||||
* This is safe to do here because we have both locks and the submit | * This is safe to do here because the submit queue is quiesced. We | ||||
Not Done Inline ActionsThis sentence is now stale -- we don't have both locks cem: This sentence is now stale -- we don't have both locks | |||||
Done Inline ActionsOK mav: OK | |||||
* queue is quiesced. We know that we will drain all outstanding | * know that we will drain all outstanding events, so ioat_reset_hw | ||||
* events, so ioat_reset_hw can't deadlock. It is necessary to | * can't deadlock. It is necessary to protect other ioat_process_event | ||||
* protect other ioat_process_event threads from racing ioat_reset_hw, | * threads from racing ioat_reset_hw, reading an indeterminate hw | ||||
* reading an indeterminate hw state, and attempting to continue | * state, and attempting to continue issuing completions. | ||||
* issuing completions. | |||||
*/ | */ | ||||
mtx_lock(&ioat->cleanup_lock); | |||||
ioat->resetting_cleanup = TRUE; | ioat->resetting_cleanup = TRUE; | ||||
chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); | ||||
if (1 <= g_ioat_debug_level) | if (1 <= g_ioat_debug_level) | ||||
ioat_halted_debug(ioat, chanerr); | ioat_halted_debug(ioat, chanerr); | ||||
ioat->stats.last_halt_chanerr = chanerr; | ioat->stats.last_halt_chanerr = chanerr; | ||||
while (ioat_get_active(ioat) > 0) { | while (ioat_get_active(ioat) > 0) { | ||||
desc = ioat_get_ring_entry(ioat, ioat->tail); | desc = ioat_get_ring_entry(ioat, ioat->tail); | ||||
dmadesc = &desc->bus_dmadesc; | dmadesc = &desc->bus_dmadesc; | ||||
CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb %p(%p)", | CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb %p(%p)", | ||||
ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn, | ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn, | ||||
dmadesc->callback_arg); | dmadesc->callback_arg); | ||||
if (dmadesc->callback_fn != NULL) | if (dmadesc->callback_fn != NULL) | ||||
dmadesc->callback_fn(dmadesc->callback_arg, | dmadesc->callback_fn(dmadesc->callback_arg, | ||||
chanerr_to_errno(chanerr)); | chanerr_to_errno(chanerr)); | ||||
ioat_putn_locked(ioat, 1, IOAT_ACTIVE_DESCR_REF); | |||||
ioat->tail++; | ioat->tail++; | ||||
ioat->stats.descriptors_processed++; | ioat->stats.descriptors_processed++; | ||||
ioat->stats.descriptors_error++; | ioat->stats.descriptors_error++; | ||||
} | } | ||||
CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, | CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, | ||||
ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); | ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); | ||||
if (ioat->is_completion_pending) { | |||||
ioat->is_completion_pending = FALSE; | |||||
callout_stop(&ioat->poll_timer); | |||||
} | |||||
/* Clear error status */ | /* Clear error status */ | ||||
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); | ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
mtx_unlock(&ioat->submit_lock); | |||||
ioat_log_message(0, "Resetting channel to recover from error\n"); | ioat_log_message(0, "Resetting channel to recover from error\n"); | ||||
error = taskqueue_enqueue(taskqueue_thread, &ioat->reset_task); | error = taskqueue_enqueue(taskqueue_thread, &ioat->reset_task); | ||||
KASSERT(error == 0, | KASSERT(error == 0, | ||||
("%s: taskqueue_enqueue failed: %d", __func__, error)); | ("%s: taskqueue_enqueue failed: %d", __func__, error)); | ||||
} | } | ||||
static void | static void | ||||
Show All 12 Lines | |||||
/* | /* | ||||
* User API functions | * User API functions | ||||
*/ | */ | ||||
unsigned | unsigned | ||||
ioat_get_nchannels(void) | ioat_get_nchannels(void) | ||||
{ | { | ||||
return (ioat_channel_index); | return (ioat_channel_index); | ||||
Not Done Inline ActionsThis isn't accurate after arbitrary detach (it wasn't before, either, but it might as well be fixed or probably just removed). cem: This isn't accurate after arbitrary detach (it wasn't before, either, but it might as well be… | |||||
Done Inline ActionsIt is accurate in sense of reporting maximal value, though yes, there can be holes in the list. Not sure how to better handle that. mav: It is accurate in sense of reporting maximal value, though yes, there can be holes in the list. | |||||
Not Done Inline ActionsSure, but from that standpoint, constant IOAT_MAX_CHANNELS is equally valid :-). I think it is safe to remove, and anyone who cares can enumerate newbus and check unit numbers. cem: Sure, but from that standpoint, constant IOAT_MAX_CHANNELS is equally valid :-). I think it is… | |||||
} | } | ||||
bus_dmaengine_t | bus_dmaengine_t | ||||
ioat_get_dmaengine(uint32_t index, int flags) | ioat_get_dmaengine(uint32_t index, int flags) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
KASSERT((flags & ~(M_NOWAIT | M_WAITOK)) == 0, | KASSERT((flags & ~(M_NOWAIT | M_WAITOK)) == 0, | ||||
("invalid flags: 0x%08x", flags)); | ("invalid flags: 0x%08x", flags)); | ||||
KASSERT((flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK), | KASSERT((flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK), | ||||
("invalid wait | nowait")); | ("invalid wait | nowait")); | ||||
if (index >= ioat_channel_index) | mtx_lock(&ioat_list_mtx); | ||||
if (index >= ioat_channel_index || | |||||
(ioat = ioat_channel[index]) == NULL) { | |||||
mtx_unlock(&ioat_list_mtx); | |||||
return (NULL); | return (NULL); | ||||
} | |||||
mtx_lock(&ioat->submit_lock); | |||||
mtx_unlock(&ioat_list_mtx); | |||||
ioat = ioat_channel[index]; | if (ioat->destroying) { | ||||
if (ioat == NULL || ioat->destroying) | mtx_unlock(&ioat->submit_lock); | ||||
return (NULL); | return (NULL); | ||||
} | |||||
ioat_get(ioat); | |||||
if (ioat->quiescing) { | if (ioat->quiescing) { | ||||
if ((flags & M_NOWAIT) != 0) | if ((flags & M_NOWAIT) != 0) { | ||||
ioat_put(ioat); | |||||
mtx_unlock(&ioat->submit_lock); | |||||
return (NULL); | return (NULL); | ||||
} | |||||
mtx_lock(IOAT_REFLK); | |||||
while (ioat->quiescing && !ioat->destroying) | while (ioat->quiescing && !ioat->destroying) | ||||
msleep(&ioat->quiescing, IOAT_REFLK, 0, "getdma", 0); | msleep(&ioat->quiescing, &ioat->submit_lock, 0, "getdma", 0); | ||||
mtx_unlock(IOAT_REFLK); | |||||
if (ioat->destroying) | if (ioat->destroying) { | ||||
ioat_put(ioat); | |||||
mtx_unlock(&ioat->submit_lock); | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | |||||
* There's a race here between the quiescing check and HW reset or | |||||
* module destroy. | |||||
*/ | |||||
return (&ioat_get(ioat, IOAT_DMAENGINE_REF)->dmaengine); | |||||
} | } | ||||
mtx_unlock(&ioat->submit_lock); | |||||
return (&ioat->dmaengine); | |||||
} | |||||
void | void | ||||
ioat_put_dmaengine(bus_dmaengine_t dmaengine) | ioat_put_dmaengine(bus_dmaengine_t dmaengine) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat = to_ioat_softc(dmaengine); | ||||
Not Done Inline Actionsnitpick: this is a style(9) regression. declarations should be separate from initialization, as it was before. cem: nitpick: this is a style(9) regression. declarations should be separate from initialization… | |||||
Done Inline ActionsWhile formally you are right, I find code much more compact and as result more readable when I don't need to add separate lines for trivial assignment of function arguments. mav: While formally you are right, I find code much more compact and as result more readable when I… | |||||
Not Done Inline ActionsSure, personal preferences vary. But part of having a style guide is that a project ends up with consistent style in spite of having many authors. Personally, I find it surprising when declarations have initializations associated with them. So I'd prefer leaving these two lines as-is. cem: Sure, personal preferences vary. But part of having a style guide is that a project ends up… | |||||
ioat = to_ioat_softc(dmaengine); | mtx_lock(&ioat->submit_lock); | ||||
ioat_put(ioat, IOAT_DMAENGINE_REF); | ioat_put(ioat); | ||||
mtx_unlock(&ioat->submit_lock); | |||||
} | } | ||||
int | int | ||||
ioat_get_hwversion(bus_dmaengine_t dmaengine) | ioat_get_hwversion(bus_dmaengine_t dmaengine) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = to_ioat_softc(dmaengine); | ioat = to_ioat_softc(dmaengine); | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
void | void | ||||
ioat_release(bus_dmaengine_t dmaengine) | ioat_release(bus_dmaengine_t dmaengine) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = to_ioat_softc(dmaengine); | ioat = to_ioat_softc(dmaengine); | ||||
CTR4(KTR_IOAT, "%s channel=%u dispatch1 hw_head=%u head=%u", __func__, | CTR3(KTR_IOAT, "%s channel=%u dispatch1 head=%u", __func__, | ||||
ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); | ioat->chan_idx, ioat->head); | ||||
KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */); | KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */); | ||||
CTR4(KTR_IOAT, "%s channel=%u dispatch2 hw_head=%u head=%u", __func__, | CTR3(KTR_IOAT, "%s channel=%u dispatch2 head=%u", __func__, | ||||
ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); | ioat->chan_idx, ioat->head); | ||||
if (ioat->acq_head != ioat->head) { | if (ioat->acq_head != ioat->head) { | ||||
ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, | ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, | ||||
(uint16_t)ioat->hw_head); | (uint16_t)ioat->head); | ||||
if (!ioat->is_completion_pending) { | if (!callout_pending(&ioat->poll_timer)) { | ||||
ioat->is_completion_pending = TRUE; | callout_reset(&ioat->poll_timer, hz / 100, | ||||
Not Done Inline ActionsA real value here is closer to hz/100000; 1 was fine for all values of hz that I've heard of people using. You could do something more specific with callout_reset_sbt if you want. cem: A real value here is closer to `hz/100000`; `1` was fine for all values of `hz` that I've heard… | |||||
Done Inline ActionsReal value for what? I don't want here 100KHz of interrupts for polling, I'd prefer 100Hz, as written. At most I agree on 1KHz. Is there hardware where interrupts not functioning at all and where polling is not a workaround for sometimes lost interrupts, but a main event source? mav: Real value for what? I don't want here 100KHz of interrupts for polling, I'd prefer 100Hz, as… | |||||
Not Done Inline ActionsReal value for "if this much time has elapsed and we haven't gotten an interrupt-driven completion, the operation has probably timed out." I think 1 is probably fine — it will line up with the native hardclock frequency no matter what, and we already poll the system at hardclock on at least one CPU. Forcing 100 Hz on a 1kHz system introduces an extraneous +9 ms delay on a ~1.5µs operation. When this hardware is running full-out, it can generate a tremendous number of interrupts (it can do ~650k ops/s, and every single one can produce an interrupt). From that perspective, it may not make sense to enable interrupts all the time and instead poll. We don't use the poll-only mode at ISLN, but it is not an unreasonable option for some usecases. [We also do not use interrupt coalescing due to the impact on latency.] cem: Real value for "if this much time has elapsed and we haven't gotten an interrupt-driven… | |||||
callout_reset(&ioat->poll_timer, 1, | |||||
ioat_poll_timer_callback, ioat); | ioat_poll_timer_callback, ioat); | ||||
} | } | ||||
} | } | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
} | } | ||||
static struct ioat_descriptor * | static struct ioat_descriptor * | ||||
ioat_op_generic(struct ioat_softc *ioat, uint8_t op, | ioat_op_generic(struct ioat_softc *ioat, uint8_t op, | ||||
▲ Show 20 Lines • Show All 399 Lines • ▼ Show 20 Lines | CTR3(KTR_IOAT, "%s channel=%u starved (%u)", __func__, | ||||
ioat->chan_idx, num_descs); | ioat->chan_idx, num_descs); | ||||
if (!dug && !ioat->is_submitter_processing) { | if (!dug && !ioat->is_submitter_processing) { | ||||
ioat->is_submitter_processing = TRUE; | ioat->is_submitter_processing = TRUE; | ||||
mtx_unlock(&ioat->submit_lock); | mtx_unlock(&ioat->submit_lock); | ||||
CTR2(KTR_IOAT, "%s channel=%u attempting to process events", | CTR2(KTR_IOAT, "%s channel=%u attempting to process events", | ||||
__func__, ioat->chan_idx); | __func__, ioat->chan_idx); | ||||
ioat_process_events(ioat); | ioat_process_events(ioat, FALSE); | ||||
mtx_lock(&ioat->submit_lock); | mtx_lock(&ioat->submit_lock); | ||||
dug = TRUE; | dug = TRUE; | ||||
KASSERT(ioat->is_submitter_processing == TRUE, | KASSERT(ioat->is_submitter_processing == TRUE, | ||||
("is_submitter_processing")); | ("is_submitter_processing")); | ||||
ioat->is_submitter_processing = FALSE; | ioat->is_submitter_processing = FALSE; | ||||
wakeup(&ioat->tail); | wakeup(&ioat->tail); | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
ioat_poll_timer_callback(void *arg) | ioat_poll_timer_callback(void *arg) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
ioat = arg; | ioat = arg; | ||||
ioat_log_message(3, "%s\n", __func__); | ioat_log_message(3, "%s\n", __func__); | ||||
ioat_process_events(ioat); | ioat_process_events(ioat, FALSE); | ||||
mtx_lock(&ioat->submit_lock); | |||||
if (ioat_get_active(ioat) > 0) | |||||
callout_schedule(&ioat->poll_timer, hz / 100); | |||||
mtx_unlock(&ioat->submit_lock); | |||||
} | } | ||||
/* | /* | ||||
* Support Functions | * Support Functions | ||||
*/ | */ | ||||
static void | static void | ||||
ioat_submit_single(struct ioat_softc *ioat) | ioat_submit_single(struct ioat_softc *ioat) | ||||
{ | { | ||||
mtx_assert(&ioat->submit_lock, MA_OWNED); | mtx_assert(&ioat->submit_lock, MA_OWNED); | ||||
ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); | ioat->head++; | ||||
atomic_add_rel_int(&ioat->head, 1); | CTR4(KTR_IOAT, "%s channel=%u head=%u tail=%u", __func__, | ||||
atomic_add_rel_int(&ioat->hw_head, 1); | ioat->chan_idx, ioat->head, ioat->tail); | ||||
CTR5(KTR_IOAT, "%s channel=%u head=%u hw_head=%u tail=%u", __func__, | |||||
ioat->chan_idx, ioat->head, ioat->hw_head & UINT16_MAX, | |||||
ioat->tail); | |||||
ioat->stats.descriptors_submitted++; | ioat->stats.descriptors_submitted++; | ||||
} | } | ||||
static int | static int | ||||
ioat_reset_hw(struct ioat_softc *ioat) | ioat_reset_hw(struct ioat_softc *ioat) | ||||
{ | { | ||||
uint64_t status; | uint64_t status; | ||||
uint32_t chanerr; | uint32_t chanerr; | ||||
unsigned timeout; | unsigned timeout; | ||||
int error; | int error; | ||||
CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); | CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); | ||||
mtx_lock(IOAT_REFLK); | mtx_lock(&ioat->submit_lock); | ||||
while (ioat->resetting && !ioat->destroying) | while (ioat->resetting && !ioat->destroying) | ||||
msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0); | msleep(&ioat->resetting, &ioat->submit_lock, 0, "IRH_drain", 0); | ||||
if (ioat->destroying) { | if (ioat->destroying) { | ||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(&ioat->submit_lock); | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
ioat->resetting = TRUE; | ioat->resetting = TRUE; | ||||
ioat->quiescing = TRUE; | ioat->quiescing = TRUE; | ||||
ioat_drain_locked(ioat); | mtx_unlock(&ioat->submit_lock); | ||||
Not Done Inline ActionsI don't think it's safe to remove this ioat_drain_locked(), or something similar to this. cem: I don't think it's safe to remove this `ioat_drain_locked()`, or something similar to this. | |||||
Done Inline ActionsOops! Thank you for spotting it. Seems like I moved the loop from here to detach instead of copying. I'll fix this. mav: Oops! Thank you for spotting it. Seems like I moved the loop from here to detach instead of… | |||||
Done Inline ActionsWith this fixed I successfully triggered resets many times with sysctl under heavy load without any problems. mav: With this fixed I successfully triggered resets many times with sysctl under heavy load without… | |||||
Not Done Inline ActionsThanks! That's good to hear. As a stress test, it might be good to randomly introduce reset repeatedly for "a long time" and make sure it does not fall over. But this is a good early indication. cem: Thanks! That's good to hear. As a stress test, it might be good to randomly introduce reset… | |||||
mtx_unlock(IOAT_REFLK); | mtx_lock(&ioat->cleanup_lock); | ||||
while (ioat_get_active(ioat) > 0) | |||||
msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1); | |||||
/* | /* | ||||
* Suspend ioat_process_events while the hardware and softc are in an | * Suspend ioat_process_events while the hardware and softc are in an | ||||
* indeterminate state. | * indeterminate state. | ||||
*/ | */ | ||||
mtx_lock(&ioat->cleanup_lock); | |||||
ioat->resetting_cleanup = TRUE; | ioat->resetting_cleanup = TRUE; | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__, | CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__, | ||||
ioat->chan_idx); | ioat->chan_idx); | ||||
status = ioat_get_chansts(ioat); | status = ioat_get_chansts(ioat); | ||||
if (is_ioat_active(status) || is_ioat_idle(status)) | if (is_ioat_active(status) || is_ioat_idle(status)) | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | ioat_reset_hw(struct ioat_softc *ioat) | ||||
/* | /* | ||||
* Bring device back online after reset. Writing CHAINADDR brings the | * Bring device back online after reset. Writing CHAINADDR brings the | ||||
* device back to active. | * device back to active. | ||||
* | * | ||||
* The internal ring counter resets to zero, so we have to start over | * The internal ring counter resets to zero, so we have to start over | ||||
* at zero as well. | * at zero as well. | ||||
*/ | */ | ||||
ioat->tail = ioat->head = ioat->hw_head = 0; | ioat->tail = ioat->head = 0; | ||||
ioat->last_seen = 0; | ioat->last_seen = 0; | ||||
*ioat->comp_update = 0; | *ioat->comp_update = 0; | ||||
KASSERT(!ioat->is_completion_pending, ("bogus completion_pending")); | |||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); | ||||
ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); | ||||
ioat_write_chainaddr(ioat, RING_PHYS_ADDR(ioat, 0)); | ioat_write_chainaddr(ioat, RING_PHYS_ADDR(ioat, 0)); | ||||
error = 0; | error = 0; | ||||
CTR2(KTR_IOAT, "%s channel=%u configured channel", __func__, | CTR2(KTR_IOAT, "%s channel=%u configured channel", __func__, | ||||
ioat->chan_idx); | ioat->chan_idx); | ||||
out: | out: | ||||
/* Enqueues a null operation and ensures it completes. */ | /* Enqueues a null operation and ensures it completes. */ | ||||
if (error == 0) { | if (error == 0) { | ||||
error = ioat_start_channel(ioat); | error = ioat_start_channel(ioat); | ||||
CTR2(KTR_IOAT, "%s channel=%u started channel", __func__, | CTR2(KTR_IOAT, "%s channel=%u started channel", __func__, | ||||
ioat->chan_idx); | ioat->chan_idx); | ||||
} | } | ||||
/* | /* | ||||
* Resume completions now that ring state is consistent. | * Resume completions now that ring state is consistent. | ||||
*/ | */ | ||||
mtx_lock(&ioat->cleanup_lock); | mtx_lock(&ioat->cleanup_lock); | ||||
ioat->resetting_cleanup = FALSE; | ioat->resetting_cleanup = FALSE; | ||||
mtx_unlock(&ioat->cleanup_lock); | mtx_unlock(&ioat->cleanup_lock); | ||||
/* Unblock submission of new work */ | /* Unblock submission of new work */ | ||||
mtx_lock(IOAT_REFLK); | mtx_lock(&ioat->submit_lock); | ||||
ioat->quiescing = FALSE; | ioat->quiescing = FALSE; | ||||
wakeup(&ioat->quiescing); | wakeup(&ioat->quiescing); | ||||
ioat->resetting = FALSE; | ioat->resetting = FALSE; | ||||
wakeup(&ioat->resetting); | wakeup(&ioat->resetting); | ||||
if (ioat->is_completion_pending) | |||||
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, | |||||
ioat); | |||||
Not Done Inline ActionsThis change probably leaves the nop started in ioat_start_channel() incomplete indefinitely. cem: This change probably leaves the nop started in `ioat_start_channel()` incomplete indefinitely. | |||||
Done Inline ActionsWhy? ioat_release() called by ioat_start_channel() should reset the callout. mav: Why? ioat_release() called by ioat_start_channel() should reset the callout. | |||||
Not Done Inline ActionsNvm, I think you're right. cem: Nvm, I think you're right. | |||||
CTR2(KTR_IOAT, "%s channel=%u reset done", __func__, ioat->chan_idx); | CTR2(KTR_IOAT, "%s channel=%u reset done", __func__, ioat->chan_idx); | ||||
mtx_unlock(IOAT_REFLK); | mtx_unlock(&ioat->submit_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
sysctl_handle_chansts(SYSCTL_HANDLER_ARGS) | sysctl_handle_chansts(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
struct ioat_softc *ioat; | struct ioat_softc *ioat; | ||||
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines | ioat_setup_sysctl(device_t device) | ||||
state = SYSCTL_CHILDREN(tmp); | state = SYSCTL_CHILDREN(tmp); | ||||
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "ring_size_order", CTLFLAG_RD, | SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "ring_size_order", CTLFLAG_RD, | ||||
&ioat->ring_size_order, 0, "SW descriptor ring size order"); | &ioat->ring_size_order, 0, "SW descriptor ring size order"); | ||||
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, | SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, | ||||
0, "SW descriptor head pointer index"); | 0, "SW descriptor head pointer index"); | ||||
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, | SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, | ||||
0, "SW descriptor tail pointer index"); | 0, "SW descriptor tail pointer index"); | ||||
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "hw_head", CTLFLAG_RD, | |||||
&ioat->hw_head, 0, "HW DMACOUNT"); | |||||
SYSCTL_ADD_UQUAD(ctx, state, OID_AUTO, "last_completion", CTLFLAG_RD, | SYSCTL_ADD_UQUAD(ctx, state, OID_AUTO, "last_completion", CTLFLAG_RD, | ||||
ioat->comp_update, "HW addr of last completion"); | ioat->comp_update, "HW addr of last completion"); | ||||
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_submitter_processing", | SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_submitter_processing", | ||||
CTLFLAG_RD, &ioat->is_submitter_processing, 0, | CTLFLAG_RD, &ioat->is_submitter_processing, 0, | ||||
"submitter processing"); | "submitter processing"); | ||||
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_completion_pending", | |||||
CTLFLAG_RD, &ioat->is_completion_pending, 0, "completion pending"); | |||||
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_reset_pending", CTLFLAG_RD, | |||||
&ioat->is_reset_pending, 0, "reset pending"); | |||||
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_channel_running", CTLFLAG_RD, | |||||
&ioat->is_channel_running, 0, "channel running"); | |||||
SYSCTL_ADD_PROC(ctx, state, OID_AUTO, "chansts", | SYSCTL_ADD_PROC(ctx, state, OID_AUTO, "chansts", | ||||
CTLTYPE_STRING | CTLFLAG_RD, ioat, 0, sysctl_handle_chansts, "A", | CTLTYPE_STRING | CTLFLAG_RD, ioat, 0, sysctl_handle_chansts, "A", | ||||
"String of the channel status"); | "String of the channel status"); | ||||
SYSCTL_ADD_U16(ctx, state, OID_AUTO, "intrdelay", CTLFLAG_RD, | SYSCTL_ADD_U16(ctx, state, OID_AUTO, "intrdelay", CTLFLAG_RD, | ||||
&ioat->cached_intrdelay, 0, | &ioat->cached_intrdelay, 0, | ||||
"Current INTRDELAY on this channel (cached, microseconds)"); | "Current INTRDELAY on this channel (cached, microseconds)"); | ||||
Show All 29 Lines | SYSCTL_ADD_U32(ctx, statpar, OID_AUTO, "last_halt_chanerr", CTLFLAG_RW, | ||||
&ioat->stats.last_halt_chanerr, 0, | &ioat->stats.last_halt_chanerr, 0, | ||||
"The raw CHANERR when the channel was last halted"); | "The raw CHANERR when the channel was last halted"); | ||||
SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "desc_per_interrupt", | SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "desc_per_interrupt", | ||||
CTLTYPE_STRING | CTLFLAG_RD, ioat, 0, sysctl_handle_dpi, "A", | CTLTYPE_STRING | CTLFLAG_RD, ioat, 0, sysctl_handle_dpi, "A", | ||||
"Descriptors per interrupt"); | "Descriptors per interrupt"); | ||||
} | } | ||||
static inline struct ioat_softc * | static void | ||||
ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind) | ioat_get(struct ioat_softc *ioat) | ||||
{ | { | ||||
uint32_t old; | |||||
KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); | mtx_assert(&ioat->submit_lock, MA_OWNED); | ||||
KASSERT(ioat->refcnt < UINT32_MAX, ("refcnt overflow")); | |||||
old = atomic_fetchadd_32(&ioat->refcnt, 1); | ioat->refcnt++; | ||||
KASSERT(old < UINT32_MAX, ("refcnt overflow")); | |||||
#ifdef INVARIANTS | |||||
old = atomic_fetchadd_32(&ioat->refkinds[kind], 1); | |||||
KASSERT(old < UINT32_MAX, ("refcnt kind overflow")); | |||||
#endif | |||||
return (ioat); | |||||
} | } | ||||
static inline void | static void | ||||
ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) | ioat_put(struct ioat_softc *ioat) | ||||
{ | { | ||||
_ioat_putn(ioat, n, kind, FALSE); | mtx_assert(&ioat->submit_lock, MA_OWNED); | ||||
} | KASSERT(ioat->refcnt >= 1, ("refcnt error")); | ||||
static inline void | if (--ioat->refcnt == 0) | ||||
ioat_putn_locked(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) | wakeup(&ioat->refcnt); | ||||
{ | |||||
_ioat_putn(ioat, n, kind, TRUE); | |||||
} | } | ||||
static inline void | |||||
_ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind, | |||||
boolean_t locked) | |||||
{ | |||||
uint32_t old; | |||||
KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); | |||||
if (n == 0) | |||||
return; | |||||
#ifdef INVARIANTS | |||||
old = atomic_fetchadd_32(&ioat->refkinds[kind], -n); | |||||
KASSERT(old >= n, ("refcnt kind underflow")); | |||||
#endif | |||||
/* Skip acquiring the lock if resulting refcnt > 0. */ | |||||
for (;;) { | |||||
old = ioat->refcnt; | |||||
if (old <= n) | |||||
break; | |||||
if (atomic_cmpset_32(&ioat->refcnt, old, old - n)) | |||||
return; | |||||
} | |||||
if (locked) | |||||
mtx_assert(IOAT_REFLK, MA_OWNED); | |||||
else | |||||
mtx_lock(IOAT_REFLK); | |||||
old = atomic_fetchadd_32(&ioat->refcnt, -n); | |||||
KASSERT(old >= n, ("refcnt error")); | |||||
if (old == n) | |||||
wakeup(IOAT_REFLK); | |||||
if (!locked) | |||||
mtx_unlock(IOAT_REFLK); | |||||
} | |||||
static inline void | |||||
ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind) | |||||
{ | |||||
ioat_putn(ioat, 1, kind); | |||||
} | |||||
static void | static void | ||||
ioat_drain_locked(struct ioat_softc *ioat) | ioat_drain_locked(struct ioat_softc *ioat) | ||||
{ | { | ||||
mtx_assert(IOAT_REFLK, MA_OWNED); | mtx_assert(&ioat->submit_lock, MA_OWNED); | ||||
while (ioat->refcnt > 0) | while (ioat->refcnt > 0) | ||||
msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0); | msleep(&ioat->refcnt, &ioat->submit_lock, 0, "ioat_drain", 0); | ||||
} | } | ||||
#ifdef DDB | #ifdef DDB | ||||
#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) | #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) | ||||
#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) | #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) | ||||
DB_SHOW_COMMAND(ioat, db_show_ioat) | DB_SHOW_COMMAND(ioat, db_show_ioat) | ||||
{ | { | ||||
struct ioat_softc *sc; | struct ioat_softc *sc; | ||||
Show All 26 Lines | DB_SHOW_COMMAND(ioat, db_show_ioat) | ||||
db_printf(" c_func: %p\n", sc->poll_timer.c_func); | db_printf(" c_func: %p\n", sc->poll_timer.c_func); | ||||
db_printf(" c_lock: %p\n", sc->poll_timer.c_lock); | db_printf(" c_lock: %p\n", sc->poll_timer.c_lock); | ||||
db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags); | db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags); | ||||
db_printf(" quiescing: %d\n", (int)sc->quiescing); | db_printf(" quiescing: %d\n", (int)sc->quiescing); | ||||
db_printf(" destroying: %d\n", (int)sc->destroying); | db_printf(" destroying: %d\n", (int)sc->destroying); | ||||
db_printf(" is_submitter_processing: %d\n", | db_printf(" is_submitter_processing: %d\n", | ||||
(int)sc->is_submitter_processing); | (int)sc->is_submitter_processing); | ||||
db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending); | |||||
db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending); | |||||
db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running); | |||||
db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported); | db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported); | ||||
db_printf(" resetting: %d\n", (int)sc->resetting); | db_printf(" resetting: %d\n", (int)sc->resetting); | ||||
db_printf(" head: %u\n", sc->head); | db_printf(" head: %u\n", sc->head); | ||||
db_printf(" tail: %u\n", sc->tail); | db_printf(" tail: %u\n", sc->tail); | ||||
db_printf(" hw_head: %u\n", sc->hw_head); | |||||
db_printf(" ring_size_order: %u\n", sc->ring_size_order); | db_printf(" ring_size_order: %u\n", sc->ring_size_order); | ||||
db_printf(" last_seen: 0x%lx\n", sc->last_seen); | db_printf(" last_seen: 0x%lx\n", sc->last_seen); | ||||
db_printf(" ring: %p\n", sc->ring); | db_printf(" ring: %p\n", sc->ring); | ||||
db_printf(" descriptors: %p\n", sc->hw_desc_ring); | db_printf(" descriptors: %p\n", sc->hw_desc_ring); | ||||
db_printf(" descriptors (phys): 0x%jx\n", | db_printf(" descriptors (phys): 0x%jx\n", | ||||
(uintmax_t)sc->hw_desc_bus_addr); | (uintmax_t)sc->hw_desc_bus_addr); | ||||
db_printf(" ring[%u] (tail):\n", sc->tail % | db_printf(" ring[%u] (tail):\n", sc->tail % | ||||
Show All 24 Lines | for (idx = 0; idx < (1 << sc->ring_size_order); idx++) | ||||
if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK) | if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK) | ||||
== RING_PHYS_ADDR(sc, idx)) | == RING_PHYS_ADDR(sc, idx)) | ||||
db_printf(" ring[%u] == hardware tail\n", idx); | db_printf(" ring[%u] == hardware tail\n", idx); | ||||
db_printf(" cleanup_lock: "); | db_printf(" cleanup_lock: "); | ||||
db_show_lock(&sc->cleanup_lock); | db_show_lock(&sc->cleanup_lock); | ||||
db_printf(" refcnt: %u\n", sc->refcnt); | db_printf(" refcnt: %u\n", sc->refcnt); | ||||
#ifdef INVARIANTS | |||||
CTASSERT(IOAT_NUM_REF_KINDS == 2); | |||||
db_printf(" refkinds: [ENG=%u, DESCR=%u]\n", sc->refkinds[0], | |||||
sc->refkinds[1]); | |||||
#endif | |||||
db_printf(" stats:\n"); | db_printf(" stats:\n"); | ||||
db_printf(" interrupts: %lu\n", sc->stats.interrupts); | db_printf(" interrupts: %lu\n", sc->stats.interrupts); | ||||
db_printf(" descriptors_processed: %lu\n", sc->stats.descriptors_processed); | db_printf(" descriptors_processed: %lu\n", sc->stats.descriptors_processed); | ||||
db_printf(" descriptors_error: %lu\n", sc->stats.descriptors_error); | db_printf(" descriptors_error: %lu\n", sc->stats.descriptors_error); | ||||
db_printf(" descriptors_submitted: %lu\n", sc->stats.descriptors_submitted); | db_printf(" descriptors_submitted: %lu\n", sc->stats.descriptors_submitted); | ||||
db_printf(" channel_halts: %u\n", sc->stats.channel_halts); | db_printf(" channel_halts: %u\n", sc->stats.channel_halts); | ||||
db_printf(" last_halt_chanerr: %u\n", sc->stats.last_halt_chanerr); | db_printf(" last_halt_chanerr: %u\n", sc->stats.last_halt_chanerr); | ||||
Show All 24 Lines |
Maybe ENFILE? Or print something to console/log if we ever encounter this condition (probably unlikely).