Index: sys/dev/ioat/ioat.c =================================================================== --- sys/dev/ioat/ioat.c +++ sys/dev/ioat/ioat.c @@ -1,6 +1,7 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. + * Copyright (C) 2018 Alexander Motin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -62,7 +63,6 @@ #ifndef BUS_SPACE_MAXADDR_40BIT #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL #endif -#define IOAT_REFLK (&ioat->submit_lock) static int ioat_probe(device_t device); static int ioat_attach(device_t device); @@ -77,7 +77,7 @@ static void ioat_interrupt_handler(void *arg); static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat); static int chanerr_to_errno(uint32_t); -static void ioat_process_events(struct ioat_softc *ioat); +static void ioat_process_events(struct ioat_softc *ioat, boolean_t intr); static inline uint32_t ioat_get_active(struct ioat_softc *ioat); static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); static void ioat_free_ring(struct ioat_softc *, uint32_t size, @@ -97,15 +97,8 @@ static void ioat_reset_hw_task(void *, int); static void ioat_setup_sysctl(device_t device); static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); -static inline struct ioat_softc *ioat_get(struct ioat_softc *, - enum ioat_ref_kind); -static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind); -static inline void _ioat_putn(struct ioat_softc *, uint32_t, - enum ioat_ref_kind, boolean_t); -static inline void ioat_putn(struct ioat_softc *, uint32_t, - enum ioat_ref_kind); -static inline void ioat_putn_locked(struct ioat_softc *, uint32_t, - enum ioat_ref_kind); +static void ioat_get(struct ioat_softc *); +static void ioat_put(struct ioat_softc *); static void ioat_drain_locked(struct ioat_softc *); #define ioat_log_message(v, ...) do { \ @@ -157,6 +150,8 @@ static unsigned ioat_channel_index = 0; SYSCTL_UINT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, "Number of IOAT channels attached"); +static struct mtx ioat_list_mtx; +MTX_SYSINIT(ioat_list_mtx, &ioat_list_mtx, "ioat list mtx", MTX_DEF); static struct _pcsid { @@ -266,7 +261,7 @@ ioat_attach(device_t device) { struct ioat_softc *ioat; - int error; + int error, i; ioat = DEVICE2SOFTC(device); ioat->device = device; @@ -297,11 +292,25 @@ if (error != 0) goto err; - ioat_process_events(ioat); + ioat_process_events(ioat, FALSE); ioat_setup_sysctl(device); - ioat->chan_idx = ioat_channel_index; - ioat_channel[ioat_channel_index++] = ioat; + mtx_lock(&ioat_list_mtx); + for (i = 0; i < IOAT_MAX_CHANNELS; i++) { + if (ioat_channel[i] == NULL) + break; + } + if (i >= IOAT_MAX_CHANNELS) { + mtx_unlock(&ioat_list_mtx); + error = ENXIO; + goto err; + } + ioat->chan_idx = i; + ioat_channel[i] = ioat; + if (i >= ioat_channel_index) + ioat_channel_index = i + 1; + mtx_unlock(&ioat_list_mtx); + ioat_test_attach(); err: @@ -317,19 +326,28 @@ ioat = DEVICE2SOFTC(device); + mtx_lock(&ioat_list_mtx); + ioat_channel[ioat->chan_idx] = NULL; + while (ioat_channel_index > 0 && + ioat_channel[ioat_channel_index - 1] == NULL) + ioat_channel_index--; + mtx_unlock(&ioat_list_mtx); + ioat_test_detach(); taskqueue_drain(taskqueue_thread, &ioat->reset_task); - mtx_lock(IOAT_REFLK); + mtx_lock(&ioat->submit_lock); ioat->quiescing = TRUE; ioat->destroying = TRUE; wakeup(&ioat->quiescing); wakeup(&ioat->resetting); - ioat_channel[ioat->chan_idx] = NULL; - ioat_drain_locked(ioat); - mtx_unlock(IOAT_REFLK); + mtx_unlock(&ioat->submit_lock); + mtx_lock(&ioat->cleanup_lock); + while (ioat_get_active(ioat) > 0) + msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1); + mtx_unlock(&ioat->cleanup_lock); ioat_teardown_intr(ioat); callout_drain(&ioat->poll_timer); @@ -458,15 +476,12 @@ TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); /* Establish lock order for Witness */ - mtx_lock(&ioat->submit_lock); mtx_lock(&ioat->cleanup_lock); - mtx_unlock(&ioat->cleanup_lock); + mtx_lock(&ioat->submit_lock); mtx_unlock(&ioat->submit_lock); + mtx_unlock(&ioat->cleanup_lock); ioat->is_submitter_processing = FALSE; - ioat->is_completion_pending = FALSE; - ioat->is_reset_pending = FALSE; - ioat->is_channel_running = FALSE; bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, @@ -521,7 +536,7 @@ dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1); } - ioat->head = ioat->hw_head = 0; + ioat->head = 0; ioat->tail = 0; ioat->last_seen = 0; *ioat->comp_update = 0; @@ -641,7 +656,7 @@ struct ioat_softc *ioat = arg; ioat->stats.interrupts++; - ioat_process_events(ioat); + ioat_process_events(ioat, TRUE); } static int @@ -661,13 +676,12 @@ } static void -ioat_process_events(struct ioat_softc *ioat) +ioat_process_events(struct ioat_softc *ioat, boolean_t intr) { struct ioat_descriptor *desc; struct bus_dmadesc *dmadesc; uint64_t comp_update, status; uint32_t completed, chanerr; - boolean_t pending; int error; mtx_lock(&ioat->cleanup_lock); @@ -721,42 +735,21 @@ if (completed != 0) { ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1); ioat->stats.descriptors_processed += completed; + wakeup(&ioat->tail); } out: ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); - - /* Perform a racy check first; only take the locks if it passes. */ - pending = (ioat_get_active(ioat) != 0); - if (!pending && ioat->is_completion_pending) { - mtx_unlock(&ioat->cleanup_lock); - mtx_lock(&ioat->submit_lock); - mtx_lock(&ioat->cleanup_lock); - - pending = (ioat_get_active(ioat) != 0); - if (!pending && ioat->is_completion_pending) { - ioat->is_completion_pending = FALSE; - callout_stop(&ioat->poll_timer); - } - mtx_unlock(&ioat->submit_lock); - } mtx_unlock(&ioat->cleanup_lock); - if (pending) - callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, - ioat); - - if (completed != 0) { - ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); - wakeup(&ioat->tail); - } - /* * The device doesn't seem to reliably push suspend/halt statuses to * the channel completion memory address, so poll the device register - * here. + * here. For performance reasons skip it on interrupts, do it only + * on much more rare polling events. */ - comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS; + if (!intr) + comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS; if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) return; @@ -767,16 +760,17 @@ * work with error status and restart the engine. */ mtx_lock(&ioat->submit_lock); - mtx_lock(&ioat->cleanup_lock); ioat->quiescing = TRUE; + mtx_unlock(&ioat->submit_lock); + /* - * This is safe to do here because we have both locks and the submit - * queue is quiesced. We know that we will drain all outstanding - * events, so ioat_reset_hw can't deadlock. It is necessary to - * protect other ioat_process_event threads from racing ioat_reset_hw, - * reading an indeterminate hw state, and attempting to continue - * issuing completions. + * This is safe to do here because the submit queue is quiesced. We + * know that we will drain all outstanding events, so ioat_reset_hw + * can't deadlock. It is necessary to protect other ioat_process_event + * threads from racing ioat_reset_hw, reading an indeterminate hw + * state, and attempting to continue issuing completions. */ + mtx_lock(&ioat->cleanup_lock); ioat->resetting_cleanup = TRUE; chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); @@ -795,7 +789,6 @@ dmadesc->callback_fn(dmadesc->callback_arg, chanerr_to_errno(chanerr)); - ioat_putn_locked(ioat, 1, IOAT_ACTIVE_DESCR_REF); ioat->tail++; ioat->stats.descriptors_processed++; ioat->stats.descriptors_error++; @@ -803,16 +796,10 @@ CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); - if (ioat->is_completion_pending) { - ioat->is_completion_pending = FALSE; - callout_stop(&ioat->poll_timer); - } - /* Clear error status */ ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); mtx_unlock(&ioat->cleanup_lock); - mtx_unlock(&ioat->submit_lock); ioat_log_message(0, "Resetting channel to recover from error\n"); error = taskqueue_enqueue(taskqueue_thread, &ioat->reset_task); @@ -854,40 +841,49 @@ KASSERT((flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK), ("invalid wait | nowait")); - if (index >= ioat_channel_index) + mtx_lock(&ioat_list_mtx); + if (index >= ioat_channel_index || + (ioat = ioat_channel[index]) == NULL) { + mtx_unlock(&ioat_list_mtx); return (NULL); + } + mtx_lock(&ioat->submit_lock); + mtx_unlock(&ioat_list_mtx); - ioat = ioat_channel[index]; - if (ioat == NULL || ioat->destroying) + if (ioat->destroying) { + mtx_unlock(&ioat->submit_lock); return (NULL); + } + ioat_get(ioat); if (ioat->quiescing) { - if ((flags & M_NOWAIT) != 0) + if ((flags & M_NOWAIT) != 0) { + ioat_put(ioat); + mtx_unlock(&ioat->submit_lock); return (NULL); + } - mtx_lock(IOAT_REFLK); while (ioat->quiescing && !ioat->destroying) - msleep(&ioat->quiescing, IOAT_REFLK, 0, "getdma", 0); - mtx_unlock(IOAT_REFLK); + msleep(&ioat->quiescing, &ioat->submit_lock, 0, "getdma", 0); - if (ioat->destroying) + if (ioat->destroying) { + ioat_put(ioat); + mtx_unlock(&ioat->submit_lock); return (NULL); + } } - - /* - * There's a race here between the quiescing check and HW reset or - * module destroy. - */ - return (&ioat_get(ioat, IOAT_DMAENGINE_REF)->dmaengine); + mtx_unlock(&ioat->submit_lock); + return (&ioat->dmaengine); } void ioat_put_dmaengine(bus_dmaengine_t dmaengine) { - struct ioat_softc *ioat; + struct ioat_softc *ioat = to_ioat_softc(dmaengine); - ioat = to_ioat_softc(dmaengine); - ioat_put(ioat, IOAT_DMAENGINE_REF); + mtx_lock(&ioat->submit_lock); + ioat_put(ioat); + mtx_unlock(&ioat->submit_lock); } int @@ -975,19 +971,18 @@ struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); - CTR4(KTR_IOAT, "%s channel=%u dispatch1 hw_head=%u head=%u", __func__, - ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); + CTR3(KTR_IOAT, "%s channel=%u dispatch1 head=%u", __func__, + ioat->chan_idx, ioat->head); KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */); - CTR4(KTR_IOAT, "%s channel=%u dispatch2 hw_head=%u head=%u", __func__, - ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); + CTR3(KTR_IOAT, "%s channel=%u dispatch2 head=%u", __func__, + ioat->chan_idx, ioat->head); if (ioat->acq_head != ioat->head) { ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, - (uint16_t)ioat->hw_head); + (uint16_t)ioat->head); - if (!ioat->is_completion_pending) { - ioat->is_completion_pending = TRUE; - callout_reset(&ioat->poll_timer, 1, + if (!callout_pending(&ioat->poll_timer)) { + callout_reset(&ioat->poll_timer, hz / 100, ioat_poll_timer_callback, ioat); } } @@ -1403,7 +1398,7 @@ CTR2(KTR_IOAT, "%s channel=%u attempting to process events", __func__, ioat->chan_idx); - ioat_process_events(ioat); + ioat_process_events(ioat, FALSE); mtx_lock(&ioat->submit_lock); dug = TRUE; @@ -1481,7 +1476,12 @@ ioat = arg; ioat_log_message(3, "%s\n", __func__); - ioat_process_events(ioat); + ioat_process_events(ioat, FALSE); + + mtx_lock(&ioat->submit_lock); + if (ioat_get_active(ioat) > 0) + callout_schedule(&ioat->poll_timer, hz / 100); + mtx_unlock(&ioat->submit_lock); } /* @@ -1493,12 +1493,9 @@ mtx_assert(&ioat->submit_lock, MA_OWNED); - ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); - atomic_add_rel_int(&ioat->head, 1); - atomic_add_rel_int(&ioat->hw_head, 1); - CTR5(KTR_IOAT, "%s channel=%u head=%u hw_head=%u tail=%u", __func__, - ioat->chan_idx, ioat->head, ioat->hw_head & UINT16_MAX, - ioat->tail); + ioat->head++; + CTR4(KTR_IOAT, "%s channel=%u head=%u tail=%u", __func__, + ioat->chan_idx, ioat->head, ioat->tail); ioat->stats.descriptors_submitted++; } @@ -1513,24 +1510,24 @@ CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); - mtx_lock(IOAT_REFLK); + mtx_lock(&ioat->submit_lock); while (ioat->resetting && !ioat->destroying) - msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0); + msleep(&ioat->resetting, &ioat->submit_lock, 0, "IRH_drain", 0); if (ioat->destroying) { - mtx_unlock(IOAT_REFLK); + mtx_unlock(&ioat->submit_lock); return (ENXIO); } ioat->resetting = TRUE; - ioat->quiescing = TRUE; - ioat_drain_locked(ioat); - mtx_unlock(IOAT_REFLK); + mtx_unlock(&ioat->submit_lock); + mtx_lock(&ioat->cleanup_lock); + while (ioat_get_active(ioat) > 0) + msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1); /* * Suspend ioat_process_events while the hardware and softc are in an * indeterminate state. */ - mtx_lock(&ioat->cleanup_lock); ioat->resetting_cleanup = TRUE; mtx_unlock(&ioat->cleanup_lock); @@ -1621,10 +1618,9 @@ * The internal ring counter resets to zero, so we have to start over * at zero as well. */ - ioat->tail = ioat->head = ioat->hw_head = 0; + ioat->tail = ioat->head = 0; ioat->last_seen = 0; *ioat->comp_update = 0; - KASSERT(!ioat->is_completion_pending, ("bogus completion_pending")); ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); @@ -1649,18 +1645,15 @@ mtx_unlock(&ioat->cleanup_lock); /* Unblock submission of new work */ - mtx_lock(IOAT_REFLK); + mtx_lock(&ioat->submit_lock); ioat->quiescing = FALSE; wakeup(&ioat->quiescing); ioat->resetting = FALSE; wakeup(&ioat->resetting); - if (ioat->is_completion_pending) - callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, - ioat); CTR2(KTR_IOAT, "%s channel=%u reset done", __func__, ioat->chan_idx); - mtx_unlock(IOAT_REFLK); + mtx_unlock(&ioat->submit_lock); return (error); } @@ -1804,8 +1797,6 @@ 0, "SW descriptor head pointer index"); SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0, "SW descriptor tail pointer index"); - SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "hw_head", CTLFLAG_RD, - &ioat->hw_head, 0, "HW DMACOUNT"); SYSCTL_ADD_UQUAD(ctx, state, OID_AUTO, "last_completion", CTLFLAG_RD, ioat->comp_update, "HW addr of last completion"); @@ -1813,12 +1804,6 @@ SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_submitter_processing", CTLFLAG_RD, &ioat->is_submitter_processing, 0, "submitter processing"); - SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_completion_pending", - CTLFLAG_RD, &ioat->is_completion_pending, 0, "completion pending"); - SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_reset_pending", CTLFLAG_RD, - &ioat->is_reset_pending, 0, "reset pending"); - SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_channel_running", CTLFLAG_RD, - &ioat->is_channel_running, 0, "channel running"); SYSCTL_ADD_PROC(ctx, state, OID_AUTO, "chansts", CTLTYPE_STRING | CTLFLAG_RD, ioat, 0, sysctl_handle_chansts, "A", @@ -1864,91 +1849,35 @@ "Descriptors per interrupt"); } -static inline struct ioat_softc * -ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind) +static void +ioat_get(struct ioat_softc *ioat) { - uint32_t old; - KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); + mtx_assert(&ioat->submit_lock, MA_OWNED); + KASSERT(ioat->refcnt < UINT32_MAX, ("refcnt overflow")); - old = atomic_fetchadd_32(&ioat->refcnt, 1); - KASSERT(old < UINT32_MAX, ("refcnt overflow")); - -#ifdef INVARIANTS - old = atomic_fetchadd_32(&ioat->refkinds[kind], 1); - KASSERT(old < UINT32_MAX, ("refcnt kind overflow")); -#endif - - return (ioat); + ioat->refcnt++; } -static inline void -ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) +static void +ioat_put(struct ioat_softc *ioat) { - _ioat_putn(ioat, n, kind, FALSE); -} + mtx_assert(&ioat->submit_lock, MA_OWNED); + KASSERT(ioat->refcnt >= 1, ("refcnt error")); -static inline void -ioat_putn_locked(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) -{ - - _ioat_putn(ioat, n, kind, TRUE); + if (--ioat->refcnt == 0) + wakeup(&ioat->refcnt); } -static inline void -_ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind, - boolean_t locked) -{ - uint32_t old; - - KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); - - if (n == 0) - return; - -#ifdef INVARIANTS - old = atomic_fetchadd_32(&ioat->refkinds[kind], -n); - KASSERT(old >= n, ("refcnt kind underflow")); -#endif - - /* Skip acquiring the lock if resulting refcnt > 0. */ - for (;;) { - old = ioat->refcnt; - if (old <= n) - break; - if (atomic_cmpset_32(&ioat->refcnt, old, old - n)) - return; - } - - if (locked) - mtx_assert(IOAT_REFLK, MA_OWNED); - else - mtx_lock(IOAT_REFLK); - - old = atomic_fetchadd_32(&ioat->refcnt, -n); - KASSERT(old >= n, ("refcnt error")); - - if (old == n) - wakeup(IOAT_REFLK); - if (!locked) - mtx_unlock(IOAT_REFLK); -} - -static inline void -ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind) -{ - - ioat_putn(ioat, 1, kind); -} - static void ioat_drain_locked(struct ioat_softc *ioat) { - mtx_assert(IOAT_REFLK, MA_OWNED); + mtx_assert(&ioat->submit_lock, MA_OWNED); + while (ioat->refcnt > 0) - msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0); + msleep(&ioat->refcnt, &ioat->submit_lock, 0, "ioat_drain", 0); } #ifdef DDB @@ -1991,15 +1920,11 @@ db_printf(" destroying: %d\n", (int)sc->destroying); db_printf(" is_submitter_processing: %d\n", (int)sc->is_submitter_processing); - db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending); - db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending); - db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running); db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported); db_printf(" resetting: %d\n", (int)sc->resetting); db_printf(" head: %u\n", sc->head); db_printf(" tail: %u\n", sc->tail); - db_printf(" hw_head: %u\n", sc->hw_head); db_printf(" ring_size_order: %u\n", sc->ring_size_order); db_printf(" last_seen: 0x%lx\n", sc->last_seen); db_printf(" ring: %p\n", sc->ring); @@ -2040,11 +1965,6 @@ db_show_lock(&sc->cleanup_lock); db_printf(" refcnt: %u\n", sc->refcnt); -#ifdef INVARIANTS - CTASSERT(IOAT_NUM_REF_KINDS == 2); - db_printf(" refkinds: [ENG=%u, DESCR=%u]\n", sc->refkinds[0], - sc->refkinds[1]); -#endif db_printf(" stats:\n"); db_printf(" interrupts: %lu\n", sc->stats.interrupts); db_printf(" descriptors_processed: %lu\n", sc->stats.descriptors_processed); Index: sys/dev/ioat/ioat_internal.h =================================================================== --- sys/dev/ioat/ioat_internal.h +++ sys/dev/ioat/ioat_internal.h @@ -426,12 +426,6 @@ #define IOAT_OP_OLD_XOR 0x85 #define IOAT_OP_OLD_XOR_VAL 0x86 -enum ioat_ref_kind { - IOAT_DMAENGINE_REF = 0, - IOAT_ACTIVE_DESCR_REF, - IOAT_NUM_REF_KINDS -}; - /* One of these per allocated PCI device. */ struct ioat_softc { bus_dmaengine_t dmaengine; @@ -442,22 +436,22 @@ offsetof(struct ioat_softc, dmaengine)); \ }) + device_t device; int version; unsigned chan_idx; - struct mtx submit_lock; - device_t device; bus_space_tag_t pci_bus_tag; bus_space_handle_t pci_bus_handle; - int pci_resource_id; struct resource *pci_resource; + int pci_resource_id; uint32_t max_xfer_size; uint32_t capabilities; + uint32_t ring_size_order; uint16_t intrdelay_max; uint16_t cached_intrdelay; - struct resource *res; int rid; + struct resource *res; void *tag; bus_dma_tag_t hw_desc_tag; @@ -468,27 +462,13 @@ uint64_t *comp_update; bus_addr_t comp_update_bus_addr; - struct callout poll_timer; - struct callout shrink_timer; - struct task reset_task; - boolean_t quiescing; boolean_t destroying; boolean_t is_submitter_processing; - boolean_t is_completion_pending; /* submit_lock */ - boolean_t is_reset_pending; - boolean_t is_channel_running; boolean_t intrdelay_supported; boolean_t resetting; /* submit_lock */ boolean_t resetting_cleanup; /* cleanup_lock */ - uint32_t head; - uint32_t acq_head; - uint32_t tail; - uint32_t hw_head; - uint32_t ring_size_order; - bus_addr_t last_seen; - struct ioat_descriptor *ring; union ioat_hw_descriptor { @@ -506,11 +486,17 @@ #define RING_PHYS_ADDR(sc, i) (sc)->hw_desc_bus_addr + \ (((i) % (1 << (sc)->ring_size_order)) * sizeof(struct ioat_dma_hw_descriptor)) + struct mtx_padalign submit_lock; struct mtx cleanup_lock; - volatile uint32_t refcnt; -#ifdef INVARIANTS - volatile uint32_t refkinds[IOAT_NUM_REF_KINDS]; -#endif + + struct task reset_task; + struct callout poll_timer; + + uint32_t refcnt; + uint32_t head; + uint32_t acq_head; + uint32_t tail; + bus_addr_t last_seen; struct { uint64_t interrupts;