Differential D19231 Diff 54189 head/sys/dev/ioat/ioat.c

Changeset View

Standalone View

head/sys/dev/ioat/ioat.c

/*-		/*-
* Copyright (C) 2012 Intel Corporation		* Copyright (C) 2012 Intel Corporation
* All rights reserved.		* All rights reserved.
		* Copyright (C) 2018 Alexander Motin <mav@FreeBSD.org>
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
* 1. Redistributions of source code must retain the above copyright		* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.		* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright		* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the		* notice, this list of conditions and the following disclaimer in the
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines

#include "ioat.h"		#include "ioat.h"
#include "ioat_hw.h"		#include "ioat_hw.h"
#include "ioat_internal.h"		#include "ioat_internal.h"

#ifndef BUS_SPACE_MAXADDR_40BIT		#ifndef BUS_SPACE_MAXADDR_40BIT
#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL		#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL
#endif		#endif
#define IOAT_REFLK (&ioat->submit_lock)

static int ioat_probe(device_t device);		static int ioat_probe(device_t device);
static int ioat_attach(device_t device);		static int ioat_attach(device_t device);
static int ioat_detach(device_t device);		static int ioat_detach(device_t device);
static int ioat_setup_intr(struct ioat_softc *ioat);		static int ioat_setup_intr(struct ioat_softc *ioat);
static int ioat_teardown_intr(struct ioat_softc *ioat);		static int ioat_teardown_intr(struct ioat_softc *ioat);
static int ioat3_attach(device_t device);		static int ioat3_attach(device_t device);
static int ioat_start_channel(struct ioat_softc *ioat);		static int ioat_start_channel(struct ioat_softc *ioat);
static int ioat_map_pci_bar(struct ioat_softc *ioat);		static int ioat_map_pci_bar(struct ioat_softc *ioat);
static void ioat_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg,		static void ioat_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg,
int error);		int error);
static void ioat_interrupt_handler(void *arg);		static void ioat_interrupt_handler(void *arg);
static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat);		static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat);
static int chanerr_to_errno(uint32_t);		static int chanerr_to_errno(uint32_t);
static void ioat_process_events(struct ioat_softc *ioat);		static void ioat_process_events(struct ioat_softc *ioat, boolean_t intr);
static inline uint32_t ioat_get_active(struct ioat_softc *ioat);		static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);		static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
static void ioat_free_ring(struct ioat_softc *, uint32_t size,		static void ioat_free_ring(struct ioat_softc *, uint32_t size,
struct ioat_descriptor *);		struct ioat_descriptor *);
static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags);		static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags);
static union ioat_hw_descriptor ioat_get_descriptor(struct ioat_softc ,		static union ioat_hw_descriptor ioat_get_descriptor(struct ioat_softc ,
uint32_t index);		uint32_t index);
static struct ioat_descriptor ioat_get_ring_entry(struct ioat_softc ,		static struct ioat_descriptor ioat_get_ring_entry(struct ioat_softc ,
uint32_t index);		uint32_t index);
static void ioat_halted_debug(struct ioat_softc *, uint32_t);		static void ioat_halted_debug(struct ioat_softc *, uint32_t);
static void ioat_poll_timer_callback(void *arg);		static void ioat_poll_timer_callback(void *arg);
static void dump_descriptor(void *hw_desc);		static void dump_descriptor(void *hw_desc);
static void ioat_submit_single(struct ioat_softc *ioat);		static void ioat_submit_single(struct ioat_softc *ioat);
static void ioat_comp_update_map(void arg, bus_dma_segment_t seg, int nseg,		static void ioat_comp_update_map(void arg, bus_dma_segment_t seg, int nseg,
int error);		int error);
static int ioat_reset_hw(struct ioat_softc *ioat);		static int ioat_reset_hw(struct ioat_softc *ioat);
static void ioat_reset_hw_task(void *, int);		static void ioat_reset_hw_task(void *, int);
static void ioat_setup_sysctl(device_t device);		static void ioat_setup_sysctl(device_t device);
static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);		static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);
static inline struct ioat_softc ioat_get(struct ioat_softc ,		static void ioat_get(struct ioat_softc *);
enum ioat_ref_kind);		static void ioat_put(struct ioat_softc *);
static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind);
static inline void _ioat_putn(struct ioat_softc *, uint32_t,
enum ioat_ref_kind, boolean_t);
static inline void ioat_putn(struct ioat_softc *, uint32_t,
enum ioat_ref_kind);
static inline void ioat_putn_locked(struct ioat_softc *, uint32_t,
enum ioat_ref_kind);
static void ioat_drain_locked(struct ioat_softc *);		static void ioat_drain_locked(struct ioat_softc *);

#define ioat_log_message(v, ...) do { \		#define ioat_log_message(v, ...) do { \
if ((v) <= g_ioat_debug_level) { \		if ((v) <= g_ioat_debug_level) { \
device_printf(ioat->device, __VA_ARGS__); \		device_printf(ioat->device, __VA_ARGS__); \
} \		} \
} while (0)		} while (0)

Show All 35 Lines

/*		/*
* Private data structures		* Private data structures
*/		*/
static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];		static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
static unsigned ioat_channel_index = 0;		static unsigned ioat_channel_index = 0;
SYSCTL_UINT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,		SYSCTL_UINT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
"Number of IOAT channels attached");		"Number of IOAT channels attached");
		static struct mtx ioat_list_mtx;
		MTX_SYSINIT(ioat_list_mtx, &ioat_list_mtx, "ioat list mtx", MTX_DEF);

static struct _pcsid		static struct _pcsid
{		{
u_int32_t type;		u_int32_t type;
const char *desc;		const char *desc;
} pci_ids[] = {		} pci_ids[] = {
{ 0x34308086, "TBG IOAT Ch0" },		{ 0x34308086, "TBG IOAT Ch0" },
{ 0x34318086, "TBG IOAT Ch1" },		{ 0x34318086, "TBG IOAT Ch1" },
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines	ioat_probe(device_t device)
}		}
return (ENXIO);		return (ENXIO);
}		}

static int		static int
ioat_attach(device_t device)		ioat_attach(device_t device)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;
int error;		int error, i;

ioat = DEVICE2SOFTC(device);		ioat = DEVICE2SOFTC(device);
ioat->device = device;		ioat->device = device;

error = ioat_map_pci_bar(ioat);		error = ioat_map_pci_bar(ioat);
if (error != 0)		if (error != 0)
goto err;		goto err;

Show All 14 Lines	ioat_attach(device_t device)
error = ioat_setup_intr(ioat);		error = ioat_setup_intr(ioat);
if (error != 0)		if (error != 0)
goto err;		goto err;

error = ioat_reset_hw(ioat);		error = ioat_reset_hw(ioat);
if (error != 0)		if (error != 0)
goto err;		goto err;

ioat_process_events(ioat);		ioat_process_events(ioat, FALSE);
ioat_setup_sysctl(device);		ioat_setup_sysctl(device);

ioat->chan_idx = ioat_channel_index;		mtx_lock(&ioat_list_mtx);
ioat_channel[ioat_channel_index++] = ioat;		for (i = 0; i < IOAT_MAX_CHANNELS; i++) {
		if (ioat_channel[i] == NULL)
		break;
		}
		if (i >= IOAT_MAX_CHANNELS) {
		mtx_unlock(&ioat_list_mtx);
		device_printf(device, "Too many I/OAT devices in system\n");
		error = ENXIO;
		goto err;
		}
		ioat->chan_idx = i;
		ioat_channel[i] = ioat;
		if (i >= ioat_channel_index)
		ioat_channel_index = i + 1;
		mtx_unlock(&ioat_list_mtx);

ioat_test_attach();		ioat_test_attach();

err:		err:
if (error != 0)		if (error != 0)
ioat_detach(device);		ioat_detach(device);
return (error);		return (error);
}		}

static int		static int
ioat_detach(device_t device)		ioat_detach(device_t device)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

ioat = DEVICE2SOFTC(device);		ioat = DEVICE2SOFTC(device);

		mtx_lock(&ioat_list_mtx);
		ioat_channel[ioat->chan_idx] = NULL;
		while (ioat_channel_index > 0 &&
		ioat_channel[ioat_channel_index - 1] == NULL)
		ioat_channel_index--;
		mtx_unlock(&ioat_list_mtx);

ioat_test_detach();		ioat_test_detach();
taskqueue_drain(taskqueue_thread, &ioat->reset_task);		taskqueue_drain(taskqueue_thread, &ioat->reset_task);

mtx_lock(IOAT_REFLK);		mtx_lock(&ioat->submit_lock);
ioat->quiescing = TRUE;		ioat->quiescing = TRUE;
ioat->destroying = TRUE;		ioat->destroying = TRUE;
wakeup(&ioat->quiescing);		wakeup(&ioat->quiescing);
wakeup(&ioat->resetting);		wakeup(&ioat->resetting);

ioat_channel[ioat->chan_idx] = NULL;

ioat_drain_locked(ioat);		ioat_drain_locked(ioat);
mtx_unlock(IOAT_REFLK);		mtx_unlock(&ioat->submit_lock);
		mtx_lock(&ioat->cleanup_lock);
		while (ioat_get_active(ioat) > 0)
		msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1);
		mtx_unlock(&ioat->cleanup_lock);

ioat_teardown_intr(ioat);		ioat_teardown_intr(ioat);
callout_drain(&ioat->poll_timer);		callout_drain(&ioat->poll_timer);

pci_disable_busmaster(device);		pci_disable_busmaster(device);

if (ioat->pci_resource != NULL)		if (ioat->pci_resource != NULL)
bus_release_resource(device, SYS_RES_MEMORY,		bus_release_resource(device, SYS_RES_MEMORY,
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines	ioat3_attach(device_t device)
/* TODO: need to check DCA here if we ever do XOR/PQ */		/* TODO: need to check DCA here if we ever do XOR/PQ */

mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);		mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF);		mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF);
callout_init(&ioat->poll_timer, 1);		callout_init(&ioat->poll_timer, 1);
TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat);		TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat);

/* Establish lock order for Witness */		/* Establish lock order for Witness */
mtx_lock(&ioat->submit_lock);
mtx_lock(&ioat->cleanup_lock);		mtx_lock(&ioat->cleanup_lock);
mtx_unlock(&ioat->cleanup_lock);		mtx_lock(&ioat->submit_lock);
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);
		mtx_unlock(&ioat->cleanup_lock);

ioat->is_submitter_processing = FALSE;		ioat->is_submitter_processing = FALSE;
ioat->is_completion_pending = FALSE;
ioat->is_reset_pending = FALSE;
ioat->is_channel_running = FALSE;

bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,		bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,
BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,		BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,		sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,
&ioat->comp_update_tag);		&ioat->comp_update_tag);

error = bus_dmamem_alloc(ioat->comp_update_tag,		error = bus_dmamem_alloc(ioat->comp_update_tag,
(void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);		(void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
Show All 38 Lines	for (i = 0; i < num_descriptors; i++) {
ring[i].id = i;		ring[i].id = i;
}		}

for (i = 0; i < num_descriptors; i++) {		for (i = 0; i < num_descriptors; i++) {
dma_hw_desc = &ioat->hw_desc_ring[i].dma;		dma_hw_desc = &ioat->hw_desc_ring[i].dma;
dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1);		dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1);
}		}

ioat->head = ioat->hw_head = 0;		ioat->head = 0;
ioat->tail = 0;		ioat->tail = 0;
ioat->last_seen = 0;		ioat->last_seen = 0;
*ioat->comp_update = 0;		*ioat->comp_update = 0;
return (0);		return (0);
}		}

static int		static int
ioat_map_pci_bar(struct ioat_softc *ioat)		ioat_map_pci_bar(struct ioat_softc *ioat)
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines
}		}

static void		static void
ioat_interrupt_handler(void *arg)		ioat_interrupt_handler(void *arg)
{		{
struct ioat_softc *ioat = arg;		struct ioat_softc *ioat = arg;

ioat->stats.interrupts++;		ioat->stats.interrupts++;
ioat_process_events(ioat);		ioat_process_events(ioat, TRUE);
}		}

static int		static int
chanerr_to_errno(uint32_t chanerr)		chanerr_to_errno(uint32_t chanerr)
{		{

if (chanerr == 0)		if (chanerr == 0)
return (0);		return (0);
if ((chanerr & (IOAT_CHANERR_XSADDERR \| IOAT_CHANERR_XDADDERR)) != 0)		if ((chanerr & (IOAT_CHANERR_XSADDERR \| IOAT_CHANERR_XDADDERR)) != 0)
return (EFAULT);		return (EFAULT);
if ((chanerr & (IOAT_CHANERR_RDERR \| IOAT_CHANERR_WDERR)) != 0)		if ((chanerr & (IOAT_CHANERR_RDERR \| IOAT_CHANERR_WDERR)) != 0)
return (EIO);		return (EIO);
/* This one is probably our fault: */		/* This one is probably our fault: */
if ((chanerr & IOAT_CHANERR_NDADDERR) != 0)		if ((chanerr & IOAT_CHANERR_NDADDERR) != 0)
return (EIO);		return (EIO);
return (EIO);		return (EIO);
}		}

static void		static void
ioat_process_events(struct ioat_softc *ioat)		ioat_process_events(struct ioat_softc *ioat, boolean_t intr)
{		{
struct ioat_descriptor *desc;		struct ioat_descriptor *desc;
struct bus_dmadesc *dmadesc;		struct bus_dmadesc *dmadesc;
uint64_t comp_update, status;		uint64_t comp_update, status;
uint32_t completed, chanerr;		uint32_t completed, chanerr;
boolean_t pending;
int error;		int error;

mtx_lock(&ioat->cleanup_lock);		mtx_lock(&ioat->cleanup_lock);

/*		/*
* Don't run while the hardware is being reset. Reset is responsible		* Don't run while the hardware is being reset. Reset is responsible
* for blocking new work and draining & completing existing work, so		* for blocking new work and draining & completing existing work, so
* there is nothing to do until new work is queued after reset anyway.		* there is nothing to do until new work is queued after reset anyway.
Show All 37 Lines	while (RING_PHYS_ADDR(ioat, ioat->tail - 1) != status) {
ioat->tail++;		ioat->tail++;
}		}
CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,		CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,
ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));		ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));

if (completed != 0) {		if (completed != 0) {
ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1);		ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1);
ioat->stats.descriptors_processed += completed;		ioat->stats.descriptors_processed += completed;
		wakeup(&ioat->tail);
}		}

out:		out:
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);		ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);

/* Perform a racy check first; only take the locks if it passes. */
pending = (ioat_get_active(ioat) != 0);
if (!pending && ioat->is_completion_pending) {
mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);
mtx_lock(&ioat->submit_lock);
mtx_lock(&ioat->cleanup_lock);

pending = (ioat_get_active(ioat) != 0);
if (!pending && ioat->is_completion_pending) {
ioat->is_completion_pending = FALSE;
callout_stop(&ioat->poll_timer);
}
mtx_unlock(&ioat->submit_lock);
}
mtx_unlock(&ioat->cleanup_lock);

if (pending)
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
ioat);

if (completed != 0) {
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
wakeup(&ioat->tail);
}

/*		/*
* The device doesn't seem to reliably push suspend/halt statuses to		* The device doesn't seem to reliably push suspend/halt statuses to
* the channel completion memory address, so poll the device register		* the channel completion memory address, so poll the device register
* here.		* here. For performance reasons skip it on interrupts, do it only
		* on much more rare polling events.
*/		*/
		if (!intr)
comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS;		comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS;
if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update))		if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update))
return;		return;

ioat->stats.channel_halts++;		ioat->stats.channel_halts++;

/*		/*
* Fatal programming error on this DMA channel. Flush any outstanding		* Fatal programming error on this DMA channel. Flush any outstanding
* work with error status and restart the engine.		* work with error status and restart the engine.
*/		*/
mtx_lock(&ioat->submit_lock);		mtx_lock(&ioat->submit_lock);
mtx_lock(&ioat->cleanup_lock);
ioat->quiescing = TRUE;		ioat->quiescing = TRUE;
		mtx_unlock(&ioat->submit_lock);

/*		/*
* This is safe to do here because we have both locks and the submit		* This is safe to do here because the submit queue is quiesced. We
* queue is quiesced. We know that we will drain all outstanding		* know that we will drain all outstanding events, so ioat_reset_hw
* events, so ioat_reset_hw can't deadlock. It is necessary to		* can't deadlock. It is necessary to protect other ioat_process_event
* protect other ioat_process_event threads from racing ioat_reset_hw,		* threads from racing ioat_reset_hw, reading an indeterminate hw
* reading an indeterminate hw state, and attempting to continue		* state, and attempting to continue issuing completions.
* issuing completions.
*/		*/
		mtx_lock(&ioat->cleanup_lock);
ioat->resetting_cleanup = TRUE;		ioat->resetting_cleanup = TRUE;

chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);		chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
if (1 <= g_ioat_debug_level)		if (1 <= g_ioat_debug_level)
ioat_halted_debug(ioat, chanerr);		ioat_halted_debug(ioat, chanerr);
ioat->stats.last_halt_chanerr = chanerr;		ioat->stats.last_halt_chanerr = chanerr;

while (ioat_get_active(ioat) > 0) {		while (ioat_get_active(ioat) > 0) {
desc = ioat_get_ring_entry(ioat, ioat->tail);		desc = ioat_get_ring_entry(ioat, ioat->tail);
dmadesc = &desc->bus_dmadesc;		dmadesc = &desc->bus_dmadesc;
CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb %p(%p)",		CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb %p(%p)",
ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn,		ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn,
dmadesc->callback_arg);		dmadesc->callback_arg);

if (dmadesc->callback_fn != NULL)		if (dmadesc->callback_fn != NULL)
dmadesc->callback_fn(dmadesc->callback_arg,		dmadesc->callback_fn(dmadesc->callback_arg,
chanerr_to_errno(chanerr));		chanerr_to_errno(chanerr));

ioat_putn_locked(ioat, 1, IOAT_ACTIVE_DESCR_REF);
ioat->tail++;		ioat->tail++;
ioat->stats.descriptors_processed++;		ioat->stats.descriptors_processed++;
ioat->stats.descriptors_error++;		ioat->stats.descriptors_error++;
}		}
CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,		CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__,
ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));		ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat));

if (ioat->is_completion_pending) {
ioat->is_completion_pending = FALSE;
callout_stop(&ioat->poll_timer);
}

/* Clear error status */		/* Clear error status */
ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);		ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);

mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);
mtx_unlock(&ioat->submit_lock);

ioat_log_message(0, "Resetting channel to recover from error\n");		ioat_log_message(0, "Resetting channel to recover from error\n");
error = taskqueue_enqueue(taskqueue_thread, &ioat->reset_task);		error = taskqueue_enqueue(taskqueue_thread, &ioat->reset_task);
KASSERT(error == 0,		KASSERT(error == 0,
("%s: taskqueue_enqueue failed: %d", __func__, error));		("%s: taskqueue_enqueue failed: %d", __func__, error));
}		}

static void		static void
Show All 25 Lines
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

KASSERT((flags & ~(M_NOWAIT \| M_WAITOK)) == 0,		KASSERT((flags & ~(M_NOWAIT \| M_WAITOK)) == 0,
("invalid flags: 0x%08x", flags));		("invalid flags: 0x%08x", flags));
KASSERT((flags & (M_NOWAIT \| M_WAITOK)) != (M_NOWAIT \| M_WAITOK),		KASSERT((flags & (M_NOWAIT \| M_WAITOK)) != (M_NOWAIT \| M_WAITOK),
("invalid wait \| nowait"));		("invalid wait \| nowait"));

if (index >= ioat_channel_index)		mtx_lock(&ioat_list_mtx);
		if (index >= ioat_channel_index \|\|
		(ioat = ioat_channel[index]) == NULL) {
		mtx_unlock(&ioat_list_mtx);
return (NULL);		return (NULL);
		}
		mtx_lock(&ioat->submit_lock);
		mtx_unlock(&ioat_list_mtx);

ioat = ioat_channel[index];		if (ioat->destroying) {
if (ioat == NULL \|\| ioat->destroying)		mtx_unlock(&ioat->submit_lock);
return (NULL);		return (NULL);
		}

		ioat_get(ioat);
if (ioat->quiescing) {		if (ioat->quiescing) {
if ((flags & M_NOWAIT) != 0)		if ((flags & M_NOWAIT) != 0) {
		ioat_put(ioat);
		mtx_unlock(&ioat->submit_lock);
return (NULL);		return (NULL);
		}

mtx_lock(IOAT_REFLK);
while (ioat->quiescing && !ioat->destroying)		while (ioat->quiescing && !ioat->destroying)
msleep(&ioat->quiescing, IOAT_REFLK, 0, "getdma", 0);		msleep(&ioat->quiescing, &ioat->submit_lock, 0, "getdma", 0);
mtx_unlock(IOAT_REFLK);

if (ioat->destroying)		if (ioat->destroying) {
		ioat_put(ioat);
		mtx_unlock(&ioat->submit_lock);
return (NULL);		return (NULL);
}		}

/*
* There's a race here between the quiescing check and HW reset or
* module destroy.
*/
return (&ioat_get(ioat, IOAT_DMAENGINE_REF)->dmaengine);
}		}
		mtx_unlock(&ioat->submit_lock);
		return (&ioat->dmaengine);
		}

void		void
ioat_put_dmaengine(bus_dmaengine_t dmaengine)		ioat_put_dmaengine(bus_dmaengine_t dmaengine)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

ioat = to_ioat_softc(dmaengine);		ioat = to_ioat_softc(dmaengine);
ioat_put(ioat, IOAT_DMAENGINE_REF);		mtx_lock(&ioat->submit_lock);
		ioat_put(ioat);
		mtx_unlock(&ioat->submit_lock);
}		}

int		int
ioat_get_hwversion(bus_dmaengine_t dmaengine)		ioat_get_hwversion(bus_dmaengine_t dmaengine)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

ioat = to_ioat_softc(dmaengine);		ioat = to_ioat_softc(dmaengine);
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines
}		}

void		void
ioat_release(bus_dmaengine_t dmaengine)		ioat_release(bus_dmaengine_t dmaengine)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

ioat = to_ioat_softc(dmaengine);		ioat = to_ioat_softc(dmaengine);
CTR4(KTR_IOAT, "%s channel=%u dispatch1 hw_head=%u head=%u", __func__,		CTR3(KTR_IOAT, "%s channel=%u dispatch1 head=%u", __func__,
ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head);		ioat->chan_idx, ioat->head);
KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */);		KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */);
CTR4(KTR_IOAT, "%s channel=%u dispatch2 hw_head=%u head=%u", __func__,		CTR3(KTR_IOAT, "%s channel=%u dispatch2 head=%u", __func__,
ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head);		ioat->chan_idx, ioat->head);

if (ioat->acq_head != ioat->head) {		if (ioat->acq_head != ioat->head) {
ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET,		ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET,
(uint16_t)ioat->hw_head);		(uint16_t)ioat->head);

if (!ioat->is_completion_pending) {		if (!callout_pending(&ioat->poll_timer)) {
ioat->is_completion_pending = TRUE;
callout_reset(&ioat->poll_timer, 1,		callout_reset(&ioat->poll_timer, 1,
ioat_poll_timer_callback, ioat);		ioat_poll_timer_callback, ioat);
}		}
}		}
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);
}		}

static struct ioat_descriptor *		static struct ioat_descriptor *
▲ Show 20 Lines • Show All 400 Lines • ▼ Show 20 Lines	CTR3(KTR_IOAT, "%s channel=%u starved (%u)", __func__,
ioat->chan_idx, num_descs);		ioat->chan_idx, num_descs);

if (!dug && !ioat->is_submitter_processing) {		if (!dug && !ioat->is_submitter_processing) {
ioat->is_submitter_processing = TRUE;		ioat->is_submitter_processing = TRUE;
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);

CTR2(KTR_IOAT, "%s channel=%u attempting to process events",		CTR2(KTR_IOAT, "%s channel=%u attempting to process events",
__func__, ioat->chan_idx);		__func__, ioat->chan_idx);
ioat_process_events(ioat);		ioat_process_events(ioat, FALSE);

mtx_lock(&ioat->submit_lock);		mtx_lock(&ioat->submit_lock);
dug = TRUE;		dug = TRUE;
KASSERT(ioat->is_submitter_processing == TRUE,		KASSERT(ioat->is_submitter_processing == TRUE,
("is_submitter_processing"));		("is_submitter_processing"));
ioat->is_submitter_processing = FALSE;		ioat->is_submitter_processing = FALSE;
wakeup(&ioat->tail);		wakeup(&ioat->tail);
continue;		continue;
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines
static void		static void
ioat_poll_timer_callback(void *arg)		ioat_poll_timer_callback(void *arg)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;

ioat = arg;		ioat = arg;
ioat_log_message(3, "%s\n", __func__);		ioat_log_message(3, "%s\n", __func__);

ioat_process_events(ioat);		ioat_process_events(ioat, FALSE);

		mtx_lock(&ioat->submit_lock);
		if (ioat_get_active(ioat) > 0)
		callout_schedule(&ioat->poll_timer, 1);
		mtx_unlock(&ioat->submit_lock);
}		}

/*		/*
* Support Functions		* Support Functions
*/		*/
static void		static void
ioat_submit_single(struct ioat_softc *ioat)		ioat_submit_single(struct ioat_softc *ioat)
{		{

mtx_assert(&ioat->submit_lock, MA_OWNED);		mtx_assert(&ioat->submit_lock, MA_OWNED);

ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);		ioat->head++;
atomic_add_rel_int(&ioat->head, 1);		CTR4(KTR_IOAT, "%s channel=%u head=%u tail=%u", __func__,
atomic_add_rel_int(&ioat->hw_head, 1);		ioat->chan_idx, ioat->head, ioat->tail);
CTR5(KTR_IOAT, "%s channel=%u head=%u hw_head=%u tail=%u", __func__,
ioat->chan_idx, ioat->head, ioat->hw_head & UINT16_MAX,
ioat->tail);

ioat->stats.descriptors_submitted++;		ioat->stats.descriptors_submitted++;
}		}

static int		static int
ioat_reset_hw(struct ioat_softc *ioat)		ioat_reset_hw(struct ioat_softc *ioat)
{		{
uint64_t status;		uint64_t status;
uint32_t chanerr;		uint32_t chanerr;
unsigned timeout;		unsigned timeout;
int error;		int error;

CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);		CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx);

mtx_lock(IOAT_REFLK);		mtx_lock(&ioat->submit_lock);
while (ioat->resetting && !ioat->destroying)		while (ioat->resetting && !ioat->destroying)
msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0);		msleep(&ioat->resetting, &ioat->submit_lock, 0, "IRH_drain", 0);
if (ioat->destroying) {		if (ioat->destroying) {
mtx_unlock(IOAT_REFLK);		mtx_unlock(&ioat->submit_lock);
return (ENXIO);		return (ENXIO);
}		}
ioat->resetting = TRUE;		ioat->resetting = TRUE;

ioat->quiescing = TRUE;		ioat->quiescing = TRUE;
ioat_drain_locked(ioat);		mtx_unlock(&ioat->submit_lock);
mtx_unlock(IOAT_REFLK);		mtx_lock(&ioat->cleanup_lock);
		while (ioat_get_active(ioat) > 0)
		msleep(&ioat->tail, &ioat->cleanup_lock, 0, "ioat_drain", 1);

/*		/*
* Suspend ioat_process_events while the hardware and softc are in an		* Suspend ioat_process_events while the hardware and softc are in an
* indeterminate state.		* indeterminate state.
*/		*/
mtx_lock(&ioat->cleanup_lock);
ioat->resetting_cleanup = TRUE;		ioat->resetting_cleanup = TRUE;
mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);

CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__,		CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__,
ioat->chan_idx);		ioat->chan_idx);

status = ioat_get_chansts(ioat);		status = ioat_get_chansts(ioat);
if (is_ioat_active(status) \|\| is_ioat_idle(status))		if (is_ioat_active(status) \|\| is_ioat_idle(status))
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines	ioat_reset_hw(struct ioat_softc *ioat)

/*		/*
* Bring device back online after reset. Writing CHAINADDR brings the		* Bring device back online after reset. Writing CHAINADDR brings the
* device back to active.		* device back to active.
*		*
* The internal ring counter resets to zero, so we have to start over		* The internal ring counter resets to zero, so we have to start over
* at zero as well.		* at zero as well.
*/		*/
ioat->tail = ioat->head = ioat->hw_head = 0;		ioat->tail = ioat->head = 0;
ioat->last_seen = 0;		ioat->last_seen = 0;
*ioat->comp_update = 0;		*ioat->comp_update = 0;
KASSERT(!ioat->is_completion_pending, ("bogus completion_pending"));

ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);		ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);		ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
ioat_write_chainaddr(ioat, RING_PHYS_ADDR(ioat, 0));		ioat_write_chainaddr(ioat, RING_PHYS_ADDR(ioat, 0));
error = 0;		error = 0;
CTR2(KTR_IOAT, "%s channel=%u configured channel", __func__,		CTR2(KTR_IOAT, "%s channel=%u configured channel", __func__,
ioat->chan_idx);		ioat->chan_idx);

out:		out:
/* Enqueues a null operation and ensures it completes. */		/* Enqueues a null operation and ensures it completes. */
if (error == 0) {		if (error == 0) {
error = ioat_start_channel(ioat);		error = ioat_start_channel(ioat);
CTR2(KTR_IOAT, "%s channel=%u started channel", __func__,		CTR2(KTR_IOAT, "%s channel=%u started channel", __func__,
ioat->chan_idx);		ioat->chan_idx);
}		}

/*		/*
* Resume completions now that ring state is consistent.		* Resume completions now that ring state is consistent.
*/		*/
mtx_lock(&ioat->cleanup_lock);		mtx_lock(&ioat->cleanup_lock);
ioat->resetting_cleanup = FALSE;		ioat->resetting_cleanup = FALSE;
mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);

/* Unblock submission of new work */		/* Unblock submission of new work */
mtx_lock(IOAT_REFLK);		mtx_lock(&ioat->submit_lock);
ioat->quiescing = FALSE;		ioat->quiescing = FALSE;
wakeup(&ioat->quiescing);		wakeup(&ioat->quiescing);

ioat->resetting = FALSE;		ioat->resetting = FALSE;
wakeup(&ioat->resetting);		wakeup(&ioat->resetting);

if (ioat->is_completion_pending)
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
ioat);
CTR2(KTR_IOAT, "%s channel=%u reset done", __func__, ioat->chan_idx);		CTR2(KTR_IOAT, "%s channel=%u reset done", __func__, ioat->chan_idx);
mtx_unlock(IOAT_REFLK);		mtx_unlock(&ioat->submit_lock);

return (error);		return (error);
}		}

static int		static int
sysctl_handle_chansts(SYSCTL_HANDLER_ARGS)		sysctl_handle_chansts(SYSCTL_HANDLER_ARGS)
{		{
struct ioat_softc *ioat;		struct ioat_softc *ioat;
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	ioat_setup_sysctl(device_t device)
state = SYSCTL_CHILDREN(tmp);		state = SYSCTL_CHILDREN(tmp);

SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "ring_size_order", CTLFLAG_RD,		SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "ring_size_order", CTLFLAG_RD,
&ioat->ring_size_order, 0, "SW descriptor ring size order");		&ioat->ring_size_order, 0, "SW descriptor ring size order");
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "head", CTLFLAG_RD, &ioat->head,		SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "head", CTLFLAG_RD, &ioat->head,
0, "SW descriptor head pointer index");		0, "SW descriptor head pointer index");
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail,		SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail,
0, "SW descriptor tail pointer index");		0, "SW descriptor tail pointer index");
SYSCTL_ADD_UINT(ctx, state, OID_AUTO, "hw_head", CTLFLAG_RD,
&ioat->hw_head, 0, "HW DMACOUNT");

SYSCTL_ADD_UQUAD(ctx, state, OID_AUTO, "last_completion", CTLFLAG_RD,		SYSCTL_ADD_UQUAD(ctx, state, OID_AUTO, "last_completion", CTLFLAG_RD,
ioat->comp_update, "HW addr of last completion");		ioat->comp_update, "HW addr of last completion");

SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_submitter_processing",		SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_submitter_processing",
CTLFLAG_RD, &ioat->is_submitter_processing, 0,		CTLFLAG_RD, &ioat->is_submitter_processing, 0,
"submitter processing");		"submitter processing");
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_completion_pending",
CTLFLAG_RD, &ioat->is_completion_pending, 0, "completion pending");
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_reset_pending", CTLFLAG_RD,
&ioat->is_reset_pending, 0, "reset pending");
SYSCTL_ADD_INT(ctx, state, OID_AUTO, "is_channel_running", CTLFLAG_RD,
&ioat->is_channel_running, 0, "channel running");

SYSCTL_ADD_PROC(ctx, state, OID_AUTO, "chansts",		SYSCTL_ADD_PROC(ctx, state, OID_AUTO, "chansts",
CTLTYPE_STRING \| CTLFLAG_RD, ioat, 0, sysctl_handle_chansts, "A",		CTLTYPE_STRING \| CTLFLAG_RD, ioat, 0, sysctl_handle_chansts, "A",
"String of the channel status");		"String of the channel status");

SYSCTL_ADD_U16(ctx, state, OID_AUTO, "intrdelay", CTLFLAG_RD,		SYSCTL_ADD_U16(ctx, state, OID_AUTO, "intrdelay", CTLFLAG_RD,
&ioat->cached_intrdelay, 0,		&ioat->cached_intrdelay, 0,
"Current INTRDELAY on this channel (cached, microseconds)");		"Current INTRDELAY on this channel (cached, microseconds)");
Show All 29 Lines	SYSCTL_ADD_U32(ctx, statpar, OID_AUTO, "last_halt_chanerr", CTLFLAG_RW,
&ioat->stats.last_halt_chanerr, 0,		&ioat->stats.last_halt_chanerr, 0,
"The raw CHANERR when the channel was last halted");		"The raw CHANERR when the channel was last halted");

SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "desc_per_interrupt",		SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "desc_per_interrupt",
CTLTYPE_STRING \| CTLFLAG_RD, ioat, 0, sysctl_handle_dpi, "A",		CTLTYPE_STRING \| CTLFLAG_RD, ioat, 0, sysctl_handle_dpi, "A",
"Descriptors per interrupt");		"Descriptors per interrupt");
}		}

static inline struct ioat_softc *		static void
ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind)		ioat_get(struct ioat_softc *ioat)
{		{
uint32_t old;

KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));		mtx_assert(&ioat->submit_lock, MA_OWNED);
		KASSERT(ioat->refcnt < UINT32_MAX, ("refcnt overflow"));

old = atomic_fetchadd_32(&ioat->refcnt, 1);		ioat->refcnt++;
KASSERT(old < UINT32_MAX, ("refcnt overflow"));

#ifdef INVARIANTS
old = atomic_fetchadd_32(&ioat->refkinds[kind], 1);
KASSERT(old < UINT32_MAX, ("refcnt kind overflow"));
#endif

return (ioat);
}		}

static inline void		static void
ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)		ioat_put(struct ioat_softc *ioat)
{		{

_ioat_putn(ioat, n, kind, FALSE);		mtx_assert(&ioat->submit_lock, MA_OWNED);
}		KASSERT(ioat->refcnt >= 1, ("refcnt error"));

static inline void		if (--ioat->refcnt == 0)
ioat_putn_locked(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)		wakeup(&ioat->refcnt);
{

_ioat_putn(ioat, n, kind, TRUE);
}		}

static inline void
_ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind,
boolean_t locked)
{
uint32_t old;

KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));

if (n == 0)
return;

#ifdef INVARIANTS
old = atomic_fetchadd_32(&ioat->refkinds[kind], -n);
KASSERT(old >= n, ("refcnt kind underflow"));
#endif

/* Skip acquiring the lock if resulting refcnt > 0. */
for (;;) {
old = ioat->refcnt;
if (old <= n)
break;
if (atomic_cmpset_32(&ioat->refcnt, old, old - n))
return;
}

if (locked)
mtx_assert(IOAT_REFLK, MA_OWNED);
else
mtx_lock(IOAT_REFLK);

old = atomic_fetchadd_32(&ioat->refcnt, -n);
KASSERT(old >= n, ("refcnt error"));

if (old == n)
wakeup(IOAT_REFLK);
if (!locked)
mtx_unlock(IOAT_REFLK);
}

static inline void
ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind)
{

ioat_putn(ioat, 1, kind);
}

static void		static void
ioat_drain_locked(struct ioat_softc *ioat)		ioat_drain_locked(struct ioat_softc *ioat)
{		{

mtx_assert(IOAT_REFLK, MA_OWNED);		mtx_assert(&ioat->submit_lock, MA_OWNED);

while (ioat->refcnt > 0)		while (ioat->refcnt > 0)
msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0);		msleep(&ioat->refcnt, &ioat->submit_lock, 0, "ioat_drain", 0);
}		}

#ifdef DDB		#ifdef DDB
#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)		#define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)
#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)		#define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)
DB_SHOW_COMMAND(ioat, db_show_ioat)		DB_SHOW_COMMAND(ioat, db_show_ioat)
{		{
struct ioat_softc *sc;		struct ioat_softc *sc;
Show All 26 Lines	DB_SHOW_COMMAND(ioat, db_show_ioat)
db_printf(" c_func: %p\n", sc->poll_timer.c_func);		db_printf(" c_func: %p\n", sc->poll_timer.c_func);
db_printf(" c_lock: %p\n", sc->poll_timer.c_lock);		db_printf(" c_lock: %p\n", sc->poll_timer.c_lock);
db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags);		db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags);

db_printf(" quiescing: %d\n", (int)sc->quiescing);		db_printf(" quiescing: %d\n", (int)sc->quiescing);
db_printf(" destroying: %d\n", (int)sc->destroying);		db_printf(" destroying: %d\n", (int)sc->destroying);
db_printf(" is_submitter_processing: %d\n",		db_printf(" is_submitter_processing: %d\n",
(int)sc->is_submitter_processing);		(int)sc->is_submitter_processing);
db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending);
db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending);
db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running);
db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported);		db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported);
db_printf(" resetting: %d\n", (int)sc->resetting);		db_printf(" resetting: %d\n", (int)sc->resetting);

db_printf(" head: %u\n", sc->head);		db_printf(" head: %u\n", sc->head);
db_printf(" tail: %u\n", sc->tail);		db_printf(" tail: %u\n", sc->tail);
db_printf(" hw_head: %u\n", sc->hw_head);
db_printf(" ring_size_order: %u\n", sc->ring_size_order);		db_printf(" ring_size_order: %u\n", sc->ring_size_order);
db_printf(" last_seen: 0x%lx\n", sc->last_seen);		db_printf(" last_seen: 0x%lx\n", sc->last_seen);
db_printf(" ring: %p\n", sc->ring);		db_printf(" ring: %p\n", sc->ring);
db_printf(" descriptors: %p\n", sc->hw_desc_ring);		db_printf(" descriptors: %p\n", sc->hw_desc_ring);
db_printf(" descriptors (phys): 0x%jx\n",		db_printf(" descriptors (phys): 0x%jx\n",
(uintmax_t)sc->hw_desc_bus_addr);		(uintmax_t)sc->hw_desc_bus_addr);

db_printf(" ring[%u] (tail):\n", sc->tail %		db_printf(" ring[%u] (tail):\n", sc->tail %
Show All 24 Lines	for (idx = 0; idx < (1 << sc->ring_size_order); idx++)
if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK)		if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK)
== RING_PHYS_ADDR(sc, idx))		== RING_PHYS_ADDR(sc, idx))
db_printf(" ring[%u] == hardware tail\n", idx);		db_printf(" ring[%u] == hardware tail\n", idx);

db_printf(" cleanup_lock: ");		db_printf(" cleanup_lock: ");
db_show_lock(&sc->cleanup_lock);		db_show_lock(&sc->cleanup_lock);

db_printf(" refcnt: %u\n", sc->refcnt);		db_printf(" refcnt: %u\n", sc->refcnt);
#ifdef INVARIANTS
CTASSERT(IOAT_NUM_REF_KINDS == 2);
db_printf(" refkinds: [ENG=%u, DESCR=%u]\n", sc->refkinds[0],
sc->refkinds[1]);
#endif
db_printf(" stats:\n");		db_printf(" stats:\n");
db_printf(" interrupts: %lu\n", sc->stats.interrupts);		db_printf(" interrupts: %lu\n", sc->stats.interrupts);
db_printf(" descriptors_processed: %lu\n", sc->stats.descriptors_processed);		db_printf(" descriptors_processed: %lu\n", sc->stats.descriptors_processed);
db_printf(" descriptors_error: %lu\n", sc->stats.descriptors_error);		db_printf(" descriptors_error: %lu\n", sc->stats.descriptors_error);
db_printf(" descriptors_submitted: %lu\n", sc->stats.descriptors_submitted);		db_printf(" descriptors_submitted: %lu\n", sc->stats.descriptors_submitted);

db_printf(" channel_halts: %u\n", sc->stats.channel_halts);		db_printf(" channel_halts: %u\n", sc->stats.channel_halts);
db_printf(" last_halt_chanerr: %u\n", sc->stats.last_halt_chanerr);		db_printf(" last_halt_chanerr: %u\n", sc->stats.last_halt_chanerr);
Show All 24 Lines