Differential D7097 Diff 18165 head/sys/dev/ioat/ioat.c

Changeset View

Standalone View

head/sys/dev/ioat/ioat.c

Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines

#include "ioat.h"		#include "ioat.h"
#include "ioat_hw.h"		#include "ioat_hw.h"
#include "ioat_internal.h"		#include "ioat_internal.h"

#ifndef BUS_SPACE_MAXADDR_40BIT		#ifndef BUS_SPACE_MAXADDR_40BIT
#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL		#define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL
#endif		#endif
#define IOAT_INTR_TIMO (hz / 10)
#define IOAT_REFLK (&ioat->submit_lock)		#define IOAT_REFLK (&ioat->submit_lock)
		#define IOAT_SHRINK_PERIOD (10 * hz)

static int ioat_probe(device_t device);		static int ioat_probe(device_t device);
static int ioat_attach(device_t device);		static int ioat_attach(device_t device);
static int ioat_detach(device_t device);		static int ioat_detach(device_t device);
static int ioat_setup_intr(struct ioat_softc *ioat);		static int ioat_setup_intr(struct ioat_softc *ioat);
static int ioat_teardown_intr(struct ioat_softc *ioat);		static int ioat_teardown_intr(struct ioat_softc *ioat);
static int ioat3_attach(device_t device);		static int ioat3_attach(device_t device);
static int ioat_start_channel(struct ioat_softc *ioat);		static int ioat_start_channel(struct ioat_softc *ioat);
Show All 17 Lines	static struct ioat_descriptor ioat_get_ring_entry(struct ioat_softc ioat,
uint32_t index);		uint32_t index);
static struct ioat_descriptor *ioat_prealloc_ring(struct ioat_softc ,		static struct ioat_descriptor *ioat_prealloc_ring(struct ioat_softc ,
uint32_t size, boolean_t need_dscr, int mflags);		uint32_t size, boolean_t need_dscr, int mflags);
static int ring_grow(struct ioat_softc *, uint32_t oldorder,		static int ring_grow(struct ioat_softc *, uint32_t oldorder,
struct ioat_descriptor **);		struct ioat_descriptor **);
static int ring_shrink(struct ioat_softc *, uint32_t oldorder,		static int ring_shrink(struct ioat_softc *, uint32_t oldorder,
struct ioat_descriptor **);		struct ioat_descriptor **);
static void ioat_halted_debug(struct ioat_softc *, uint32_t);		static void ioat_halted_debug(struct ioat_softc *, uint32_t);
static void ioat_timer_callback(void *arg);		static void ioat_poll_timer_callback(void *arg);
		static void ioat_shrink_timer_callback(void *arg);
static void dump_descriptor(void *hw_desc);		static void dump_descriptor(void *hw_desc);
static void ioat_submit_single(struct ioat_softc *ioat);		static void ioat_submit_single(struct ioat_softc *ioat);
static void ioat_comp_update_map(void arg, bus_dma_segment_t seg, int nseg,		static void ioat_comp_update_map(void arg, bus_dma_segment_t seg, int nseg,
int error);		int error);
static int ioat_reset_hw(struct ioat_softc *ioat);		static int ioat_reset_hw(struct ioat_softc *ioat);
static void ioat_reset_hw_task(void *, int);		static void ioat_reset_hw_task(void *, int);
static void ioat_setup_sysctl(device_t device);		static void ioat_setup_sysctl(device_t device);
static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);		static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines	ioat_detach(device_t device)
wakeup(&ioat->quiescing);		wakeup(&ioat->quiescing);

ioat_channel[ioat->chan_idx] = NULL;		ioat_channel[ioat->chan_idx] = NULL;

ioat_drain_locked(ioat);		ioat_drain_locked(ioat);
mtx_unlock(IOAT_REFLK);		mtx_unlock(IOAT_REFLK);

ioat_teardown_intr(ioat);		ioat_teardown_intr(ioat);
callout_drain(&ioat->timer);		callout_drain(&ioat->poll_timer);
		callout_drain(&ioat->shrink_timer);

pci_disable_busmaster(device);		pci_disable_busmaster(device);

if (ioat->pci_resource != NULL)		if (ioat->pci_resource != NULL)
bus_release_resource(device, SYS_RES_MEMORY,		bus_release_resource(device, SYS_RES_MEMORY,
ioat->pci_resource_id, ioat->pci_resource);		ioat->pci_resource_id, ioat->pci_resource);

if (ioat->ring != NULL)		if (ioat->ring != NULL)
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines	ioat->intrdelay_supported = (ioat_read_2(ioat, IOAT_INTRDELAY_OFFSET) &
IOAT_INTRDELAY_SUPPORTED) != 0;		IOAT_INTRDELAY_SUPPORTED) != 0;
if (ioat->intrdelay_supported)		if (ioat->intrdelay_supported)
ioat->intrdelay_max = IOAT_INTRDELAY_US_MASK;		ioat->intrdelay_max = IOAT_INTRDELAY_US_MASK;

/* TODO: need to check DCA here if we ever do XOR/PQ */		/* TODO: need to check DCA here if we ever do XOR/PQ */

mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);		mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF);		mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF);
callout_init(&ioat->timer, 1);		callout_init(&ioat->poll_timer, 1);
		callout_init(&ioat->shrink_timer, 1);
TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat);		TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat);

/* Establish lock order for Witness */		/* Establish lock order for Witness */
mtx_lock(&ioat->submit_lock);		mtx_lock(&ioat->submit_lock);
mtx_lock(&ioat->cleanup_lock);		mtx_lock(&ioat->cleanup_lock);
mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);

▲ Show 20 Lines • Show All 189 Lines • ▼ Show 20 Lines

static void		static void
ioat_process_events(struct ioat_softc *ioat)		ioat_process_events(struct ioat_softc *ioat)
{		{
struct ioat_descriptor *desc;		struct ioat_descriptor *desc;
struct bus_dmadesc *dmadesc;		struct bus_dmadesc *dmadesc;
uint64_t comp_update, status;		uint64_t comp_update, status;
uint32_t completed, chanerr;		uint32_t completed, chanerr;
		boolean_t pending;
int error;		int error;

mtx_lock(&ioat->cleanup_lock);		mtx_lock(&ioat->cleanup_lock);

completed = 0;		completed = 0;
comp_update = *ioat->comp_update;		comp_update = *ioat->comp_update;
status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;		status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;

Show All 18 Lines	while (1) {

completed++;		completed++;
ioat->tail++;		ioat->tail++;
if (desc->hw_desc_bus_addr == status)		if (desc->hw_desc_bus_addr == status)
break;		break;
}		}

ioat->last_seen = desc->hw_desc_bus_addr;		ioat->last_seen = desc->hw_desc_bus_addr;

if (ioat->head == ioat->tail) {
ioat->is_completion_pending = FALSE;
callout_reset(&ioat->timer, IOAT_INTR_TIMO,
ioat_timer_callback, ioat);
}

ioat->stats.descriptors_processed += completed;		ioat->stats.descriptors_processed += completed;

out:		out:
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);		ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);

		/* Perform a racy check first; only take the locks if it passes. */
		pending = (ioat_get_active(ioat) != 0);
		if (!pending && ioat->is_completion_pending) {
mtx_unlock(&ioat->cleanup_lock);		mtx_unlock(&ioat->cleanup_lock);
		mtx_lock(&ioat->submit_lock);
		mtx_lock(&ioat->cleanup_lock);

		pending = (ioat_get_active(ioat) != 0);
		if (!pending && ioat->is_completion_pending) {
		ioat->is_completion_pending = FALSE;
		callout_reset(&ioat->shrink_timer, IOAT_SHRINK_PERIOD,
		ioat_shrink_timer_callback, ioat);
		callout_stop(&ioat->poll_timer);
		}
		mtx_unlock(&ioat->submit_lock);
		}
		mtx_unlock(&ioat->cleanup_lock);

		if (pending)
		callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
		ioat);

if (completed != 0) {		if (completed != 0) {
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);		ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
wakeup(&ioat->tail);		wakeup(&ioat->tail);
}		}

if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update))		if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update))
return;		return;

▲ Show 20 Lines • Show All 905 Lines • ▼ Show 20 Lines	ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr)
desc = ioat_get_ring_entry(ioat, ioat->tail + 0);		desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
dump_descriptor(desc->u.raw);		dump_descriptor(desc->u.raw);

desc = ioat_get_ring_entry(ioat, ioat->tail + 1);		desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
dump_descriptor(desc->u.raw);		dump_descriptor(desc->u.raw);
}		}

static void		static void
ioat_timer_callback(void *arg)		ioat_poll_timer_callback(void *arg)
{		{
struct ioat_descriptor **newring;
struct ioat_softc *ioat;		struct ioat_softc *ioat;
uint32_t order;

ioat = arg;		ioat = arg;
ioat_log_message(1, "%s\n", __func__);		ioat_log_message(3, "%s\n", __func__);

if (ioat->is_completion_pending) {
ioat_process_events(ioat);		ioat_process_events(ioat);
return;
}		}

		static void
		ioat_shrink_timer_callback(void *arg)
		{
		struct ioat_descriptor **newring;
		struct ioat_softc *ioat;
		uint32_t order;

		ioat = arg;
		ioat_log_message(1, "%s\n", __func__);

/* Slowly scale the ring down if idle. */		/* Slowly scale the ring down if idle. */
mtx_lock(&ioat->submit_lock);		mtx_lock(&ioat->submit_lock);
order = ioat->ring_size_order;		order = ioat->ring_size_order;
if (ioat->is_resize_pending \|\| order == IOAT_MIN_ORDER) {		if (ioat->is_resize_pending \|\| order == IOAT_MIN_ORDER) {
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);
goto out;		goto out;
}		}
ioat->is_resize_pending = TRUE;		ioat->is_resize_pending = TRUE;
Show All 9 Lines	ioat_shrink_timer_callback(void *arg)
if (newring != NULL)		if (newring != NULL)
ring_shrink(ioat, order, newring);		ring_shrink(ioat, order, newring);

ioat->is_resize_pending = FALSE;		ioat->is_resize_pending = FALSE;
mtx_unlock(&ioat->submit_lock);		mtx_unlock(&ioat->submit_lock);

out:		out:
if (ioat->ring_size_order > IOAT_MIN_ORDER)		if (ioat->ring_size_order > IOAT_MIN_ORDER)
callout_reset(&ioat->timer, 10 * hz,		callout_reset(&ioat->poll_timer, IOAT_SHRINK_PERIOD,
ioat_timer_callback, ioat);		ioat_shrink_timer_callback, ioat);
}		}

/*		/*
* Support Functions		* Support Functions
*/		*/
static void		static void
ioat_submit_single(struct ioat_softc *ioat)		ioat_submit_single(struct ioat_softc *ioat)
{		{

ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);		ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
atomic_add_rel_int(&ioat->head, 1);		atomic_add_rel_int(&ioat->head, 1);
atomic_add_rel_int(&ioat->hw_head, 1);		atomic_add_rel_int(&ioat->hw_head, 1);

if (!ioat->is_completion_pending) {		if (!ioat->is_completion_pending) {
ioat->is_completion_pending = TRUE;		ioat->is_completion_pending = TRUE;
callout_reset(&ioat->timer, IOAT_INTR_TIMO,		callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
ioat_timer_callback, ioat);		ioat);
		callout_stop(&ioat->shrink_timer);
}		}

ioat->stats.descriptors_submitted++;		ioat->stats.descriptors_submitted++;
}		}

static int		static int
ioat_reset_hw(struct ioat_softc *ioat)		ioat_reset_hw(struct ioat_softc *ioat)
{		{
▲ Show 20 Lines • Show All 450 Lines • ▼ Show 20 Lines	DB_SHOW_COMMAND(ioat, db_show_ioat)
db_printf(" submit_lock: ");		db_printf(" submit_lock: ");
db_show_lock(&sc->submit_lock);		db_show_lock(&sc->submit_lock);

db_printf(" capabilities: %b\n", (int)sc->capabilities,		db_printf(" capabilities: %b\n", (int)sc->capabilities,
IOAT_DMACAP_STR);		IOAT_DMACAP_STR);
db_printf(" cached_intrdelay: %u\n", sc->cached_intrdelay);		db_printf(" cached_intrdelay: %u\n", sc->cached_intrdelay);
db_printf(" comp_update: 0x%jx\n", (uintmax_t)sc->comp_update);		db_printf(" comp_update: 0x%jx\n", (uintmax_t)sc->comp_update);

db_printf(" timer:\n");		db_printf(" poll_timer:\n");
db_printf(" c_time: %ju\n", (uintmax_t)sc->timer.c_time);		db_printf(" c_time: %ju\n", (uintmax_t)sc->poll_timer.c_time);
db_printf(" c_arg: %p\n", sc->timer.c_arg);		db_printf(" c_arg: %p\n", sc->poll_timer.c_arg);
db_printf(" c_func: %p\n", sc->timer.c_func);		db_printf(" c_func: %p\n", sc->poll_timer.c_func);
db_printf(" c_lock: %p\n", sc->timer.c_lock);		db_printf(" c_lock: %p\n", sc->poll_timer.c_lock);
db_printf(" c_flags: 0x%x\n", (unsigned)sc->timer.c_flags);		db_printf(" c_flags: 0x%x\n", (unsigned)sc->poll_timer.c_flags);

		db_printf(" shrink_timer:\n");
		db_printf(" c_time: %ju\n", (uintmax_t)sc->shrink_timer.c_time);
		db_printf(" c_arg: %p\n", sc->shrink_timer.c_arg);
		db_printf(" c_func: %p\n", sc->shrink_timer.c_func);
		db_printf(" c_lock: %p\n", sc->shrink_timer.c_lock);
		db_printf(" c_flags: 0x%x\n", (unsigned)sc->shrink_timer.c_flags);

db_printf(" quiescing: %d\n", (int)sc->quiescing);		db_printf(" quiescing: %d\n", (int)sc->quiescing);
db_printf(" destroying: %d\n", (int)sc->destroying);		db_printf(" destroying: %d\n", (int)sc->destroying);
db_printf(" is_resize_pending: %d\n", (int)sc->is_resize_pending);		db_printf(" is_resize_pending: %d\n", (int)sc->is_resize_pending);
db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending);		db_printf(" is_completion_pending: %d\n", (int)sc->is_completion_pending);
db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending);		db_printf(" is_reset_pending: %d\n", (int)sc->is_reset_pending);
db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running);		db_printf(" is_channel_running: %d\n", (int)sc->is_channel_running);
db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported);		db_printf(" intrdelay_supported: %d\n", (int)sc->intrdelay_supported);
▲ Show 20 Lines • Show All 49 Lines • Show Last 20 Lines