Differential D25574 Diff 74274 sys/x86/iommu/intel_qi.c

Changeset View

Standalone View

sys/x86/iommu/intel_qi.c

Show First 20 Lines • Show All 57 Lines • ▼ Show 20 Lines
#include <x86/include/busdma_impl.h>		#include <x86/include/busdma_impl.h>
#include <x86/iommu/intel_reg.h>		#include <x86/iommu/intel_reg.h>
#include <x86/iommu/busdma_dmar.h>		#include <x86/iommu/busdma_dmar.h>
#include <dev/pci/pcireg.h>		#include <dev/pci/pcireg.h>
#include <x86/iommu/intel_dmar.h>		#include <x86/iommu/intel_dmar.h>

static bool		static bool
dmar_qi_seq_processed(const struct dmar_unit *unit,		dmar_qi_seq_processed(const struct dmar_unit *unit,
const struct dmar_qi_genseq *pseq)		const struct iommu_qi_genseq *pseq)
{		{

return (pseq->gen < unit->inv_waitd_gen \|\|		return (pseq->gen < unit->inv_waitd_gen \|\|
(pseq->gen == unit->inv_waitd_gen &&		(pseq->gen == unit->inv_waitd_gen &&
pseq->seq <= unit->inv_waitd_seq_hw));		pseq->seq <= unit->inv_waitd_seq_hw));
}		}

static int		static int
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines	dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID \|
(intr ? DMAR_IQ_DESCR_WAIT_IF : 0) \|		(intr ? DMAR_IQ_DESCR_WAIT_IF : 0) \|
(memw ? DMAR_IQ_DESCR_WAIT_SW : 0) \|		(memw ? DMAR_IQ_DESCR_WAIT_SW : 0) \|
(fence ? DMAR_IQ_DESCR_WAIT_FN : 0) \|		(fence ? DMAR_IQ_DESCR_WAIT_FN : 0) \|
(memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),		(memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
memw ? unit->inv_waitd_seq_hw_phys : 0);		memw ? unit->inv_waitd_seq_hw_phys : 0);
}		}

static void		static void
dmar_qi_emit_wait_seq(struct dmar_unit unit, struct dmar_qi_genseq pseq,		dmar_qi_emit_wait_seq(struct dmar_unit unit, struct iommu_qi_genseq pseq,
bool emit_wait)		bool emit_wait)
{		{
struct dmar_qi_genseq gsec;		struct iommu_qi_genseq gsec;
uint32_t seq;		uint32_t seq;

KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));		KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
if (unit->inv_waitd_seq == 0xffffffff) {		if (unit->inv_waitd_seq == 0xffffffff) {
gsec.gen = unit->inv_waitd_gen;		gsec.gen = unit->inv_waitd_gen;
gsec.seq = unit->inv_waitd_seq;		gsec.seq = unit->inv_waitd_seq;
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
Show All 9 Lines	dmar_qi_emit_wait_seq(struct dmar_unit unit, struct iommu_qi_genseq pseq,
pseq->seq = seq;		pseq->seq = seq;
if (emit_wait) {		if (emit_wait) {
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_descr(unit, seq, true, true, false);		dmar_qi_emit_wait_descr(unit, seq, true, true, false);
}		}
}		}

static void		static void
dmar_qi_wait_for_seq(struct dmar_unit unit, const struct dmar_qi_genseq gseq,		dmar_qi_wait_for_seq(struct dmar_unit unit, const struct iommu_qi_genseq gseq,
bool nowait)		bool nowait)
{		{

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
unit->inv_seq_waiters++;		unit->inv_seq_waiters++;
while (!dmar_qi_seq_processed(unit, gseq)) {		while (!dmar_qi_seq_processed(unit, gseq)) {
if (cold \|\| nowait) {		if (cold \|\| nowait) {
cpu_spinwait();		cpu_spinwait();
} else {		} else {
msleep(&unit->inv_seq_waiters, &unit->lock, 0,		msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
"dmarse", hz);		"dmarse", hz);
}		}
}		}
unit->inv_seq_waiters--;		unit->inv_seq_waiters--;
}		}

void		void
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,		dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)		iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;
dmar_gaddr_t isize;		iommu_gaddr_t isize;
int am;		int am;

unit = domain->dmar;		unit = domain->dmar;
DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
for (; size > 0; base += isize, size -= isize) {		for (; size > 0; base += isize, size -= isize) {
am = calc_am(unit, base, size, &isize);		am = calc_am(unit, base, size, &isize);
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \|		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \|
DMAR_IQ_DESCR_IOTLB_PAGE \| DMAR_IQ_DESCR_IOTLB_DW \|		DMAR_IQ_DESCR_IOTLB_PAGE \| DMAR_IQ_DESCR_IOTLB_DW \|
DMAR_IQ_DESCR_IOTLB_DR \|		DMAR_IQ_DESCR_IOTLB_DR \|
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),		DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
base \| am);		base \| am);
}		}
dmar_qi_emit_wait_seq(unit, pseq, emit_wait);		dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
}		}

void		void
dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)		dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
{		{
struct dmar_qi_genseq gseq;		struct iommu_qi_genseq gseq;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);		dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV \| DMAR_IQ_DESCR_CTX_GLOB, 0);		dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV \| DMAR_IQ_DESCR_CTX_GLOB, 0);
dmar_qi_emit_wait_seq(unit, &gseq, true);		dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);		dmar_qi_wait_for_seq(unit, &gseq, false);
}		}

void		void
dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)		dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
{		{
struct dmar_qi_genseq gseq;		struct iommu_qi_genseq gseq;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);		dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \| DMAR_IQ_DESCR_IOTLB_GLOB \|		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \| DMAR_IQ_DESCR_IOTLB_GLOB \|
DMAR_IQ_DESCR_IOTLB_DW \| DMAR_IQ_DESCR_IOTLB_DR, 0);		DMAR_IQ_DESCR_IOTLB_DW \| DMAR_IQ_DESCR_IOTLB_DR, 0);
dmar_qi_emit_wait_seq(unit, &gseq, true);		dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);		dmar_qi_wait_for_seq(unit, &gseq, false);
}		}

void		void
dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)		dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
{		{
struct dmar_qi_genseq gseq;		struct iommu_qi_genseq gseq;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);		dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);		dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
dmar_qi_emit_wait_seq(unit, &gseq, true);		dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);		dmar_qi_wait_for_seq(unit, &gseq, false);
}		}

void		void
dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)		dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
{		{
struct dmar_qi_genseq gseq;		struct iommu_qi_genseq gseq;
u_int c, l;		u_int c, l;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
KASSERT(start < unit->irte_cnt && start < start + cnt &&		KASSERT(start < unit->irte_cnt && start < start + cnt &&
start + cnt <= unit->irte_cnt,		start + cnt <= unit->irte_cnt,
("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));		("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
for (; cnt > 0; cnt -= c, start += c) {		for (; cnt > 0; cnt -= c, start += c) {
l = ffs(start \| cnt) - 1;		l = ffs(start \| cnt) - 1;
Show All 26 Lines
}		}

int		int
dmar_qi_intr(void *arg)		dmar_qi_intr(void *arg)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;

unit = arg;		unit = arg;
KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));		KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
		unit->iommu.unit));
taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);		taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
return (FILTER_HANDLED);		return (FILTER_HANDLED);
}		}

static void		static void
dmar_qi_task(void *arg, int pending __unused)		dmar_qi_task(void *arg, int pending __unused)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;
struct dmar_map_entry *entry;		struct iommu_map_entry *entry;
uint32_t ics;		uint32_t ics;

unit = arg;		unit = arg;

DMAR_LOCK(unit);		DMAR_LOCK(unit);
for (;;) {		for (;;) {
entry = TAILQ_FIRST(&unit->tlb_flush_entries);		entry = TAILQ_FIRST(&unit->tlb_flush_entries);
if (entry == NULL)		if (entry == NULL)
break;		break;
if (!dmar_qi_seq_processed(unit, &entry->gseq))		if (!dmar_qi_seq_processed(unit, &entry->gseq))
break;		break;
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);		TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
DMAR_UNLOCK(unit);		DMAR_UNLOCK(unit);
dmar_domain_free_entry(entry, (entry->flags &		dmar_domain_free_entry(entry, (entry->flags &
DMAR_MAP_ENTRY_QI_NF) == 0);		IOMMU_MAP_ENTRY_QI_NF) == 0);
DMAR_LOCK(unit);		DMAR_LOCK(unit);
}		}
ics = dmar_read4(unit, DMAR_ICS_REG);		ics = dmar_read4(unit, DMAR_ICS_REG);
if ((ics & DMAR_ICS_IWC) != 0) {		if ((ics & DMAR_ICS_IWC) != 0) {
ics = DMAR_ICS_IWC;		ics = DMAR_ICS_IWC;
dmar_write4(unit, DMAR_ICS_REG, ics);		dmar_write4(unit, DMAR_ICS_REG, ics);
}		}
if (unit->inv_seq_waiters > 0)		if (unit->inv_seq_waiters > 0)
Show All 15 Lines	dmar_init_qi(struct dmar_unit *unit)
if (!unit->qi_enabled)		if (!unit->qi_enabled)
return (0);		return (0);

TAILQ_INIT(&unit->tlb_flush_entries);		TAILQ_INIT(&unit->tlb_flush_entries);
TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);		TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,		unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
taskqueue_thread_enqueue, &unit->qi_taskqueue);		taskqueue_thread_enqueue, &unit->qi_taskqueue);
taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,		taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
"dmar%d qi taskq", unit->unit);		"dmar%d qi taskq", unit->iommu.unit);

unit->inv_waitd_gen = 0;		unit->inv_waitd_gen = 0;
unit->inv_waitd_seq = 1;		unit->inv_waitd_seq = 1;

qi_sz = DMAR_IQA_QS_DEF;		qi_sz = DMAR_IQA_QS_DEF;
TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);		TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
if (qi_sz > DMAR_IQA_QS_MAX)		if (qi_sz > DMAR_IQA_QS_MAX)
qi_sz = DMAR_IQA_QS_MAX;		qi_sz = DMAR_IQA_QS_MAX;
Show All 22 Lines	dmar_init_qi(struct dmar_unit *unit)
DMAR_UNLOCK(unit);		DMAR_UNLOCK(unit);

return (0);		return (0);
}		}

void		void
dmar_fini_qi(struct dmar_unit *unit)		dmar_fini_qi(struct dmar_unit *unit)
{		{
struct dmar_qi_genseq gseq;		struct iommu_qi_genseq gseq;

if (!unit->qi_enabled)		if (!unit->qi_enabled)
return;		return;
taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);		taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
taskqueue_free(unit->qi_taskqueue);		taskqueue_free(unit->qi_taskqueue);
unit->qi_taskqueue = NULL;		unit->qi_taskqueue = NULL;

DMAR_LOCK(unit);		DMAR_LOCK(unit);
/* quisce */		/* quisce */
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, &gseq, true);		dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);		dmar_qi_wait_for_seq(unit, &gseq, false);
/* only after the quisce, disable queue */		/* only after the quisce, disable queue */
dmar_disable_qi_intr(unit);		dmar_disable_qi_intr(unit);
dmar_disable_qi(unit);		dmar_disable_qi(unit);
KASSERT(unit->inv_seq_waiters == 0,		KASSERT(unit->inv_seq_waiters == 0,
("dmar%d: waiters on disabled queue", unit->unit));		("dmar%d: waiters on disabled queue", unit->iommu.unit));
DMAR_UNLOCK(unit);		DMAR_UNLOCK(unit);

kmem_free(unit->inv_queue, unit->inv_queue_size);		kmem_free(unit->inv_queue, unit->inv_queue_size);
unit->inv_queue = 0;		unit->inv_queue = 0;
unit->inv_queue_size = 0;		unit->inv_queue_size = 0;
unit->qi_enabled = 0;		unit->qi_enabled = 0;
}		}

void		void
dmar_enable_qi_intr(struct dmar_unit *unit)		dmar_enable_qi_intr(struct dmar_unit *unit)
{		{
uint32_t iectl;		uint32_t iectl;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));		KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
		unit->iommu.unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);		iectl = dmar_read4(unit, DMAR_IECTL_REG);
iectl &= ~DMAR_IECTL_IM;		iectl &= ~DMAR_IECTL_IM;
dmar_write4(unit, DMAR_IECTL_REG, iectl);		dmar_write4(unit, DMAR_IECTL_REG, iectl);
}		}

void		void
dmar_disable_qi_intr(struct dmar_unit *unit)		dmar_disable_qi_intr(struct dmar_unit *unit)
{		{
uint32_t iectl;		uint32_t iectl;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));		KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
		unit->iommu.unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);		iectl = dmar_read4(unit, DMAR_IECTL_REG);
dmar_write4(unit, DMAR_IECTL_REG, iectl \| DMAR_IECTL_IM);		dmar_write4(unit, DMAR_IECTL_REG, iectl \| DMAR_IECTL_IM);
}		}