Differential D25574 Diff 74192 sys/x86/iommu/intel_qi.c

Changeset View

Standalone View

sys/x86/iommu/intel_qi.c

Show First 20 Lines • Show All 207 Lines • ▼ Show 20 Lines
{		{

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
unit->inv_seq_waiters++;		unit->inv_seq_waiters++;
while (!dmar_qi_seq_processed(unit, gseq)) {		while (!dmar_qi_seq_processed(unit, gseq)) {
if (cold \|\| nowait) {		if (cold \|\| nowait) {
cpu_spinwait();		cpu_spinwait();
} else {		} else {
msleep(&unit->inv_seq_waiters, &unit->lock, 0,		msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
"dmarse", hz);		"dmarse", hz);
}		}
}		}
unit->inv_seq_waiters--;		unit->inv_seq_waiters--;
}		}

void		void
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,		dmar_qi_invalidate_locked(struct iommu_domain *domain, dmar_gaddr_t base,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)		dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;
dmar_gaddr_t isize;		dmar_gaddr_t isize;
int am;		int am;

unit = domain->dmar;		unit = (struct dmar_unit *)domain->iommu;
DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
for (; size > 0; base += isize, size -= isize) {		for (; size > 0; base += isize, size -= isize) {
am = calc_am(unit, base, size, &isize);		am = calc_am(unit, base, size, &isize);
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \|		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \|
DMAR_IQ_DESCR_IOTLB_PAGE \| DMAR_IQ_DESCR_IOTLB_DW \|		DMAR_IQ_DESCR_IOTLB_PAGE \| DMAR_IQ_DESCR_IOTLB_DW \|
DMAR_IQ_DESCR_IOTLB_DR \|		DMAR_IQ_DESCR_IOTLB_DR \|
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),		DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines
}		}

int		int
dmar_qi_intr(void *arg)		dmar_qi_intr(void *arg)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;

unit = arg;		unit = arg;
KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));		KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
		unit->iommu.unit));
taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);		taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
return (FILTER_HANDLED);		return (FILTER_HANDLED);
}		}

static void		static void
dmar_qi_task(void *arg, int pending __unused)		dmar_qi_task(void *arg, int pending __unused)
{		{
struct dmar_unit *unit;		struct dmar_unit *unit;
struct dmar_map_entry *entry;		struct iommu_map_entry *entry;
uint32_t ics;		uint32_t ics;

unit = arg;		unit = arg;

DMAR_LOCK(unit);		DMAR_LOCK(unit);
for (;;) {		for (;;) {
entry = TAILQ_FIRST(&unit->tlb_flush_entries);		entry = TAILQ_FIRST(&unit->tlb_flush_entries);
if (entry == NULL)		if (entry == NULL)
break;		break;
if (!dmar_qi_seq_processed(unit, &entry->gseq))		if (!dmar_qi_seq_processed(unit, &entry->gseq))
break;		break;
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);		TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
DMAR_UNLOCK(unit);		DMAR_UNLOCK(unit);
dmar_domain_free_entry(entry, (entry->flags &		iommu_domain_free_entry(entry, (entry->flags &
DMAR_MAP_ENTRY_QI_NF) == 0);		IOMMU_MAP_ENTRY_QI_NF) == 0);
DMAR_LOCK(unit);		DMAR_LOCK(unit);
}		}
ics = dmar_read4(unit, DMAR_ICS_REG);		ics = dmar_read4(unit, DMAR_ICS_REG);
if ((ics & DMAR_ICS_IWC) != 0) {		if ((ics & DMAR_ICS_IWC) != 0) {
ics = DMAR_ICS_IWC;		ics = DMAR_ICS_IWC;
dmar_write4(unit, DMAR_ICS_REG, ics);		dmar_write4(unit, DMAR_ICS_REG, ics);
}		}
if (unit->inv_seq_waiters > 0)		if (unit->inv_seq_waiters > 0)
Show All 15 Lines	dmar_init_qi(struct dmar_unit *unit)
if (!unit->qi_enabled)		if (!unit->qi_enabled)
return (0);		return (0);

TAILQ_INIT(&unit->tlb_flush_entries);		TAILQ_INIT(&unit->tlb_flush_entries);
TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);		TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,		unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
taskqueue_thread_enqueue, &unit->qi_taskqueue);		taskqueue_thread_enqueue, &unit->qi_taskqueue);
taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,		taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
"dmar%d qi taskq", unit->unit);		"dmar%d qi taskq", unit->iommu.unit);

unit->inv_waitd_gen = 0;		unit->inv_waitd_gen = 0;
unit->inv_waitd_seq = 1;		unit->inv_waitd_seq = 1;

qi_sz = DMAR_IQA_QS_DEF;		qi_sz = DMAR_IQA_QS_DEF;
TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);		TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
if (qi_sz > DMAR_IQA_QS_MAX)		if (qi_sz > DMAR_IQA_QS_MAX)
qi_sz = DMAR_IQA_QS_MAX;		qi_sz = DMAR_IQA_QS_MAX;
Show All 40 Lines	dmar_fini_qi(struct dmar_unit *unit)
dmar_qi_ensure(unit, 1);		dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, &gseq, true);		dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);		dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);		dmar_qi_wait_for_seq(unit, &gseq, false);
/* only after the quisce, disable queue */		/* only after the quisce, disable queue */
dmar_disable_qi_intr(unit);		dmar_disable_qi_intr(unit);
dmar_disable_qi(unit);		dmar_disable_qi(unit);
KASSERT(unit->inv_seq_waiters == 0,		KASSERT(unit->inv_seq_waiters == 0,
("dmar%d: waiters on disabled queue", unit->unit));		("dmar%d: waiters on disabled queue", unit->iommu.unit));
DMAR_UNLOCK(unit);		DMAR_UNLOCK(unit);

kmem_free(unit->inv_queue, unit->inv_queue_size);		kmem_free(unit->inv_queue, unit->inv_queue_size);
unit->inv_queue = 0;		unit->inv_queue = 0;
unit->inv_queue_size = 0;		unit->inv_queue_size = 0;
unit->qi_enabled = 0;		unit->qi_enabled = 0;
}		}

void		void
dmar_enable_qi_intr(struct dmar_unit *unit)		dmar_enable_qi_intr(struct dmar_unit *unit)
{		{
uint32_t iectl;		uint32_t iectl;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));		KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
		unit->iommu.unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);		iectl = dmar_read4(unit, DMAR_IECTL_REG);
iectl &= ~DMAR_IECTL_IM;		iectl &= ~DMAR_IECTL_IM;
dmar_write4(unit, DMAR_IECTL_REG, iectl);		dmar_write4(unit, DMAR_IECTL_REG, iectl);
}		}

void		void
dmar_disable_qi_intr(struct dmar_unit *unit)		dmar_disable_qi_intr(struct dmar_unit *unit)
{		{
uint32_t iectl;		uint32_t iectl;

DMAR_ASSERT_LOCKED(unit);		DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));		KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
		unit->iommu.unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);		iectl = dmar_read4(unit, DMAR_IECTL_REG);
dmar_write4(unit, DMAR_IECTL_REG, iectl \| DMAR_IECTL_IM);		dmar_write4(unit, DMAR_IECTL_REG, iectl \| DMAR_IECTL_IM);
}		}