Changeset View
Changeset View
Standalone View
Standalone View
head/sys/x86/iommu/intel_qi.c
Show First 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | |||||
#include <x86/include/busdma_impl.h> | #include <x86/include/busdma_impl.h> | ||||
#include <x86/iommu/intel_reg.h> | #include <x86/iommu/intel_reg.h> | ||||
#include <x86/iommu/busdma_dmar.h> | #include <x86/iommu/busdma_dmar.h> | ||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <x86/iommu/intel_dmar.h> | #include <x86/iommu/intel_dmar.h> | ||||
static bool | static bool | ||||
dmar_qi_seq_processed(const struct dmar_unit *unit, | dmar_qi_seq_processed(const struct dmar_unit *unit, | ||||
const struct dmar_qi_genseq *pseq) | const struct iommu_qi_genseq *pseq) | ||||
{ | { | ||||
return (pseq->gen < unit->inv_waitd_gen || | return (pseq->gen < unit->inv_waitd_gen || | ||||
(pseq->gen == unit->inv_waitd_gen && | (pseq->gen == unit->inv_waitd_gen && | ||||
pseq->seq <= unit->inv_waitd_seq_hw)); | pseq->seq <= unit->inv_waitd_seq_hw)); | ||||
} | } | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | | ||||
(intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | | (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | | ||||
(memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | | (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | | ||||
(fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | | (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | | ||||
(memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), | (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), | ||||
memw ? unit->inv_waitd_seq_hw_phys : 0); | memw ? unit->inv_waitd_seq_hw_phys : 0); | ||||
} | } | ||||
static void | static void | ||||
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq, | dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq, | ||||
bool emit_wait) | bool emit_wait) | ||||
{ | { | ||||
struct dmar_qi_genseq gsec; | struct iommu_qi_genseq gsec; | ||||
uint32_t seq; | uint32_t seq; | ||||
KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); | KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
if (unit->inv_waitd_seq == 0xffffffff) { | if (unit->inv_waitd_seq == 0xffffffff) { | ||||
gsec.gen = unit->inv_waitd_gen; | gsec.gen = unit->inv_waitd_gen; | ||||
gsec.seq = unit->inv_waitd_seq; | gsec.seq = unit->inv_waitd_seq; | ||||
dmar_qi_ensure(unit, 1); | dmar_qi_ensure(unit, 1); | ||||
Show All 9 Lines | dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq, | ||||
pseq->seq = seq; | pseq->seq = seq; | ||||
if (emit_wait) { | if (emit_wait) { | ||||
dmar_qi_ensure(unit, 1); | dmar_qi_ensure(unit, 1); | ||||
dmar_qi_emit_wait_descr(unit, seq, true, true, false); | dmar_qi_emit_wait_descr(unit, seq, true, true, false); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, | dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq, | ||||
bool nowait) | bool nowait) | ||||
{ | { | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
unit->inv_seq_waiters++; | unit->inv_seq_waiters++; | ||||
while (!dmar_qi_seq_processed(unit, gseq)) { | while (!dmar_qi_seq_processed(unit, gseq)) { | ||||
if (cold || nowait) { | if (cold || nowait) { | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
} else { | } else { | ||||
msleep(&unit->inv_seq_waiters, &unit->lock, 0, | msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0, | ||||
"dmarse", hz); | "dmarse", hz); | ||||
} | } | ||||
} | } | ||||
unit->inv_seq_waiters--; | unit->inv_seq_waiters--; | ||||
} | } | ||||
void | void | ||||
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, | dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait) | iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
dmar_gaddr_t isize; | iommu_gaddr_t isize; | ||||
int am; | int am; | ||||
unit = domain->dmar; | unit = domain->dmar; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
for (; size > 0; base += isize, size -= isize) { | for (; size > 0; base += isize, size -= isize) { | ||||
am = calc_am(unit, base, size, &isize); | am = calc_am(unit, base, size, &isize); | ||||
dmar_qi_ensure(unit, 1); | dmar_qi_ensure(unit, 1); | ||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | | dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | | ||||
DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | | DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | | ||||
DMAR_IQ_DESCR_IOTLB_DR | | DMAR_IQ_DESCR_IOTLB_DR | | ||||
DMAR_IQ_DESCR_IOTLB_DID(domain->domain), | DMAR_IQ_DESCR_IOTLB_DID(domain->domain), | ||||
base | am); | base | am); | ||||
} | } | ||||
dmar_qi_emit_wait_seq(unit, pseq, emit_wait); | dmar_qi_emit_wait_seq(unit, pseq, emit_wait); | ||||
dmar_qi_advance_tail(unit); | dmar_qi_advance_tail(unit); | ||||
} | } | ||||
void | void | ||||
dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) | dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) | ||||
{ | { | ||||
struct dmar_qi_genseq gseq; | struct iommu_qi_genseq gseq; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
dmar_qi_ensure(unit, 2); | dmar_qi_ensure(unit, 2); | ||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); | dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); | ||||
dmar_qi_emit_wait_seq(unit, &gseq, true); | dmar_qi_emit_wait_seq(unit, &gseq, true); | ||||
dmar_qi_advance_tail(unit); | dmar_qi_advance_tail(unit); | ||||
dmar_qi_wait_for_seq(unit, &gseq, false); | dmar_qi_wait_for_seq(unit, &gseq, false); | ||||
} | } | ||||
void | void | ||||
dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) | dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) | ||||
{ | { | ||||
struct dmar_qi_genseq gseq; | struct iommu_qi_genseq gseq; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
dmar_qi_ensure(unit, 2); | dmar_qi_ensure(unit, 2); | ||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | | dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | | ||||
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); | DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); | ||||
dmar_qi_emit_wait_seq(unit, &gseq, true); | dmar_qi_emit_wait_seq(unit, &gseq, true); | ||||
dmar_qi_advance_tail(unit); | dmar_qi_advance_tail(unit); | ||||
dmar_qi_wait_for_seq(unit, &gseq, false); | dmar_qi_wait_for_seq(unit, &gseq, false); | ||||
} | } | ||||
void | void | ||||
dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) | dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) | ||||
{ | { | ||||
struct dmar_qi_genseq gseq; | struct iommu_qi_genseq gseq; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
dmar_qi_ensure(unit, 2); | dmar_qi_ensure(unit, 2); | ||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); | dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); | ||||
dmar_qi_emit_wait_seq(unit, &gseq, true); | dmar_qi_emit_wait_seq(unit, &gseq, true); | ||||
dmar_qi_advance_tail(unit); | dmar_qi_advance_tail(unit); | ||||
dmar_qi_wait_for_seq(unit, &gseq, false); | dmar_qi_wait_for_seq(unit, &gseq, false); | ||||
} | } | ||||
void | void | ||||
dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) | dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) | ||||
{ | { | ||||
struct dmar_qi_genseq gseq; | struct iommu_qi_genseq gseq; | ||||
u_int c, l; | u_int c, l; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
KASSERT(start < unit->irte_cnt && start < start + cnt && | KASSERT(start < unit->irte_cnt && start < start + cnt && | ||||
start + cnt <= unit->irte_cnt, | start + cnt <= unit->irte_cnt, | ||||
("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); | ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); | ||||
for (; cnt > 0; cnt -= c, start += c) { | for (; cnt > 0; cnt -= c, start += c) { | ||||
l = ffs(start | cnt) - 1; | l = ffs(start | cnt) - 1; | ||||
Show All 26 Lines | |||||
} | } | ||||
int | int | ||||
dmar_qi_intr(void *arg) | dmar_qi_intr(void *arg) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
unit = arg; | unit = arg; | ||||
KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit)); | KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", | ||||
unit->iommu.unit)); | |||||
taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); | taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
} | } | ||||
static void | static void | ||||
dmar_qi_task(void *arg, int pending __unused) | dmar_qi_task(void *arg, int pending __unused) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
struct dmar_map_entry *entry; | struct iommu_map_entry *entry; | ||||
uint32_t ics; | uint32_t ics; | ||||
unit = arg; | unit = arg; | ||||
DMAR_LOCK(unit); | DMAR_LOCK(unit); | ||||
for (;;) { | for (;;) { | ||||
entry = TAILQ_FIRST(&unit->tlb_flush_entries); | entry = TAILQ_FIRST(&unit->tlb_flush_entries); | ||||
if (entry == NULL) | if (entry == NULL) | ||||
break; | break; | ||||
if (!dmar_qi_seq_processed(unit, &entry->gseq)) | if (!dmar_qi_seq_processed(unit, &entry->gseq)) | ||||
break; | break; | ||||
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); | TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); | ||||
DMAR_UNLOCK(unit); | DMAR_UNLOCK(unit); | ||||
dmar_domain_free_entry(entry, (entry->flags & | dmar_domain_free_entry(entry, (entry->flags & | ||||
DMAR_MAP_ENTRY_QI_NF) == 0); | IOMMU_MAP_ENTRY_QI_NF) == 0); | ||||
DMAR_LOCK(unit); | DMAR_LOCK(unit); | ||||
} | } | ||||
ics = dmar_read4(unit, DMAR_ICS_REG); | ics = dmar_read4(unit, DMAR_ICS_REG); | ||||
if ((ics & DMAR_ICS_IWC) != 0) { | if ((ics & DMAR_ICS_IWC) != 0) { | ||||
ics = DMAR_ICS_IWC; | ics = DMAR_ICS_IWC; | ||||
dmar_write4(unit, DMAR_ICS_REG, ics); | dmar_write4(unit, DMAR_ICS_REG, ics); | ||||
} | } | ||||
if (unit->inv_seq_waiters > 0) | if (unit->inv_seq_waiters > 0) | ||||
Show All 15 Lines | dmar_init_qi(struct dmar_unit *unit) | ||||
if (!unit->qi_enabled) | if (!unit->qi_enabled) | ||||
return (0); | return (0); | ||||
TAILQ_INIT(&unit->tlb_flush_entries); | TAILQ_INIT(&unit->tlb_flush_entries); | ||||
TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); | TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); | ||||
unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, | unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, | ||||
taskqueue_thread_enqueue, &unit->qi_taskqueue); | taskqueue_thread_enqueue, &unit->qi_taskqueue); | ||||
taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, | taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, | ||||
"dmar%d qi taskq", unit->unit); | "dmar%d qi taskq", unit->iommu.unit); | ||||
unit->inv_waitd_gen = 0; | unit->inv_waitd_gen = 0; | ||||
unit->inv_waitd_seq = 1; | unit->inv_waitd_seq = 1; | ||||
qi_sz = DMAR_IQA_QS_DEF; | qi_sz = DMAR_IQA_QS_DEF; | ||||
TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); | TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); | ||||
if (qi_sz > DMAR_IQA_QS_MAX) | if (qi_sz > DMAR_IQA_QS_MAX) | ||||
qi_sz = DMAR_IQA_QS_MAX; | qi_sz = DMAR_IQA_QS_MAX; | ||||
Show All 22 Lines | dmar_init_qi(struct dmar_unit *unit) | ||||
DMAR_UNLOCK(unit); | DMAR_UNLOCK(unit); | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
dmar_fini_qi(struct dmar_unit *unit) | dmar_fini_qi(struct dmar_unit *unit) | ||||
{ | { | ||||
struct dmar_qi_genseq gseq; | struct iommu_qi_genseq gseq; | ||||
if (!unit->qi_enabled) | if (!unit->qi_enabled) | ||||
return; | return; | ||||
taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); | taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); | ||||
taskqueue_free(unit->qi_taskqueue); | taskqueue_free(unit->qi_taskqueue); | ||||
unit->qi_taskqueue = NULL; | unit->qi_taskqueue = NULL; | ||||
DMAR_LOCK(unit); | DMAR_LOCK(unit); | ||||
/* quisce */ | /* quisce */ | ||||
dmar_qi_ensure(unit, 1); | dmar_qi_ensure(unit, 1); | ||||
dmar_qi_emit_wait_seq(unit, &gseq, true); | dmar_qi_emit_wait_seq(unit, &gseq, true); | ||||
dmar_qi_advance_tail(unit); | dmar_qi_advance_tail(unit); | ||||
dmar_qi_wait_for_seq(unit, &gseq, false); | dmar_qi_wait_for_seq(unit, &gseq, false); | ||||
/* only after the quisce, disable queue */ | /* only after the quisce, disable queue */ | ||||
dmar_disable_qi_intr(unit); | dmar_disable_qi_intr(unit); | ||||
dmar_disable_qi(unit); | dmar_disable_qi(unit); | ||||
KASSERT(unit->inv_seq_waiters == 0, | KASSERT(unit->inv_seq_waiters == 0, | ||||
("dmar%d: waiters on disabled queue", unit->unit)); | ("dmar%d: waiters on disabled queue", unit->iommu.unit)); | ||||
DMAR_UNLOCK(unit); | DMAR_UNLOCK(unit); | ||||
kmem_free(unit->inv_queue, unit->inv_queue_size); | kmem_free(unit->inv_queue, unit->inv_queue_size); | ||||
unit->inv_queue = 0; | unit->inv_queue = 0; | ||||
unit->inv_queue_size = 0; | unit->inv_queue_size = 0; | ||||
unit->qi_enabled = 0; | unit->qi_enabled = 0; | ||||
} | } | ||||
void | void | ||||
dmar_enable_qi_intr(struct dmar_unit *unit) | dmar_enable_qi_intr(struct dmar_unit *unit) | ||||
{ | { | ||||
uint32_t iectl; | uint32_t iectl; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); | KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", | ||||
unit->iommu.unit)); | |||||
iectl = dmar_read4(unit, DMAR_IECTL_REG); | iectl = dmar_read4(unit, DMAR_IECTL_REG); | ||||
iectl &= ~DMAR_IECTL_IM; | iectl &= ~DMAR_IECTL_IM; | ||||
dmar_write4(unit, DMAR_IECTL_REG, iectl); | dmar_write4(unit, DMAR_IECTL_REG, iectl); | ||||
} | } | ||||
void | void | ||||
dmar_disable_qi_intr(struct dmar_unit *unit) | dmar_disable_qi_intr(struct dmar_unit *unit) | ||||
{ | { | ||||
uint32_t iectl; | uint32_t iectl; | ||||
DMAR_ASSERT_LOCKED(unit); | DMAR_ASSERT_LOCKED(unit); | ||||
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); | KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", | ||||
unit->iommu.unit)); | |||||
iectl = dmar_read4(unit, DMAR_IECTL_REG); | iectl = dmar_read4(unit, DMAR_IECTL_REG); | ||||
dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); | dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); | ||||
} | } |