Differential D25574 Diff 74125 sys/x86/iommu/intel_intrmap.c

Changeset View

Standalone View

sys/x86/iommu/intel_intrmap.c

Show First 20 Lines • Show All 54 Lines • ▼ Show 20 Lines
#include <x86/include/busdma_impl.h>		#include <x86/include/busdma_impl.h>
#include <x86/iommu/intel_reg.h>		#include <x86/iommu/intel_reg.h>
#include <x86/iommu/busdma_dmar.h>		#include <x86/iommu/busdma_dmar.h>
#include <dev/pci/pcireg.h>		#include <dev/pci/pcireg.h>
#include <x86/iommu/intel_dmar.h>		#include <x86/iommu/intel_dmar.h>
#include <dev/pci/pcivar.h>		#include <dev/pci/pcivar.h>
#include <x86/iommu/iommu_intrmap.h>		#include <x86/iommu/iommu_intrmap.h>

static struct dmar_unit dmar_ir_find(device_t src, uint16_t rid,		static struct iommu_unit dmar_ir_find(device_t src, uint16_t rid,
int *is_dmar);		int *is_dmar);
static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,		static void dmar_ir_program_irte(struct iommu_unit *unit, u_int idx,
uint64_t low, uint16_t rid);		uint64_t low, uint16_t rid);
static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);		static int dmar_ir_free_irte(struct iommu_unit *unit, u_int cookie);

int		int
iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)		iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
{		{
struct dmar_unit *unit;		struct iommu_unit *unit;
vmem_addr_t vmem_res;		vmem_addr_t vmem_res;
u_int idx, i;		u_int idx, i;
int error;		int error;

unit = dmar_ir_find(src, NULL, NULL);		unit = dmar_ir_find(src, NULL, NULL);
if (unit == NULL \|\| !unit->ir_enabled) {		if (unit == NULL \|\| !unit->ir_enabled) {
for (i = 0; i < count; i++)		for (i = 0; i < count; i++)
cookies[i] = -1;		cookies[i] = -1;
Show All 12 Lines	for (i = 0; i < count; i++)
cookies[i] = idx + i;		cookies[i] = idx + i;
return (0);		return (0);
}		}

int		int
iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,		iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
uint64_t addr, uint32_t data)		uint64_t addr, uint32_t data)
{		{
struct dmar_unit *unit;		struct iommu_unit *unit;
uint64_t low;		uint64_t low;
uint16_t rid;		uint16_t rid;
int is_dmar;		int is_dmar;

unit = dmar_ir_find(src, &rid, &is_dmar);		unit = dmar_ir_find(src, &rid, &is_dmar);
if (is_dmar) {		if (is_dmar) {
KASSERT(unit == NULL, ("DMAR cannot translate itself"));		KASSERT(unit == NULL, ("DMAR cannot translate itself"));

Show All 28 Lines	if (addr != NULL) {
*data = 0;		*data = 0;
}		}
return (0);		return (0);
}		}

int		int
iommu_unmap_msi_intr(device_t src, u_int cookie)		iommu_unmap_msi_intr(device_t src, u_int cookie)
{		{
struct dmar_unit *unit;		struct iommu_unit *unit;

if (cookie == -1)		if (cookie == -1)
return (0);		return (0);
unit = dmar_ir_find(src, NULL, NULL);		unit = dmar_ir_find(src, NULL, NULL);
return (dmar_ir_free_irte(unit, cookie));		return (dmar_ir_free_irte(unit, cookie));
}		}

int		int
iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,		iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
bool activehi, int irq, u_int cookie, uint32_t hi, uint32_t *lo)		bool activehi, int irq, u_int cookie, uint32_t hi, uint32_t *lo)
{		{
struct dmar_unit *unit;		struct iommu_unit *unit;
vmem_addr_t vmem_res;		vmem_addr_t vmem_res;
uint64_t low, iorte;		uint64_t low, iorte;
u_int idx;		u_int idx;
int error;		int error;
uint16_t rid;		uint16_t rid;

unit = dmar_find_ioapic(ioapic_id, &rid);		unit = dmar_find_ioapic(ioapic_id, &rid);
if (unit == NULL \|\| !unit->ir_enabled) {		if (unit == NULL \|\| !unit->ir_enabled) {
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines	iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
}		}
*cookie = idx;		*cookie = idx;
return (0);		return (0);
}		}

int		int
iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)		iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
{		{
struct dmar_unit *unit;		struct iommu_unit *unit;
u_int idx;		u_int idx;

idx = *cookie;		idx = *cookie;
if (idx == -1)		if (idx == -1)
return (0);		return (0);
*cookie = -1;		*cookie = -1;
unit = dmar_find_ioapic(ioapic_id, NULL);		unit = dmar_find_ioapic(ioapic_id, NULL);
KASSERT(unit != NULL && unit->ir_enabled,		KASSERT(unit != NULL && unit->ir_enabled,
("unmap: cookie %d unit %p", idx, unit));		("unmap: cookie %d unit %p", idx, unit));
return (dmar_ir_free_irte(unit, idx));		return (dmar_ir_free_irte(unit, idx));
}		}

static struct dmar_unit *		static struct iommu_unit *
dmar_ir_find(device_t src, uint16_t rid, int is_dmar)		dmar_ir_find(device_t src, uint16_t rid, int is_dmar)
{		{
devclass_t src_class;		devclass_t src_class;
struct dmar_unit *unit;		struct iommu_unit *unit;

/*		/*
* We need to determine if the interrupt source generates FSB		* We need to determine if the interrupt source generates FSB
* interrupts. If yes, it is either DMAR, in which case		* interrupts. If yes, it is either DMAR, in which case
* interrupts are not remapped. Or it is HPET, and interrupts		* interrupts are not remapped. Or it is HPET, and interrupts
* are remapped. For HPET, source id is reported by HPET		* are remapped. For HPET, source id is reported by HPET
* record in DMAR ACPI table.		* record in DMAR ACPI table.
*/		*/
if (is_dmar != NULL)		if (is_dmar != NULL)
*is_dmar = FALSE;		*is_dmar = FALSE;
src_class = device_get_devclass(src);		src_class = device_get_devclass(src);
if (src_class == devclass_find("dmar")) {		if (src_class == devclass_find("dmar")) {
unit = NULL;		unit = NULL;
if (is_dmar != NULL)		if (is_dmar != NULL)
*is_dmar = TRUE;		*is_dmar = TRUE;
} else if (src_class == devclass_find("hpet")) {		} else if (src_class == devclass_find("hpet")) {
unit = dmar_find_hpet(src, rid);		unit = dmar_find_hpet(src, rid);
} else {		} else {
unit = dmar_find(src, bootverbose);		unit = dmar_find(src, bootverbose);
if (unit != NULL && rid != NULL)		if (unit != NULL && rid != NULL)
dmar_get_requester(src, rid);		iommu_get_requester(src, rid);
}		}
return (unit);		return (unit);
}		}

static void		static void
dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,		dmar_ir_program_irte(struct iommu_unit *unit, u_int idx, uint64_t low,
uint16_t rid)		uint16_t rid)
{		{
dmar_irte_t *irte;		dmar_irte_t *irte;
uint64_t high;		uint64_t high;

KASSERT(idx < unit->irte_cnt,		KASSERT(idx < unit->irte_cnt,
("bad cookie %d %d", idx, unit->irte_cnt));		("bad cookie %d %d", idx, unit->irte_cnt));
irte = &(unit->irt[idx]);		irte = &(unit->irt[idx]);
high = DMAR_IRTE2_SVT_RID \| DMAR_IRTE2_SQ_RID \|		high = DMAR_IRTE2_SVT_RID \| DMAR_IRTE2_SQ_RID \|
DMAR_IRTE2_SID_RID(rid);		DMAR_IRTE2_SID_RID(rid);
if (bootverbose) {		if (bootverbose) {
device_printf(unit->dev,		device_printf(unit->dev,
"programming irte[%d] rid %#x high %#jx low %#jx\n",		"programming irte[%d] rid %#x high %#jx low %#jx\n",
idx, rid, (uintmax_t)high, (uintmax_t)low);		idx, rid, (uintmax_t)high, (uintmax_t)low);
}		}
DMAR_LOCK(unit);		IOMMU_LOCK(unit);
if ((irte->irte1 & DMAR_IRTE1_P) != 0) {		if ((irte->irte1 & DMAR_IRTE1_P) != 0) {
/*		/*
* The rte is already valid. Assume that the request		* The rte is already valid. Assume that the request
* is to remap the interrupt for balancing. Only low		* is to remap the interrupt for balancing. Only low
* word of rte needs to be changed. Assert that the		* word of rte needs to be changed. Assert that the
* high word contains expected value.		* high word contains expected value.
*/		*/
KASSERT(irte->irte2 == high,		KASSERT(irte->irte2 == high,
("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2,		("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2,
(uintmax_t)high));		(uintmax_t)high));
dmar_pte_update(&irte->irte1, low);		dmar_pte_update(&irte->irte1, low);
} else {		} else {
dmar_pte_store(&irte->irte2, high);		dmar_pte_store(&irte->irte2, high);
dmar_pte_store(&irte->irte1, low);		dmar_pte_store(&irte->irte1, low);
}		}
dmar_qi_invalidate_iec(unit, idx, 1);		dmar_qi_invalidate_iec(unit, idx, 1);
DMAR_UNLOCK(unit);		IOMMU_UNLOCK(unit);

}		}

static int		static int
dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)		dmar_ir_free_irte(struct iommu_unit *unit, u_int cookie)
{		{
dmar_irte_t *irte;		dmar_irte_t *irte;

KASSERT(unit != NULL && unit->ir_enabled,		KASSERT(unit != NULL && unit->ir_enabled,
("unmap: cookie %d unit %p", cookie, unit));		("unmap: cookie %d unit %p", cookie, unit));
KASSERT(cookie < unit->irte_cnt,		KASSERT(cookie < unit->irte_cnt,
("bad cookie %u %u", cookie, unit->irte_cnt));		("bad cookie %u %u", cookie, unit->irte_cnt));
irte = &(unit->irt[cookie]);		irte = &(unit->irt[cookie]);
dmar_pte_clear(&irte->irte1);		dmar_pte_clear(&irte->irte1);
dmar_pte_clear(&irte->irte2);		dmar_pte_clear(&irte->irte2);
DMAR_LOCK(unit);		IOMMU_LOCK(unit);
dmar_qi_invalidate_iec(unit, cookie, 1);		dmar_qi_invalidate_iec(unit, cookie, 1);
DMAR_UNLOCK(unit);		IOMMU_UNLOCK(unit);
vmem_free(unit->irtids, cookie, 1);		vmem_free(unit->irtids, cookie, 1);
return (0);		return (0);
}		}

static u_int		static u_int
clp2(u_int v)		clp2(u_int v)
{		{

return (powerof2(v) ? v : 1 << fls(v));		return (powerof2(v) ? v : 1 << fls(v));
}		}

int		int
dmar_init_irt(struct dmar_unit *unit)		dmar_init_irt(struct iommu_unit *unit)
{		{

if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)		if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)
return (0);		return (0);
unit->ir_enabled = 1;		unit->ir_enabled = 1;
TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);		TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);
if (!unit->ir_enabled)		if (!unit->ir_enabled)
return (0);		return (0);
Show All 9 Lines	unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig(
unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO \| M_WAITOK, 0,		unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO \| M_WAITOK, 0,
dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ?		dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ?
VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE);		VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE);
if (unit->irt == NULL)		if (unit->irt == NULL)
return (ENOMEM);		return (ENOMEM);
unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);		unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);
unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,		unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,
M_FIRSTFIT \| M_NOWAIT);		M_FIRSTFIT \| M_NOWAIT);
DMAR_LOCK(unit);		IOMMU_LOCK(unit);
dmar_load_irt_ptr(unit);		dmar_load_irt_ptr(unit);
dmar_qi_invalidate_iec_glob(unit);		dmar_qi_invalidate_iec_glob(unit);
DMAR_UNLOCK(unit);		IOMMU_UNLOCK(unit);

/*		/*
* Initialize mappings for already configured interrupt pins.		* Initialize mappings for already configured interrupt pins.
* Required, because otherwise the interrupts fault without		* Required, because otherwise the interrupts fault without
* irtes.		* irtes.
*/		*/
intr_reprogram();		intr_reprogram();

DMAR_LOCK(unit);		IOMMU_LOCK(unit);
dmar_enable_ir(unit);		dmar_enable_ir(unit);
DMAR_UNLOCK(unit);		IOMMU_UNLOCK(unit);
return (0);		return (0);
}		}

void		void
dmar_fini_irt(struct dmar_unit *unit)		dmar_fini_irt(struct iommu_unit *unit)
{		{

unit->ir_enabled = 0;		unit->ir_enabled = 0;
if (unit->irt != NULL) {		if (unit->irt != NULL) {
dmar_disable_ir(unit);		dmar_disable_ir(unit);
dmar_qi_invalidate_iec_glob(unit);		dmar_qi_invalidate_iec_glob(unit);
vmem_destroy(unit->irtids);		vmem_destroy(unit->irtids);
kmem_free((vm_offset_t)unit->irt, unit->irte_cnt *		kmem_free((vm_offset_t)unit->irt, unit->irte_cnt *
sizeof(dmar_irte_t));		sizeof(dmar_irte_t));
}		}
}		}