D1892.diff
No OneTemporary
Actions

Size

48 KB

Referenced Files

None

Subscribers

None

D1892.diff
View Options

	Index: sys/amd64/include/intr_machdep.h
	===================================================================
	--- sys/amd64/include/intr_machdep.h
	+++ sys/amd64/include/intr_machdep.h
	@@ -106,6 +106,7 @@
	int (pic_config_intr)(struct intsrc , enum intr_trigger,
	enum intr_polarity);
	int (pic_assign_cpu)(struct intsrc , u_int apic_id);
	+ void (pic_reprogram_pin)(struct intsrc );
	TAILQ_ENTRY(pic) pics;
	};

	@@ -172,6 +173,7 @@
	int intr_remove_handler(void *cookie);
	void intr_resume(bool suspend_cancelled);
	void intr_suspend(void);
	+void intr_reprogram(void);
	void intrcnt_add(const char name, u_long *countp);
	void nexus_add_irq(u_long irq);
	int msi_alloc(device_t dev, int count, int maxcount, int *irqs);
	Index: sys/conf/files.amd64
	===================================================================
	--- sys/conf/files.amd64
	+++ sys/conf/files.amd64
	@@ -542,6 +542,7 @@
	x86/iommu/intel_fault.c optional acpi acpi_dmar pci
	x86/iommu/intel_gas.c optional acpi acpi_dmar pci
	x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci
	+x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci
	x86/iommu/intel_qi.c optional acpi acpi_dmar pci
	x86/iommu/intel_quirks.c optional acpi acpi_dmar pci
	x86/iommu/intel_utils.c optional acpi acpi_dmar pci
	Index: sys/conf/files.i386
	===================================================================
	--- sys/conf/files.i386
	+++ sys/conf/files.i386
	@@ -560,6 +560,7 @@
	x86/iommu/intel_fault.c optional acpi acpi_dmar pci
	x86/iommu/intel_gas.c optional acpi acpi_dmar pci
	x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci
	+x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci
	x86/iommu/intel_qi.c optional acpi acpi_dmar pci
	x86/iommu/intel_quirks.c optional acpi acpi_dmar pci
	x86/iommu/intel_utils.c optional acpi acpi_dmar pci
	Index: sys/i386/include/intr_machdep.h
	===================================================================
	--- sys/i386/include/intr_machdep.h
	+++ sys/i386/include/intr_machdep.h
	@@ -112,6 +112,7 @@
	int (pic_config_intr)(struct intsrc , enum intr_trigger,
	enum intr_polarity);
	int (pic_assign_cpu)(struct intsrc , u_int apic_id);
	+ void (pic_reprogram_pin)(struct intsrc );
	TAILQ_ENTRY(pic) pics;
	};

	@@ -168,6 +169,7 @@
	int intr_remove_handler(void *cookie);
	void intr_resume(bool suspend_cancelled);
	void intr_suspend(void);
	+void intr_reprogram(void);
	void intrcnt_add(const char name, u_long *countp);
	void nexus_add_irq(u_long irq);
	int msi_alloc(device_t dev, int count, int maxcount, int *irqs);
	Index: sys/x86/include/apicvar.h
	===================================================================
	--- sys/x86/include/apicvar.h
	+++ sys/x86/include/apicvar.h
	@@ -155,6 +155,11 @@
	#define APIC_BUS_PCI 2
	#define APIC_BUS_MAX APIC_BUS_PCI

	+#define IRQ_EXTINT (NUM_IO_INTS + 1)
	+#define IRQ_NMI (NUM_IO_INTS + 2)
	+#define IRQ_SMI (NUM_IO_INTS + 3)
	+#define IRQ_DISABLED (NUM_IO_INTS + 4)
	+
	/*
	* An APIC enumerator is a psuedo bus driver that enumerates APIC's including
	* CPU's and I/O APIC's.
	Index: sys/x86/iommu/busdma_dmar.c
	===================================================================
	--- sys/x86/iommu/busdma_dmar.c
	+++ sys/x86/iommu/busdma_dmar.c
	@@ -47,6 +47,7 @@
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	+#include <sys/vmem.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	@@ -92,7 +93,7 @@
	* domain, and must collectively be assigned to use either DMAR or
	* bounce mapping.
	*/
	-static device_t
	+device_t
	dmar_get_requester(device_t dev, uint16_t *rid)
	{
	devclass_t pci_class;
	@@ -255,6 +256,8 @@
	/* Not in scope of any DMAR ? */
	if (dmar == NULL)
	return (NULL);
	+ if (!dmar->dma_enabled)
	+ return (NULL);
	dmar_quirks_pre_use(dmar);
	dmar_instantiate_rmrr_ctxs(dmar);

	@@ -852,6 +855,8 @@
	dmar_init_busdma(struct dmar_unit *unit)
	{

	+ unit->dma_enabled = 1;
	+ TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
	TAILQ_INIT(&unit->delayed_maps);
	TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
	unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
	Index: sys/x86/iommu/intel_ctx.c
	===================================================================
	--- sys/x86/iommu/intel_ctx.c
	+++ sys/x86/iommu/intel_ctx.c
	@@ -48,6 +48,7 @@
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	+#include <sys/vmem.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	Index: sys/x86/iommu/intel_dmar.h
	===================================================================
	--- sys/x86/iommu/intel_dmar.h
	+++ sys/x86/iommu/intel_dmar.h
	@@ -1,5 +1,5 @@
	/*-
	- * Copyright (c) 2013 The FreeBSD Foundation
	+ * Copyright (c) 2013-2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	@@ -185,6 +185,13 @@
	u_int inv_seq_waiters; /* count of waiters for seq */
	u_int inv_queue_full; /* informational counter */

	+ /* IR */
	+ int ir_enabled;
	+ vm_paddr_t irt_phys;
	+ dmar_irte_t *irt;
	+ u_int irte_cnt;
	+ vmem_t *irtids;
	+
	/* Delayed freeing of map entries queue processing */
	struct dmar_map_entries_tailq tlb_flush_entries;
	struct task qi_task;
	@@ -194,6 +201,8 @@
	struct task dmamap_load_task;
	TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
	struct taskqueue *delayed_taskqueue;
	+
	+ int dma_enabled;
	};

	#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock)
	@@ -206,12 +215,16 @@

	#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
	#define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
	+#define DMAR_X2APIC(dmar) \
	+ (x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0)

	/* Barrier ids */
	#define DMAR_BARRIER_RMRR 0
	#define DMAR_BARRIER_USEQ 1

	struct dmar_unit *dmar_find(device_t dev);
	+struct dmar_unit dmar_find_hpet(device_t dev, uint16_t rid);
	+struct dmar_unit dmar_find_ioapic(u_int apic_id, uint16_t rid);

	u_int dmar_nd2mask(u_int nd);
	bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
	@@ -238,6 +251,9 @@
	void dmar_flush_root_to_ram(struct dmar_unit unit, dmar_root_entry_t dst);
	int dmar_enable_translation(struct dmar_unit *unit);
	int dmar_disable_translation(struct dmar_unit *unit);
	+int dmar_load_irt_ptr(struct dmar_unit *unit);
	+int dmar_enable_ir(struct dmar_unit *unit);
	+int dmar_disable_ir(struct dmar_unit *unit);
	bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
	void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);

	@@ -256,6 +272,8 @@
	dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
	void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
	void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
	+void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
	+void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt);

	vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
	void put_idmap_pgtbl(vm_object_t obj);
	@@ -282,6 +300,7 @@

	int dmar_init_busdma(struct dmar_unit *unit);
	void dmar_fini_busdma(struct dmar_unit *unit);
	+device_t dmar_get_requester(device_t dev, uint16_t *rid);

	void dmar_gas_init_ctx(struct dmar_ctx *ctx);
	void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
	@@ -304,6 +323,9 @@
	void dmar_quirks_post_ident(struct dmar_unit *dmar);
	void dmar_quirks_pre_use(struct dmar_unit *dmar);

	+int dmar_init_irt(struct dmar_unit *unit);
	+void dmar_fini_irt(struct dmar_unit *unit);
	+
	#define DMAR_GM_CANWAIT 0x0001
	#define DMAR_GM_CANSPLIT 0x0002

	@@ -374,13 +396,16 @@
	* containing the P or R and W bits, is set only after the high word
	* is written. For clear, the P bit is cleared first, then the high
	* word is cleared.
	+ *
	+ * dmar_pte_update updates the pte. For amd64, the update is atomic.
	+ * For i386, it first disables the entry by clearing the word
	+ * containing the P bit, and then defer to dmar_pte_store. The locked
	+ * cmpxchg8b is probably available on any machine having DMAR support,
	+ * but interrupt translation table may be mapped uncached.
	*/
	static inline void
	-dmar_pte_store(volatile uint64_t *dst, uint64_t val)
	+dmar_pte_store1(volatile uint64_t *dst, uint64_t val)
	{
	-
	- KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
	- dst, (uintmax_t)*dst, (uintmax_t)val));
	#ifdef __i386__
	volatile uint32_t *p;
	uint32_t hi, lo;
	@@ -396,6 +421,28 @@
	}

	static inline void
	+dmar_pte_store(volatile uint64_t *dst, uint64_t val)
	+{
	+
	+ KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
	+ dst, (uintmax_t)*dst, (uintmax_t)val));
	+ dmar_pte_store1(dst, val);
	+}
	+
	+static inline void
	+dmar_pte_update(volatile uint64_t *dst, uint64_t val)
	+{
	+
	+#ifdef __i386__
	+ volatile uint32_t *p;
	+
	+ p = (volatile uint32_t *)dst;
	+ *p = 0;
	+#endif
	+ dmar_pte_store1(dst, val);
	+}
	+
	+static inline void
	dmar_pte_clear(volatile uint64_t *dst)
	{
	#ifdef __i386__
	Index: sys/x86/iommu/intel_drv.c
	===================================================================
	--- sys/x86/iommu/intel_drv.c
	+++ sys/x86/iommu/intel_drv.c
	@@ -1,5 +1,5 @@
	/*-
	- * Copyright (c) 2013 The FreeBSD Foundation
	+ * Copyright (c) 2013-2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	@@ -50,6 +50,7 @@
	#include <sys/smp.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	+#include <sys/vmem.h>
	#include <machine/bus.h>
	#include <contrib/dev/acpica/include/acpi.h>
	#include <contrib/dev/acpica/include/accommon.h>
	@@ -65,6 +66,7 @@
	#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/busdma_dmar.h>
	#include <x86/iommu/intel_dmar.h>
	+#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>

	#ifdef DEV_APIC
	@@ -243,6 +245,7 @@
	int i;

	dmar_fini_busdma(unit);
	+ dmar_fini_irt(unit);
	dmar_fini_qi(unit);
	dmar_fini_fault_log(unit);
	for (i = 0; i < DMAR_INTR_TOTAL; i++)
	@@ -509,6 +512,11 @@
	dmar_release_resources(dev, unit);
	return (error);
	}
	+ error = dmar_init_irt(unit);
	+ if (error != 0) {
	+ dmar_release_resources(dev, unit);
	+ return (error);
	+ }
	error = dmar_init_busdma(unit);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	@@ -763,6 +771,76 @@
	return (device_get_softc(dmar_dev));
	}

	+static struct dmar_unit *
	+dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid)
	+{
	+ device_t dmar_dev;
	+ struct dmar_unit *unit;
	+ ACPI_DMAR_HARDWARE_UNIT *dmarh;
	+ ACPI_DMAR_DEVICE_SCOPE *devscope;
	+ ACPI_DMAR_PCI_PATH *path;
	+ char ptr, ptrend;
	+ int i;
	+
	+ for (i = 0; i < dmar_devcnt; i++) {
	+ dmar_dev = dmar_devs[i];
	+ if (dmar_dev == NULL)
	+ continue;
	+ unit = (struct dmar_unit *)device_get_softc(dmar_dev);
	+ dmarh = dmar_find_by_index(i);
	+ if (dmarh == NULL)
	+ continue;
	+ ptr = (char )dmarh + sizeof(dmarh);
	+ ptrend = (char *)dmarh + dmarh->Header.Length;
	+ for (;;) {
	+ if (ptr >= ptrend)
	+ break;
	+ devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr;
	+ ptr += devscope->Length;
	+ if (devscope->EntryType != entry_type)
	+ continue;
	+ if (devscope->EnumerationId != id)
	+ continue;
	+ if (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE)
	+ == 2) {
	+ if (rid != NULL) {
	+ path = (ACPI_DMAR_PCI_PATH *)
	+ (devscope + 1);
	+ *rid = PCI_RID(devscope->Bus,
	+ path->Device, path->Function);
	+ }
	+ return (unit);
	+ } else {
	+ /* XXXKIB */
	+ printf(
	+ "dmar_find_nonpci: id %d type %d path length != 2\n",
	+ id, entry_type);
	+ }
	+ }
	+ }
	+ return (NULL);
	+}
	+
	+
	+struct dmar_unit *
	+dmar_find_hpet(device_t dev, uint16_t *rid)
	+{
	+ ACPI_HANDLE handle;
	+ uint32_t hpet_id;
	+
	+ handle = acpi_get_handle(dev);
	+ if (ACPI_FAILURE(acpi_GetInteger(handle, "_UID", &hpet_id)))
	+ return (NULL);
	+ return (dmar_find_nonpci(hpet_id, ACPI_DMAR_SCOPE_TYPE_HPET, rid));
	+}
	+
	+struct dmar_unit *
	+dmar_find_ioapic(u_int apic_id, uint16_t *rid)
	+{
	+
	+ return (dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid));
	+}
	+
	struct rmrr_iter_args {
	struct dmar_ctx *ctx;
	device_t dev;
	Index: sys/x86/iommu/intel_fault.c
	===================================================================
	--- sys/x86/iommu/intel_fault.c
	+++ sys/x86/iommu/intel_fault.c
	@@ -41,6 +41,7 @@
	#include <sys/rman.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	+#include <sys/vmem.h>
	#include <machine/bus.h>
	#include <contrib/dev/acpica/include/acpi.h>
	#include <contrib/dev/acpica/include/accommon.h>
	Index: sys/x86/iommu/intel_gas.c
	===================================================================
	--- sys/x86/iommu/intel_gas.c
	+++ sys/x86/iommu/intel_gas.c
	@@ -49,6 +49,7 @@
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	+#include <sys/vmem.h>
	#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	Index: sys/x86/iommu/intel_idpgtbl.c
	===================================================================
	--- sys/x86/iommu/intel_idpgtbl.c
	+++ sys/x86/iommu/intel_idpgtbl.c
	@@ -48,6 +48,7 @@
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	+#include <sys/vmem.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	Index: sys/x86/iommu/intel_intrmap.c
	===================================================================
	--- /dev/null
	+++ sys/x86/iommu/intel_intrmap.c
	@@ -0,0 +1,380 @@
	+/*-
	+ * Copyright (c) 2015 The FreeBSD Foundation
	+ * All rights reserved.
	+ *
	+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	+ * under sponsorship from the FreeBSD Foundation.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/lock.h>
	+#include <sys/malloc.h>
	+#include <sys/memdesc.h>
	+#include <sys/rman.h>
	+#include <sys/rwlock.h>
	+#include <sys/taskqueue.h>
	+#include <sys/tree.h>
	+#include <sys/vmem.h>
	+#include <machine/bus.h>
	+#include <machine/intr_machdep.h>
	+#include <vm/vm.h>
	+#include <vm/vm_extern.h>
	+#include <vm/vm_kern.h>
	+#include <vm/vm_object.h>
	+#include <vm/vm_page.h>
	+#include <x86/include/apicreg.h>
	+#include <x86/include/apicvar.h>
	+#include <x86/include/busdma_impl.h>
	+#include <x86/iommu/intel_reg.h>
	+#include <x86/iommu/busdma_dmar.h>
	+#include <x86/iommu/intel_dmar.h>
	+#include <dev/pci/pcivar.h>
	+#include <x86/iommu/iommu_intrmap.h>
	+
	+static struct dmar_unit dmar_ir_find(device_t src, uint16_t rid,
	+ int *is_dmar);
	+static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
	+ uint64_t low, uint16_t rid);
	+static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);
	+
	+int
	+iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
	+{
	+ struct dmar_unit *unit;
	+ vmem_addr_t vmem_res;
	+ u_int idx, i;
	+ int error;
	+
	+ unit = dmar_ir_find(src, NULL, NULL);
	+ if (unit == NULL \|\| !unit->ir_enabled) {
	+ for (i = 0; i < count; i++)
	+ cookies[i] = -1;
	+ return (EOPNOTSUPP);
	+ }
	+
	+ error = vmem_alloc(unit->irtids, count, M_FIRSTFIT \| M_NOWAIT,
	+ &vmem_res);
	+ if (error != 0) {
	+ KASSERT(error != EOPNOTSUPP,
	+ ("impossible EOPNOTSUPP from vmem"));
	+ return (error);
	+ }
	+ idx = vmem_res;
	+ for (i = 0; i < count; i++)
	+ cookies[i] = idx + i;
	+ return (0);
	+}
	+
	+int
	+iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
	+ uint64_t addr, uint32_t data)
	+{
	+ struct dmar_unit *unit;
	+ uint64_t low;
	+ uint16_t rid;
	+ int is_dmar;
	+
	+ unit = dmar_ir_find(src, &rid, &is_dmar);
	+ if (is_dmar) {
	+ KASSERT(unit == NULL, ("DMAR cannot translate itself"));
	+
	+ /*
	+ * See VT-d specification, 5.1.6 Remapping Hardware -
	+ * Interrupt Programming.
	+ */
	+ *data = vector;
	+ *addr = MSI_INTEL_ADDR_BASE \| ((cpu & 0xff) << 12);
	+ if (x2apic_mode)
	+ *addr \|= ((uint64_t)cpu & 0xffffff00) << 32;
	+ else
	+ KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu));
	+ return (0);
	+ }
	+ if (unit == NULL \|\| !unit->ir_enabled \|\| cookie == -1)
	+ return (EOPNOTSUPP);
	+
	+ low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
	+ DMAR_IRTE1_DST_xAPIC(cpu)) \| DMAR_IRTE1_V(vector) \|
	+ DMAR_IRTE1_DLM_FM \| DMAR_IRTE1_TM_EDGE \| DMAR_IRTE1_RH_DIRECT \|
	+ DMAR_IRTE1_DM_PHYSICAL \| DMAR_IRTE1_P;
	+ dmar_ir_program_irte(unit, cookie, low, rid);
	+
	+ if (addr != NULL) {
	+ /*
	+ * See VT-d specification, 5.1.5.2 MSI and MSI-X
	+ * Register Programming.
	+ */
	+ *addr = MSI_INTEL_ADDR_BASE \| ((cookie & 0x7fff) << 5) \|
	+ ((cookie & 0x8000) << 2) \| 0x18;
	+ *data = 0;
	+ }
	+ return (0);
	+}
	+
	+int
	+iommu_unmap_msi_intr(device_t src, u_int cookie)
	+{
	+ struct dmar_unit *unit;
	+
	+ if (cookie == -1)
	+ return (0);
	+ unit = dmar_ir_find(src, NULL, NULL);
	+ return (dmar_ir_free_irte(unit, cookie));
	+}
	+
	+int
	+iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
	+ bool activehi, int irq, u_int cookie, uint32_t hi, uint32_t *lo)
	+{
	+ struct dmar_unit *unit;
	+ vmem_addr_t vmem_res;
	+ uint64_t low, iorte;
	+ u_int idx;
	+ int error;
	+ uint16_t rid;
	+
	+ unit = dmar_find_ioapic(ioapic_id, &rid);
	+ if (unit == NULL \|\| !unit->ir_enabled) {
	+ *cookie = -1;
	+ return (EOPNOTSUPP);
	+ }
	+
	+ error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT \| M_NOWAIT, &vmem_res);
	+ if (error != 0) {
	+ KASSERT(error != EOPNOTSUPP,
	+ ("impossible EOPNOTSUPP from vmem"));
	+ return (error);
	+ }
	+ idx = vmem_res;
	+ low = 0;
	+ switch (irq) {
	+ case IRQ_EXTINT:
	+ low \|= DMAR_IRTE1_DLM_ExtINT;
	+ break;
	+ case IRQ_NMI:
	+ low \|= DMAR_IRTE1_DLM_NMI;
	+ break;
	+ case IRQ_SMI:
	+ low \|= DMAR_IRTE1_DLM_SMI;
	+ break;
	+ default:
	+ KASSERT(vector != 0, ("No vector for IRQ %u", irq));
	+ low \|= DMAR_IRTE1_DLM_FM \| DMAR_IRTE1_V(vector);
	+ break;
	+ }
	+ low \|= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
	+ DMAR_IRTE1_DST_xAPIC(cpu)) \|
	+ (edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) \|
	+ DMAR_IRTE1_RH_DIRECT \| DMAR_IRTE1_DM_PHYSICAL \| DMAR_IRTE1_P;
	+ dmar_ir_program_irte(unit, idx, low, rid);
	+
	+ if (hi != NULL) {
	+ /*
	+ * See VT-d specification, 5.1.5.1 I/OxAPIC
	+ * Programming.
	+ */
	+ iorte = (1ULL << 48) \| ((uint64_t)(idx & 0x7fff) << 49) \|
	+ ((idx & 0x8000) != 0 ? (1 << 11) : 0) \|
	+ (edge ? IOART_TRGREDG : IOART_TRGRLVL) \|
	+ (activehi ? IOART_INTAHI : IOART_INTALO) \|
	+ IOART_DELFIXED \| vector;
	+ *hi = iorte >> 32;
	+ *lo = iorte;
	+ }
	+ *cookie = idx;
	+ return (0);
	+}
	+
	+int
	+iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
	+{
	+ struct dmar_unit *unit;
	+ u_int idx;
	+
	+ idx = *cookie;
	+ if (idx == -1)
	+ return (0);
	+ *cookie = -1;
	+ unit = dmar_find_ioapic(ioapic_id, NULL);
	+ KASSERT(unit != NULL && unit->ir_enabled,
	+ ("unmap: cookie %d unit %p", idx, unit));
	+ return (dmar_ir_free_irte(unit, idx));
	+}
	+
	+static struct dmar_unit *
	+dmar_ir_find(device_t src, uint16_t rid, int is_dmar)
	+{
	+ devclass_t src_class;
	+ struct dmar_unit *unit;
	+
	+ /*
	+ * We need to determine if the interrupt source generates FSB
	+ * interrupts. If yes, it is either DMAR, in which case
	+ * interrupts are not remapped. Or it is HPET, and interrupts
	+ * are remapped. For HPET, source id is reported by HPET
	+ * record in DMAR ACPI table.
	+ */
	+ if (is_dmar != NULL)
	+ *is_dmar = FALSE;
	+ src_class = device_get_devclass(src);
	+ if (src_class == devclass_find("dmar")) {
	+ unit = NULL;
	+ if (is_dmar != NULL)
	+ *is_dmar = TRUE;
	+ } else if (src_class == devclass_find("hpet")) {
	+ unit = dmar_find_hpet(src, rid);
	+ } else {
	+ unit = dmar_find(src);
	+ if (unit != NULL && rid != NULL)
	+ dmar_get_requester(src, rid);
	+ }
	+ return (unit);
	+}
	+
	+static void
	+dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,
	+ uint16_t rid)
	+{
	+ dmar_irte_t *irte;
	+ uint64_t high;
	+
	+ KASSERT(idx < unit->irte_cnt,
	+ ("bad cookie %d %d", idx, unit->irte_cnt));
	+ irte = &(unit->irt[idx]);
	+ high = DMAR_IRTE2_SVT_RID \| DMAR_IRTE2_SQ_RID \|
	+ DMAR_IRTE2_SID_RID(rid);
	+ device_printf(unit->dev,
	+ "programming irte[%d] rid %#x high %#jx low %#jx\n",
	+ idx, rid, (uintmax_t)high, (uintmax_t)low);
	+ DMAR_LOCK(unit);
	+ if ((irte->irte1 & DMAR_IRTE1_P) != 0) {
	+ /*
	+ * The rte is already valid. Assume that the request
	+ * is to remap the interrupt for balancing. Only low
	+ * word of rte needs to be changed. Assert that the
	+ * high word contains expected value.
	+ */
	+ KASSERT(irte->irte2 == high,
	+ ("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2,
	+ (uintmax_t)high));
	+ dmar_pte_update(&irte->irte1, low);
	+ } else {
	+ dmar_pte_store(&irte->irte2, high);
	+ dmar_pte_store(&irte->irte1, low);
	+ }
	+ dmar_qi_invalidate_iec(unit, idx, 1);
	+ DMAR_UNLOCK(unit);
	+
	+}
	+
	+static int
	+dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)
	+{
	+ dmar_irte_t *irte;
	+
	+ KASSERT(unit != NULL && unit->ir_enabled,
	+ ("unmap: cookie %d unit %p", cookie, unit));
	+ KASSERT(cookie < unit->irte_cnt,
	+ ("bad cookie %u %u", cookie, unit->irte_cnt));
	+ irte = &(unit->irt[cookie]);
	+ dmar_pte_clear(&irte->irte1);
	+ dmar_pte_clear(&irte->irte2);
	+ DMAR_LOCK(unit);
	+ dmar_qi_invalidate_iec(unit, cookie, 1);
	+ DMAR_UNLOCK(unit);
	+ vmem_free(unit->irtids, cookie, 1);
	+ return (0);
	+}
	+
	+static u_int
	+clp2(u_int v)
	+{
	+
	+ return (powerof2(v) ? v : 1 << fls(v));
	+}
	+
	+int
	+dmar_init_irt(struct dmar_unit *unit)
	+{
	+
	+ if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)
	+ return (0);
	+ unit->ir_enabled = 1;
	+ TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);
	+ if (!unit->ir_enabled)
	+ return (0);
	+ if (!unit->qi_enabled) {
	+ unit->ir_enabled = 0;
	+ if (bootverbose)
	+ device_printf(unit->dev,
	+ "QI disabled, disabling interrupt remapping\n");
	+ return (0);
	+ }
	+ unit->irte_cnt = clp2(NUM_IO_INTS);
	+ unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig(kernel_arena,
	+ unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO \| M_WAITOK, 0,
	+ dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ?
	+ VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE);
	+ if (unit->irt == NULL)
	+ return (ENOMEM);
	+ unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);
	+ unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,
	+ M_FIRSTFIT \| M_NOWAIT);
	+ DMAR_LOCK(unit);
	+ dmar_load_irt_ptr(unit);
	+ dmar_qi_invalidate_iec_glob(unit);
	+ DMAR_UNLOCK(unit);
	+
	+ /*
	+ * Initialize mappings for already configured interrupt pins.
	+ * Required, because otherwise the interrupts fault without
	+ * irtes.
	+ */
	+ intr_reprogram();
	+
	+ DMAR_LOCK(unit);
	+ dmar_enable_ir(unit);
	+ DMAR_UNLOCK(unit);
	+ return (0);
	+}
	+
	+void
	+dmar_fini_irt(struct dmar_unit *unit)
	+{
	+
	+ unit->ir_enabled = 0;
	+ if (unit->irt != NULL) {
	+ dmar_disable_ir(unit);
	+ dmar_qi_invalidate_iec_glob(unit);
	+ vmem_destroy(unit->irtids);
	+ kmem_free(kernel_arena, (vm_offset_t)unit->irt,
	+ unit->irte_cnt * sizeof(dmar_irte_t));
	+ }
	+}
	Index: sys/x86/iommu/intel_qi.c
	===================================================================
	--- sys/x86/iommu/intel_qi.c
	+++ sys/x86/iommu/intel_qi.c
	@@ -41,6 +41,7 @@
	#include <sys/rman.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	+#include <sys/vmem.h>
	#include <machine/bus.h>
	#include <contrib/dev/acpica/include/acpi.h>
	#include <contrib/dev/acpica/include/accommon.h>
	@@ -194,13 +195,14 @@
	}

	static void
	-dmar_qi_wait_for_seq(struct dmar_unit unit, const struct dmar_qi_genseq gseq)
	+dmar_qi_wait_for_seq(struct dmar_unit unit, const struct dmar_qi_genseq gseq,
	+ bool nowait)
	{

	DMAR_ASSERT_LOCKED(unit);
	unit->inv_seq_waiters++;
	while (!dmar_qi_seq_processed(unit, gseq)) {
	- if (cold) {
	+ if (cold \|\| nowait) {
	cpu_spinwait();
	} else {
	msleep(&unit->inv_seq_waiters, &unit->lock, 0,
	@@ -246,7 +248,7 @@
	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV \| DMAR_IQ_DESCR_CTX_GLOB, 0);
	dmar_qi_emit_wait_seq(unit, &gseq);
	dmar_qi_advance_tail(unit);
	- dmar_qi_wait_for_seq(unit, &gseq);
	+ dmar_qi_wait_for_seq(unit, &gseq, false);
	}

	void
	@@ -260,7 +262,60 @@
	DMAR_IQ_DESCR_IOTLB_DW \| DMAR_IQ_DESCR_IOTLB_DR, 0);
	dmar_qi_emit_wait_seq(unit, &gseq);
	dmar_qi_advance_tail(unit);
	- dmar_qi_wait_for_seq(unit, &gseq);
	+ dmar_qi_wait_for_seq(unit, &gseq, false);
	+}
	+
	+void
	+dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
	+{
	+ struct dmar_qi_genseq gseq;
	+
	+ DMAR_ASSERT_LOCKED(unit);
	+ dmar_qi_ensure(unit, 2);
	+ dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
	+ dmar_qi_emit_wait_seq(unit, &gseq);
	+ dmar_qi_advance_tail(unit);
	+ dmar_qi_wait_for_seq(unit, &gseq, false);
	+}
	+
	+void
	+dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
	+{
	+ struct dmar_qi_genseq gseq;
	+ u_int c, l;
	+
	+ DMAR_ASSERT_LOCKED(unit);
	+ KASSERT(start < unit->irte_cnt && start < start + cnt &&
	+ start + cnt <= unit->irte_cnt,
	+ ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
	+ for (; cnt > 0; cnt -= c, start += c) {
	+ l = ffs(start \| cnt) - 1;
	+ c = 1 << l;
	+ dmar_qi_ensure(unit, 1);
	+ dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV \|
	+ DMAR_IQ_DESCR_IEC_IDX \| DMAR_IQ_DESCR_IEC_IIDX(start) \|
	+ DMAR_IQ_DESCR_IEC_IM(l), 0);
	+ }
	+ dmar_qi_ensure(unit, 1);
	+ dmar_qi_emit_wait_seq(unit, &gseq);
	+ dmar_qi_advance_tail(unit);
	+
	+ /*
	+ * The caller of the function, in particular,
	+ * dmar_ir_program_irte(), may be called from the context
	+ * where the sleeping is forbidden (in fact, the
	+ * intr_table_lock mutex may be held, locked from
	+ * intr_shuffle_irqs()). Wait for the invalidation completion
	+ * using the busy wait.
	+ *
	+ * The impact on the interrupt input setup code is small, the
	+ * expected overhead is comparable with the chipset register
	+ * read. It is more harmful for the parallel DMA operations,
	+ * since we own the dmar unit lock until whole invalidation
	+ * queue is processed, which includes requests possibly issued
	+ * before our request.
	+ */
	+ dmar_qi_wait_for_seq(unit, &gseq, true);
	}

	int
	@@ -377,7 +432,7 @@
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit_wait_seq(unit, &gseq);
	dmar_qi_advance_tail(unit);
	- dmar_qi_wait_for_seq(unit, &gseq);
	+ dmar_qi_wait_for_seq(unit, &gseq, false);
	/* only after the quisce, disable queue */
	dmar_disable_qi(unit);
	KASSERT(unit->inv_seq_waiters == 0,
	Index: sys/x86/iommu/intel_quirks.c
	===================================================================
	--- sys/x86/iommu/intel_quirks.c
	+++ sys/x86/iommu/intel_quirks.c
	@@ -1,5 +1,5 @@
	/*-
	- * Copyright (c) 2013 The FreeBSD Foundation
	+ * Copyright (c) 2013, 2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	@@ -42,6 +42,7 @@
	#include <sys/smp.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	+#include <sys/vmem.h>
	#include <machine/bus.h>
	#include <contrib/dev/acpica/include/acpi.h>
	#include <contrib/dev/acpica/include/accommon.h>
	@@ -59,7 +60,7 @@
	#include <x86/iommu/intel_dmar.h>
	#include <dev/pci/pcivar.h>

	-typedef void (dmar_quirk_fun)(struct dmar_unit );
	+typedef void (dmar_quirk_cpu_fun)(struct dmar_unit );

	struct intel_dmar_quirk_cpu {
	u_int ext_family;
	@@ -67,17 +68,21 @@
	u_int family_code;
	u_int model;
	u_int stepping;
	- dmar_quirk_fun quirk;
	+ dmar_quirk_cpu_fun quirk;
	const char *descr;
	};

	+typedef void (dmar_quirk_nb_fun)(struct dmar_unit , device_t nb);
	+
	struct intel_dmar_quirk_nb {
	u_int dev_id;
	u_int rev_no;
	- dmar_quirk_fun quirk;
	+ dmar_quirk_nb_fun quirk;
	const char *descr;
	};

	+#define QUIRK_NB_ALL_REV 0xffffffff
	+
	static void
	dmar_match_quirks(struct dmar_unit *dmar,
	const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len,
	@@ -99,13 +104,14 @@
	for (i = 0; i < nb_quirks_len; i++) {
	nb_quirk = &nb_quirks[i];
	if (nb_quirk->dev_id == dev_id &&
	- nb_quirk->rev_no == rev_no) {
	+ (nb_quirk->rev_no == rev_no \|\|
	+ nb_quirk->rev_no == QUIRK_NB_ALL_REV)) {
	if (bootverbose) {
	device_printf(dmar->dev,
	"NB IOMMU quirk %s\n",
	nb_quirk->descr);
	}
	- nb_quirk->quirk(dmar);
	+ nb_quirk->quirk(dmar, nb);
	}
	}
	} else {
	@@ -139,12 +145,29 @@
	}

	static void
	-nb_5400_no_low_high_prot_mem(struct dmar_unit *unit)
	+nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused)
	{

	unit->hw_cap &= ~(DMAR_CAP_PHMR \| DMAR_CAP_PLMR);
	}

	+static void
	+nb_no_ir(struct dmar_unit *unit, device_t nb __unused)
	+{
	+
	+ unit->hw_ecap &= ~(DMAR_ECAP_IR \| DMAR_ECAP_EIM);
	+}
	+
	+static void
	+nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb)
	+{
	+ u_int rev_no;
	+
	+ rev_no = pci_get_revid(nb);
	+ if (rev_no <= 0x13)
	+ nb_no_ir(unit, nb);
	+}
	+
	static const struct intel_dmar_quirk_nb pre_use_nb[] = {
	{
	.dev_id = 0x4001, .rev_no = 0x20,
	@@ -156,6 +179,26 @@
	.quirk = nb_5400_no_low_high_prot_mem,
	.descr = "5400 E23" /* no low/high protected memory */
	},
	+ {
	+ .dev_id = 0x3403, .rev_no = QUIRK_NB_ALL_REV,
	+ .quirk = nb_5500_no_ir_rev13,
	+ .descr = "5500 E47, E53" /* interrupt remapping does not work */
	+ },
	+ {
	+ .dev_id = 0x3405, .rev_no = QUIRK_NB_ALL_REV,
	+ .quirk = nb_5500_no_ir_rev13,
	+ .descr = "5500 E47, E53" /* interrupt remapping does not work */
	+ },
	+ {
	+ .dev_id = 0x3405, .rev_no = 0x22,
	+ .quirk = nb_no_ir,
	+ .descr = "5500 E47, E53" /* interrupt remapping does not work */
	+ },
	+ {
	+ .dev_id = 0x3406, .rev_no = QUIRK_NB_ALL_REV,
	+ .quirk = nb_5500_no_ir_rev13,
	+ .descr = "5500 E47, E53" /* interrupt remapping does not work */
	+ },
	};

	static void
	Index: sys/x86/iommu/intel_reg.h
	===================================================================
	--- sys/x86/iommu/intel_reg.h
	+++ sys/x86/iommu/intel_reg.h
	@@ -103,7 +103,7 @@
	#define DMAR_IRTE1_IM_POSTED (1ULL << 15) /* Posted */
	/* Delivery Mode */
	#define DMAR_IRTE1_DLM_FM (0ULL << 5)
	-#define DMAR_IRTE1_DLM_LP (1ULL << 5
	+#define DMAR_IRTE1_DLM_LP (1ULL << 5)
	#define DMAR_IRTE1_DLM_SMI (2ULL << 5)
	#define DMAR_IRTE1_DLM_NMI (4ULL << 5)
	#define DMAR_IRTE1_DLM_INIT (5ULL << 5)
	@@ -180,7 +180,7 @@
	/* IOTLB Register Offset */
	#define DMAR_ECAP_SC (1 << 7) /* Snoop Control */
	#define DMAR_ECAP_PT (1 << 6) /* Pass Through */
	-#define DMAR_ECAP_EIM (1 << 4) /* Extended Interrupt Mode */
	+#define DMAR_ECAP_EIM (1 << 4) /* Extended Interrupt Mode (x2APIC) */
	#define DMAR_ECAP_IR (1 << 3) /* Interrupt Remapping */
	#define DMAR_ECAP_DI (1 << 2) /* Device IOTLB */
	#define DMAR_ECAP_QI (1 << 1) /* Queued Invalidation */
	@@ -320,8 +320,8 @@
	#define DMAR_IQ_DESCR_SZ (1 << DMAR_IQ_DESCR_SZ_SHIFT)
	/* Descriptor size */

	-#define DMAR_IQ_DESCR_CTX_INV 0x1 /* Context-cache Invalidate
	- Descriptor */
	+/* Context-cache Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_CTX_INV 0x1
	#define DMAR_IQ_DESCR_CTX_GLOB (0x1 << 4) /* Granularity: Global */
	#define DMAR_IQ_DESCR_CTX_DOM (0x2 << 4) /* Granularity: Domain */
	#define DMAR_IQ_DESCR_CTX_DEV (0x3 << 4) /* Granularity: Device */
	@@ -329,7 +329,8 @@
	#define DMAR_IQ_DESCR_CTX_SRC(x) (((uint64_t)(x)) << 32) /* Source Id */
	#define DMAR_IQ_DESCR_CTX_FM(x) (((uint64_t)(x)) << 48) /* Function Mask */

	-#define DMAR_IQ_DESCR_IOTLB_INV 0x2 /* IOTLB Invalidate Descriptor */
	+/* IOTLB Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_IOTLB_INV 0x2
	#define DMAR_IQ_DESCR_IOTLB_GLOB (0x1 << 4) /* Granularity: Global */
	#define DMAR_IQ_DESCR_IOTLB_DOM (0x2 << 4) /* Granularity: Domain */
	#define DMAR_IQ_DESCR_IOTLB_PAGE (0x3 << 4) /* Granularity: Page */
	@@ -337,17 +338,31 @@
	#define DMAR_IQ_DESCR_IOTLB_DR (1 << 7) /* Drain Reads */
	#define DMAR_IQ_DESCR_IOTLB_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */

	-#define DMAR_IQ_DESCR_IEC_INV 0x4 /* Invalidate Interrupt Entry Cache */
	+/* Device-TLB Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_DTLB_INV 0x3
	+
	+/* Invalidate Interrupt Entry Cache */
	+#define DMAR_IQ_DESCR_IEC_INV 0x4
	#define DMAR_IQ_DESCR_IEC_IDX (1 << 4) /* Index-Selective Invalidation */
	#define DMAR_IQ_DESCR_IEC_IIDX(x) (((uint64_t)x) << 32) /* Interrupt Index */
	#define DMAR_IQ_DESCR_IEC_IM(x) ((x) << 27) /* Index Mask */

	-#define DMAR_IQ_DESCR_WAIT_ID 0x5 /* Invalidation Wait Descriptor */
	+/* Invalidation Wait Descriptor */
	+#define DMAR_IQ_DESCR_WAIT_ID 0x5
	#define DMAR_IQ_DESCR_WAIT_IF (1 << 4) /* Interrupt Flag */
	#define DMAR_IQ_DESCR_WAIT_SW (1 << 5) /* Status Write */
	#define DMAR_IQ_DESCR_WAIT_FN (1 << 6) /* Fence */
	#define DMAR_IQ_DESCR_WAIT_SD(x) (((uint64_t)(x)) << 32) /* Status Data */

	+/* Extended IOTLB Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_EIOTLB_INV 0x6
	+
	+/* PASID-Cache Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_PASIDC_INV 0x7
	+
	+/* Extended Device-TLB Invalidate Descriptor */
	+#define DMAR_IQ_DESCR_EDTLB_INV 0x8
	+
	/* Invalidation Queue Head register */
	#define DMAR_IQH_REG 0x80
	#define DMAR_IQH_MASK 0x7fff0 /* Next cmd index mask */
	Index: sys/x86/iommu/intel_utils.c
	===================================================================
	--- sys/x86/iommu/intel_utils.c
	+++ sys/x86/iommu/intel_utils.c
	@@ -47,6 +47,7 @@
	#include <sys/systm.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	+#include <sys/vmem.h>
	#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	@@ -57,6 +58,8 @@
	#include <vm/vm_pageout.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	+#include <machine/intr_machdep.h>
	+#include <x86/include/apicvar.h>
	#include <x86/include/busdma_impl.h>
	#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/busdma_dmar.h>
	@@ -521,6 +524,55 @@
	return (0);
	}

	+int
	+dmar_load_irt_ptr(struct dmar_unit *unit)
	+{
	+ uint64_t irta, s;
	+
	+ DMAR_ASSERT_LOCKED(unit);
	+ irta = unit->irt_phys;
	+ if (DMAR_X2APIC(unit))
	+ irta \|= DMAR_IRTA_EIME;
	+ s = fls(unit->irte_cnt) - 2;
	+ KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK &&
	+ powerof2(unit->irte_cnt),
	+ ("IRTA_REG_S overflow %x", unit->irte_cnt));
	+ irta \|= s;
	+ dmar_write8(unit, DMAR_IRTA_REG, irta);
	+ dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd \| DMAR_GCMD_SIRTP);
	+ /* XXXKIB should have a timeout */
	+ while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS) == 0)
	+ cpu_spinwait();
	+ return (0);
	+}
	+
	+int
	+dmar_enable_ir(struct dmar_unit *unit)
	+{
	+
	+ DMAR_ASSERT_LOCKED(unit);
	+ unit->hw_gcmd \|= DMAR_GCMD_IRE;
	+ unit->hw_gcmd &= ~DMAR_GCMD_CFI;
	+ dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	+ /* XXXKIB should have a timeout */
	+ while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) == 0)
	+ cpu_spinwait();
	+ return (0);
	+}
	+
	+int
	+dmar_disable_ir(struct dmar_unit *unit)
	+{
	+
	+ DMAR_ASSERT_LOCKED(unit);
	+ unit->hw_gcmd &= ~DMAR_GCMD_IRE;
	+ dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	+ /* XXXKIB should have a timeout */
	+ while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) != 0)
	+ cpu_spinwait();
	+ return (0);
	+}
	+
	#define BARRIER_F \
	u_int f_done, f_inproc, f_wakeup; \
	\
	Index: sys/x86/iommu/iommu_intrmap.h
	===================================================================
	--- /dev/null
	+++ sys/x86/iommu/iommu_intrmap.h
	@@ -0,0 +1,43 @@
	+/*-
	+ * Copyright (c) 2015 The FreeBSD Foundation
	+ * All rights reserved.
	+ *
	+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	+ * under sponsorship from the FreeBSD Foundation.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#ifndef __X86_IOMMU_IOMMU_INTRMAP_H
	+#define __X86_IOMMU_IOMMU_INTRMAP_H
	+
	+int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count);
	+int iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
	+ uint64_t addr, uint32_t data);
	+int iommu_unmap_msi_intr(device_t src, u_int cookie);
	+int iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
	+ bool activehi, int irq, u_int cookie, uint32_t hi, uint32_t *lo);
	+int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie);
	+
	+#endif
	Index: sys/x86/x86/intr_machdep.c
	===================================================================
	--- sys/x86/x86/intr_machdep.c
	+++ sys/x86/x86/intr_machdep.c
	@@ -423,6 +423,23 @@
	return (0);
	}

	+void
	+intr_reprogram(void)
	+{
	+ struct intsrc *is;
	+ int v;
	+
	+ mtx_lock(&intr_table_lock);
	+ for (v = 0; v < NUM_IO_INTS; v++) {
	+ is = interrupt_sources[v];
	+ if (is == NULL)
	+ continue;
	+ if (is->is_pic->pic_reprogram_pin != NULL)
	+ is->is_pic->pic_reprogram_pin(is);
	+ }
	+ mtx_unlock(&intr_table_lock);
	+}
	+
	#ifdef DDB
	/*
	* Dump data about interrupt handlers
	Index: sys/x86/x86/io_apic.c
	===================================================================
	--- sys/x86/x86/io_apic.c
	+++ sys/x86/x86/io_apic.c
	@@ -27,6 +27,7 @@
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	+#include "opt_acpi.h"
	#include "opt_isa.h"

	#include <sys/param.h>
	@@ -51,17 +52,13 @@
	#include <x86/apicvar.h>
	#include <machine/resource.h>
	#include <machine/segments.h>
	+#include <x86/iommu/iommu_intrmap.h>

	#define IOAPIC_ISA_INTS 16
	#define IOAPIC_MEM_REGION 32
	#define IOAPIC_REDTBL_LO(i) (IOAPIC_REDTBL + (i) * 2)
	#define IOAPIC_REDTBL_HI(i) (IOAPIC_REDTBL_LO(i) + 1)

	-#define IRQ_EXTINT (NUM_IO_INTS + 1)
	-#define IRQ_NMI (NUM_IO_INTS + 2)
	-#define IRQ_SMI (NUM_IO_INTS + 3)
	-#define IRQ_DISABLED (NUM_IO_INTS + 4)
	-
	static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures");

	/*
	@@ -83,12 +80,13 @@
	u_int io_irq;
	u_int io_intpin:8;
	u_int io_vector:8;
	- u_int io_cpu:8;
	+ u_int io_cpu;
	u_int io_activehi:1;
	u_int io_edgetrigger:1;
	u_int io_masked:1;
	int io_bus:4;
	uint32_t io_lowreg;
	+ u_int io_remap_cookie;
	};

	struct ioapic {
	@@ -120,13 +118,23 @@
	static void ioapic_resume(struct pic *pic, bool suspend_cancelled);
	static int ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
	static void ioapic_program_intpin(struct ioapic_intsrc *intpin);
	+static void ioapic_reprogram_intpin(struct intsrc *isrc);

	static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
	-struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
	- ioapic_eoi_source, ioapic_enable_intr,
	- ioapic_disable_intr, ioapic_vector,
	- ioapic_source_pending, NULL, ioapic_resume,
	- ioapic_config_intr, ioapic_assign_cpu };
	+struct pic ioapic_template = {
	+ .pic_enable_source = ioapic_enable_source,
	+ .pic_disable_source = ioapic_disable_source,
	+ .pic_eoi_source = ioapic_eoi_source,
	+ .pic_enable_intr = ioapic_enable_intr,
	+ .pic_disable_intr = ioapic_disable_intr,
	+ .pic_vector = ioapic_vector,
	+ .pic_source_pending = ioapic_source_pending,
	+ .pic_suspend = NULL,
	+ .pic_resume = ioapic_resume,
	+ .pic_config_intr = ioapic_config_intr,
	+ .pic_assign_cpu = ioapic_assign_cpu,
	+ .pic_reprogram_pin = ioapic_reprogram_intpin,
	+};

	static int next_ioapic_base;
	static u_int next_id;
	@@ -295,6 +303,9 @@
	{
	struct ioapic io = (struct ioapic )intpin->io_intsrc.is_pic;
	uint32_t low, high, value;
	+#ifdef ACPI_DMAR
	+ int error;
	+#endif

	/*
	* If a pin is completely invalid or if it is valid but hasn't
	@@ -309,9 +320,34 @@
	ioapic_write(io->io_addr,
	IOAPIC_REDTBL_LO(intpin->io_intpin),
	low \| IOART_INTMSET);
	+#ifdef ACPI_DMAR
	+ mtx_unlock_spin(&icu_lock);
	+ iommu_unmap_ioapic_intr(io->io_apic_id,
	+ &intpin->io_remap_cookie);
	+ mtx_lock_spin(&icu_lock);
	+#endif
	return;
	}

	+#ifdef ACPI_DMAR
	+ mtx_unlock_spin(&icu_lock);
	+ error = iommu_map_ioapic_intr(io->io_apic_id,
	+ intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger,
	+ intpin->io_activehi, intpin->io_irq, &intpin->io_remap_cookie,
	+ &high, &low);
	+ mtx_lock_spin(&icu_lock);
	+ if (error == 0) {
	+ ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin),
	+ high);
	+ intpin->io_lowreg = low;
	+ ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
	+ low);
	+ return;
	+ } else if (error != EOPNOTSUPP) {
	+ return;
	+ }
	+#endif
	+
	/* Set the destination. */
	low = IOART_DESTPHY;
	high = intpin->io_cpu << APIC_ID_SHIFT;
	@@ -358,6 +394,15 @@
	ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low);
	}

	+static void
	+ioapic_reprogram_intpin(struct intsrc *isrc)
	+{
	+
	+ mtx_lock_spin(&icu_lock);
	+ ioapic_program_intpin((struct ioapic_intsrc *)isrc);
	+ mtx_unlock_spin(&icu_lock);
	+}
	+
	static int
	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id)
	{
	@@ -643,6 +688,15 @@
	intpin->io_cpu = PCPU_GET(apic_id);
	value = ioapic_read(apic, IOAPIC_REDTBL_LO(i));
	ioapic_write(apic, IOAPIC_REDTBL_LO(i), value \| IOART_INTMSET);
	+#ifdef ACPI_DMAR
	+ /* dummy, but sets cookie */
	+ mtx_unlock_spin(&icu_lock);
	+ iommu_map_ioapic_intr(io->io_apic_id,
	+ intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger,
	+ intpin->io_activehi, intpin->io_irq,
	+ &intpin->io_remap_cookie, NULL, NULL);
	+ mtx_lock_spin(&icu_lock);
	+#endif
	}
	mtx_unlock_spin(&icu_lock);

	Index: sys/x86/x86/msi.c
	===================================================================
	--- sys/x86/x86/msi.c
	+++ sys/x86/x86/msi.c
	@@ -37,6 +37,8 @@
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	+#include "opt_acpi.h"
	+
	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	@@ -51,6 +53,7 @@
	#include <machine/frame.h>
	#include <machine/intr_machdep.h>
	#include <x86/apicvar.h>
	+#include <x86/iommu/iommu_intrmap.h>
	#include <machine/specialreg.h>
	#include <dev/pci/pcivar.h>

	@@ -111,10 +114,11 @@
	u_int msi_irq; /* IRQ cookie. */
	u_int msi_msix; /* MSI-X message. */
	u_int msi_vector:8; /* IDT vector. */
	- u_int msi_cpu:8; /* Local APIC ID. (g) */
	+ u_int msi_cpu; /* Local APIC ID. (g) */
	u_int msi_count:8; /* Messages in this group. (g) */
	u_int msi_maxcount:8; /* Alignment for this group. (g) */
	int msi_irqs; / Group's IRQ list. (g) */
	+ u_int msi_remap_cookie;
	};

	static void msi_create_source(void);
	@@ -129,10 +133,20 @@
	enum intr_polarity pol);
	static int msi_assign_cpu(struct intsrc *isrc, u_int apic_id);

	-struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
	- msi_enable_intr, msi_disable_intr, msi_vector,
	- msi_source_pending, NULL, NULL, msi_config_intr,
	- msi_assign_cpu };
	+struct pic msi_pic = {
	+ .pic_enable_source = msi_enable_source,
	+ .pic_disable_source = msi_disable_source,
	+ .pic_eoi_source = msi_eoi_source,
	+ .pic_enable_intr = msi_enable_intr,
	+ .pic_disable_intr = msi_disable_intr,
	+ .pic_vector = msi_vector,
	+ .pic_source_pending = msi_source_pending,
	+ .pic_suspend = NULL,
	+ .pic_resume = NULL,
	+ .pic_config_intr = msi_config_intr,
	+ .pic_assign_cpu = msi_assign_cpu,
	+ .pic_reprogram_pin = NULL,
	+};

	static int msi_enabled;
	static int msi_last_irq;
	@@ -320,6 +334,10 @@
	struct msi_intsrc msi, fsrc;
	u_int cpu;
	int cnt, i, *mirqs, vector;
	+#ifdef ACPI_DMAR
	+ u_int cookies[count];
	+ int error;
	+#endif

	if (!msi_enabled)
	return (ENXIO);
	@@ -379,6 +397,24 @@
	return (ENOSPC);
	}

	+#ifdef ACPI_DMAR
	+ mtx_unlock(&msi_lock);
	+ error = iommu_alloc_msi_intr(dev, cookies, count);
	+ mtx_lock(&msi_lock);
	+ if (error == EOPNOTSUPP)
	+ error = 0;
	+ if (error != 0) {
	+ for (i = 0; i < count; i++)
	+ apic_free_vector(cpu, vector + i, irqs[i]);
	+ free(mirqs, M_MSI);
	+ return (error);
	+ }
	+ for (i = 0; i < count; i++) {
	+ msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
	+ msi->msi_remap_cookie = cookies[i];
	+ }
	+#endif
	+
	/* Assign IDT vectors and make these messages owned by 'dev'. */
	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
	for (i = 0; i < count; i++) {
	@@ -400,7 +436,6 @@
	bcopy(irqs, mirqs, count * sizeof(*mirqs));
	fsrc->msi_irqs = mirqs;
	mtx_unlock(&msi_lock);
	-
	return (0);
	}

	@@ -444,6 +479,9 @@
	msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
	KASSERT(msi->msi_first == first, ("message not in group"));
	KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
	+#ifdef ACPI_DMAR
	+ iommu_unmap_msi_intr(first->msi_dev, msi->msi_remap_cookie);
	+#endif
	msi->msi_first = NULL;
	msi->msi_dev = NULL;
	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
	@@ -451,6 +489,11 @@
	}

	/* Clear out the first message. */
	+#ifdef ACPI_DMAR
	+ mtx_unlock(&msi_lock);
	+ iommu_unmap_msi_intr(first->msi_dev, first->msi_remap_cookie);
	+ mtx_lock(&msi_lock);
	+#endif
	first->msi_first = NULL;
	first->msi_dev = NULL;
	apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
	@@ -468,6 +511,11 @@
	msi_map(int irq, uint64_t addr, uint32_t data)
	{
	struct msi_intsrc *msi;
	+ int error;
	+#ifdef ACPI_DMAR
	+ struct msi_intsrc *msi1;
	+ int i, k;
	+#endif

	mtx_lock(&msi_lock);
	msi = (struct msi_intsrc *)intr_lookup_source(irq);
	@@ -495,10 +543,36 @@
	msi = msi->msi_first;
	}

	- *addr = INTEL_ADDR(msi);
	- *data = INTEL_DATA(msi);
	+#ifdef ACPI_DMAR
	+ if (!msi->msi_msix) {
	+ for (k = msi->msi_count - 1, i = FIRST_MSI_INT; k > 0 &&
	+ i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
	+ if (i == msi->msi_irq)
	+ continue;
	+ msi1 = (struct msi_intsrc *)intr_lookup_source(i);
	+ if (!msi1->msi_msix && msi1->msi_first == msi) {
	+ mtx_unlock(&msi_lock);
	+ iommu_map_msi_intr(msi1->msi_dev,
	+ msi1->msi_cpu, msi1->msi_vector,
	+ msi1->msi_remap_cookie, NULL, NULL);
	+ k--;
	+ mtx_lock(&msi_lock);
	+ }
	+ }
	+ }
	mtx_unlock(&msi_lock);
	- return (0);
	+ error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu,
	+ msi->msi_vector, msi->msi_remap_cookie, addr, data);
	+#else
	+ mtx_unlock(&msi_lock);
	+ error = EOPNOTSUPP;
	+#endif
	+ if (error == EOPNOTSUPP) {
	+ *addr = INTEL_ADDR(msi);
	+ *data = INTEL_DATA(msi);
	+ error = 0;
	+ }
	+ return (error);
	}

	int
	@@ -507,6 +581,10 @@
	struct msi_intsrc *msi;
	u_int cpu;
	int i, vector;
	+#ifdef ACPI_DMAR
	+ u_int cookie;
	+ int error;
	+#endif

	if (!msi_enabled)
	return (ENXIO);
	@@ -548,13 +626,28 @@
	mtx_unlock(&msi_lock);
	return (ENOSPC);
	}
	+
	+ msi->msi_dev = dev;
	+#ifdef ACPI_DMAR
	+ mtx_unlock(&msi_lock);
	+ error = iommu_alloc_msi_intr(dev, &cookie, 1);
	+ mtx_lock(&msi_lock);
	+ if (error == EOPNOTSUPP)
	+ error = 0;
	+ if (error != 0) {
	+ msi->msi_dev = NULL;
	+ apic_free_vector(cpu, vector, i);
	+ return (error);
	+ }
	+ msi->msi_remap_cookie = cookie;
	+#endif
	+
	if (bootverbose)
	printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
	msi->msi_irq, cpu, vector);

	/* Setup source. */
	msi->msi_cpu = cpu;
	- msi->msi_dev = dev;
	msi->msi_first = msi;
	msi->msi_vector = vector;
	msi->msi_msix = 1;
	@@ -590,6 +683,11 @@
	KASSERT(msi->msi_dev != NULL, ("unowned message"));

	/* Clear out the message. */
	+#ifdef ACPI_DMAR
	+ mtx_unlock(&msi_lock);
	+ iommu_unmap_msi_intr(msi->msi_dev, msi->msi_remap_cookie);
	+ mtx_lock(&msi_lock);
	+#endif
	msi->msi_first = NULL;
	msi->msi_dev = NULL;
	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);

File Metadata

Mime Type: text/plain
Expires: Sun, Mar 15, 3:03 PM (15 h, 32 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 29720626
Default Alt Text: D1892.diff (48 KB)

D1892.diffNo OneTemporaryActions

D1892.diffView Options

File Metadata

Event Timeline

D1892.diff
No OneTemporary
Actions

D1892.diff
View Options