Index: sys/amd64/include/intr_machdep.h =================================================================== --- sys/amd64/include/intr_machdep.h +++ sys/amd64/include/intr_machdep.h @@ -106,6 +106,7 @@ int (*pic_config_intr)(struct intsrc *, enum intr_trigger, enum intr_polarity); int (*pic_assign_cpu)(struct intsrc *, u_int apic_id); + void (*pic_reprogram_pin)(struct intsrc *); TAILQ_ENTRY(pic) pics; }; @@ -172,6 +173,7 @@ int intr_remove_handler(void *cookie); void intr_resume(bool suspend_cancelled); void intr_suspend(void); +void intr_reprogram(void); void intrcnt_add(const char *name, u_long **countp); void nexus_add_irq(u_long irq); int msi_alloc(device_t dev, int count, int maxcount, int *irqs); Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -542,6 +542,7 @@ x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci +x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci Index: sys/conf/files.i386 =================================================================== --- sys/conf/files.i386 +++ sys/conf/files.i386 @@ -560,6 +560,7 @@ x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci +x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci Index: sys/i386/include/intr_machdep.h =================================================================== --- sys/i386/include/intr_machdep.h +++ sys/i386/include/intr_machdep.h @@ -112,6 +112,7 @@ int (*pic_config_intr)(struct intsrc *, enum intr_trigger, enum intr_polarity); int (*pic_assign_cpu)(struct intsrc *, u_int apic_id); + void (*pic_reprogram_pin)(struct intsrc *); TAILQ_ENTRY(pic) pics; }; @@ -168,6 +169,7 @@ int intr_remove_handler(void *cookie); void intr_resume(bool suspend_cancelled); void intr_suspend(void); +void intr_reprogram(void); void intrcnt_add(const char *name, u_long **countp); void nexus_add_irq(u_long irq); int msi_alloc(device_t dev, int count, int maxcount, int *irqs); Index: sys/x86/include/apicvar.h =================================================================== --- sys/x86/include/apicvar.h +++ sys/x86/include/apicvar.h @@ -155,6 +155,11 @@ #define APIC_BUS_PCI 2 #define APIC_BUS_MAX APIC_BUS_PCI +#define IRQ_EXTINT (NUM_IO_INTS + 1) +#define IRQ_NMI (NUM_IO_INTS + 2) +#define IRQ_SMI (NUM_IO_INTS + 3) +#define IRQ_DISABLED (NUM_IO_INTS + 4) + /* * An APIC enumerator is a psuedo bus driver that enumerates APIC's including * CPU's and I/O APIC's. Index: sys/x86/iommu/busdma_dmar.c =================================================================== --- sys/x86/iommu/busdma_dmar.c +++ sys/x86/iommu/busdma_dmar.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -92,7 +93,7 @@ * domain, and must collectively be assigned to use either DMAR or * bounce mapping. */ -static device_t +device_t dmar_get_requester(device_t dev, uint16_t *rid) { devclass_t pci_class; @@ -255,6 +256,8 @@ /* Not in scope of any DMAR ? */ if (dmar == NULL) return (NULL); + if (!dmar->dma_enabled) + return (NULL); dmar_quirks_pre_use(dmar); dmar_instantiate_rmrr_ctxs(dmar); @@ -852,6 +855,8 @@ dmar_init_busdma(struct dmar_unit *unit) { + unit->dma_enabled = 1; + TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled); TAILQ_INIT(&unit->delayed_maps); TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit); unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK, Index: sys/x86/iommu/intel_ctx.c =================================================================== --- sys/x86/iommu/intel_ctx.c +++ sys/x86/iommu/intel_ctx.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include Index: sys/x86/iommu/intel_dmar.h =================================================================== --- sys/x86/iommu/intel_dmar.h +++ sys/x86/iommu/intel_dmar.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2013 The FreeBSD Foundation + * Copyright (c) 2013-2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov @@ -185,6 +185,13 @@ u_int inv_seq_waiters; /* count of waiters for seq */ u_int inv_queue_full; /* informational counter */ + /* IR */ + int ir_enabled; + vm_paddr_t irt_phys; + dmar_irte_t *irt; + u_int irte_cnt; + vmem_t *irtids; + /* Delayed freeing of map entries queue processing */ struct dmar_map_entries_tailq tlb_flush_entries; struct task qi_task; @@ -194,6 +201,8 @@ struct task dmamap_load_task; TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps; struct taskqueue *delayed_taskqueue; + + int dma_enabled; }; #define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock) @@ -206,12 +215,16 @@ #define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0) #define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0) +#define DMAR_X2APIC(dmar) \ + (x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0) /* Barrier ids */ #define DMAR_BARRIER_RMRR 0 #define DMAR_BARRIER_USEQ 1 struct dmar_unit *dmar_find(device_t dev); +struct dmar_unit *dmar_find_hpet(device_t dev, uint16_t *rid); +struct dmar_unit *dmar_find_ioapic(u_int apic_id, uint16_t *rid); u_int dmar_nd2mask(u_int nd); bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl); @@ -238,6 +251,9 @@ void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst); int dmar_enable_translation(struct dmar_unit *unit); int dmar_disable_translation(struct dmar_unit *unit); +int dmar_load_irt_ptr(struct dmar_unit *unit); +int dmar_enable_ir(struct dmar_unit *unit); +int dmar_disable_ir(struct dmar_unit *unit); bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id); void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id); @@ -256,6 +272,8 @@ dmar_gaddr_t size, struct dmar_qi_genseq *pseq); void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit); void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit); +void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit); +void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt); vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr); void put_idmap_pgtbl(vm_object_t obj); @@ -282,6 +300,7 @@ int dmar_init_busdma(struct dmar_unit *unit); void dmar_fini_busdma(struct dmar_unit *unit); +device_t dmar_get_requester(device_t dev, uint16_t *rid); void dmar_gas_init_ctx(struct dmar_ctx *ctx); void dmar_gas_fini_ctx(struct dmar_ctx *ctx); @@ -304,6 +323,9 @@ void dmar_quirks_post_ident(struct dmar_unit *dmar); void dmar_quirks_pre_use(struct dmar_unit *dmar); +int dmar_init_irt(struct dmar_unit *unit); +void dmar_fini_irt(struct dmar_unit *unit); + #define DMAR_GM_CANWAIT 0x0001 #define DMAR_GM_CANSPLIT 0x0002 @@ -374,13 +396,16 @@ * containing the P or R and W bits, is set only after the high word * is written. For clear, the P bit is cleared first, then the high * word is cleared. + * + * dmar_pte_update updates the pte. For amd64, the update is atomic. + * For i386, it first disables the entry by clearing the word + * containing the P bit, and then defer to dmar_pte_store. The locked + * cmpxchg8b is probably available on any machine having DMAR support, + * but interrupt translation table may be mapped uncached. */ static inline void -dmar_pte_store(volatile uint64_t *dst, uint64_t val) +dmar_pte_store1(volatile uint64_t *dst, uint64_t val) { - - KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx", - dst, (uintmax_t)*dst, (uintmax_t)val)); #ifdef __i386__ volatile uint32_t *p; uint32_t hi, lo; @@ -396,6 +421,28 @@ } static inline void +dmar_pte_store(volatile uint64_t *dst, uint64_t val) +{ + + KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx", + dst, (uintmax_t)*dst, (uintmax_t)val)); + dmar_pte_store1(dst, val); +} + +static inline void +dmar_pte_update(volatile uint64_t *dst, uint64_t val) +{ + +#ifdef __i386__ + volatile uint32_t *p; + + p = (volatile uint32_t *)dst; + *p = 0; +#endif + dmar_pte_store1(dst, val); +} + +static inline void dmar_pte_clear(volatile uint64_t *dst) { #ifdef __i386__ Index: sys/x86/iommu/intel_drv.c =================================================================== --- sys/x86/iommu/intel_drv.c +++ sys/x86/iommu/intel_drv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2013 The FreeBSD Foundation + * Copyright (c) 2013-2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,7 @@ #include #include #include +#include #include #ifdef DEV_APIC @@ -243,6 +245,7 @@ int i; dmar_fini_busdma(unit); + dmar_fini_irt(unit); dmar_fini_qi(unit); dmar_fini_fault_log(unit); for (i = 0; i < DMAR_INTR_TOTAL; i++) @@ -509,6 +512,11 @@ dmar_release_resources(dev, unit); return (error); } + error = dmar_init_irt(unit); + if (error != 0) { + dmar_release_resources(dev, unit); + return (error); + } error = dmar_init_busdma(unit); if (error != 0) { dmar_release_resources(dev, unit); @@ -763,6 +771,76 @@ return (device_get_softc(dmar_dev)); } +static struct dmar_unit * +dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid) +{ + device_t dmar_dev; + struct dmar_unit *unit; + ACPI_DMAR_HARDWARE_UNIT *dmarh; + ACPI_DMAR_DEVICE_SCOPE *devscope; + ACPI_DMAR_PCI_PATH *path; + char *ptr, *ptrend; + int i; + + for (i = 0; i < dmar_devcnt; i++) { + dmar_dev = dmar_devs[i]; + if (dmar_dev == NULL) + continue; + unit = (struct dmar_unit *)device_get_softc(dmar_dev); + dmarh = dmar_find_by_index(i); + if (dmarh == NULL) + continue; + ptr = (char *)dmarh + sizeof(*dmarh); + ptrend = (char *)dmarh + dmarh->Header.Length; + for (;;) { + if (ptr >= ptrend) + break; + devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr; + ptr += devscope->Length; + if (devscope->EntryType != entry_type) + continue; + if (devscope->EnumerationId != id) + continue; + if (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE) + == 2) { + if (rid != NULL) { + path = (ACPI_DMAR_PCI_PATH *) + (devscope + 1); + *rid = PCI_RID(devscope->Bus, + path->Device, path->Function); + } + return (unit); + } else { + /* XXXKIB */ + printf( + "dmar_find_nonpci: id %d type %d path length != 2\n", + id, entry_type); + } + } + } + return (NULL); +} + + +struct dmar_unit * +dmar_find_hpet(device_t dev, uint16_t *rid) +{ + ACPI_HANDLE handle; + uint32_t hpet_id; + + handle = acpi_get_handle(dev); + if (ACPI_FAILURE(acpi_GetInteger(handle, "_UID", &hpet_id))) + return (NULL); + return (dmar_find_nonpci(hpet_id, ACPI_DMAR_SCOPE_TYPE_HPET, rid)); +} + +struct dmar_unit * +dmar_find_ioapic(u_int apic_id, uint16_t *rid) +{ + + return (dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid)); +} + struct rmrr_iter_args { struct dmar_ctx *ctx; device_t dev; Index: sys/x86/iommu/intel_fault.c =================================================================== --- sys/x86/iommu/intel_fault.c +++ sys/x86/iommu/intel_fault.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include Index: sys/x86/iommu/intel_gas.c =================================================================== --- sys/x86/iommu/intel_gas.c +++ sys/x86/iommu/intel_gas.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include Index: sys/x86/iommu/intel_idpgtbl.c =================================================================== --- sys/x86/iommu/intel_idpgtbl.c +++ sys/x86/iommu/intel_idpgtbl.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include Index: sys/x86/iommu/intel_intrmap.c =================================================================== --- /dev/null +++ sys/x86/iommu/intel_intrmap.c @@ -0,0 +1,380 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct dmar_unit *dmar_ir_find(device_t src, uint16_t *rid, + int *is_dmar); +static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, + uint64_t low, uint16_t rid); +static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie); + +int +iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) +{ + struct dmar_unit *unit; + vmem_addr_t vmem_res; + u_int idx, i; + int error; + + unit = dmar_ir_find(src, NULL, NULL); + if (unit == NULL || !unit->ir_enabled) { + for (i = 0; i < count; i++) + cookies[i] = -1; + return (EOPNOTSUPP); + } + + error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT, + &vmem_res); + if (error != 0) { + KASSERT(error != EOPNOTSUPP, + ("impossible EOPNOTSUPP from vmem")); + return (error); + } + idx = vmem_res; + for (i = 0; i < count; i++) + cookies[i] = idx + i; + return (0); +} + +int +iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, + uint64_t *addr, uint32_t *data) +{ + struct dmar_unit *unit; + uint64_t low; + uint16_t rid; + int is_dmar; + + unit = dmar_ir_find(src, &rid, &is_dmar); + if (is_dmar) { + KASSERT(unit == NULL, ("DMAR cannot translate itself")); + + /* + * See VT-d specification, 5.1.6 Remapping Hardware - + * Interrupt Programming. + */ + *data = vector; + *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); + if (x2apic_mode) + *addr |= ((uint64_t)cpu & 0xffffff00) << 32; + else + KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu)); + return (0); + } + if (unit == NULL || !unit->ir_enabled || cookie == -1) + return (EOPNOTSUPP); + + low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : + DMAR_IRTE1_DST_xAPIC(cpu)) | DMAR_IRTE1_V(vector) | + DMAR_IRTE1_DLM_FM | DMAR_IRTE1_TM_EDGE | DMAR_IRTE1_RH_DIRECT | + DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; + dmar_ir_program_irte(unit, cookie, low, rid); + + if (addr != NULL) { + /* + * See VT-d specification, 5.1.5.2 MSI and MSI-X + * Register Programming. + */ + *addr = MSI_INTEL_ADDR_BASE | ((cookie & 0x7fff) << 5) | + ((cookie & 0x8000) << 2) | 0x18; + *data = 0; + } + return (0); +} + +int +iommu_unmap_msi_intr(device_t src, u_int cookie) +{ + struct dmar_unit *unit; + + if (cookie == -1) + return (0); + unit = dmar_ir_find(src, NULL, NULL); + return (dmar_ir_free_irte(unit, cookie)); +} + +int +iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, + bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo) +{ + struct dmar_unit *unit; + vmem_addr_t vmem_res; + uint64_t low, iorte; + u_int idx; + int error; + uint16_t rid; + + unit = dmar_find_ioapic(ioapic_id, &rid); + if (unit == NULL || !unit->ir_enabled) { + *cookie = -1; + return (EOPNOTSUPP); + } + + error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res); + if (error != 0) { + KASSERT(error != EOPNOTSUPP, + ("impossible EOPNOTSUPP from vmem")); + return (error); + } + idx = vmem_res; + low = 0; + switch (irq) { + case IRQ_EXTINT: + low |= DMAR_IRTE1_DLM_ExtINT; + break; + case IRQ_NMI: + low |= DMAR_IRTE1_DLM_NMI; + break; + case IRQ_SMI: + low |= DMAR_IRTE1_DLM_SMI; + break; + default: + KASSERT(vector != 0, ("No vector for IRQ %u", irq)); + low |= DMAR_IRTE1_DLM_FM | DMAR_IRTE1_V(vector); + break; + } + low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : + DMAR_IRTE1_DST_xAPIC(cpu)) | + (edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) | + DMAR_IRTE1_RH_DIRECT | DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; + dmar_ir_program_irte(unit, idx, low, rid); + + if (hi != NULL) { + /* + * See VT-d specification, 5.1.5.1 I/OxAPIC + * Programming. + */ + iorte = (1ULL << 48) | ((uint64_t)(idx & 0x7fff) << 49) | + ((idx & 0x8000) != 0 ? (1 << 11) : 0) | + (edge ? IOART_TRGREDG : IOART_TRGRLVL) | + (activehi ? IOART_INTAHI : IOART_INTALO) | + IOART_DELFIXED | vector; + *hi = iorte >> 32; + *lo = iorte; + } + *cookie = idx; + return (0); +} + +int +iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) +{ + struct dmar_unit *unit; + u_int idx; + + idx = *cookie; + if (idx == -1) + return (0); + *cookie = -1; + unit = dmar_find_ioapic(ioapic_id, NULL); + KASSERT(unit != NULL && unit->ir_enabled, + ("unmap: cookie %d unit %p", idx, unit)); + return (dmar_ir_free_irte(unit, idx)); +} + +static struct dmar_unit * +dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar) +{ + devclass_t src_class; + struct dmar_unit *unit; + + /* + * We need to determine if the interrupt source generates FSB + * interrupts. If yes, it is either DMAR, in which case + * interrupts are not remapped. Or it is HPET, and interrupts + * are remapped. For HPET, source id is reported by HPET + * record in DMAR ACPI table. + */ + if (is_dmar != NULL) + *is_dmar = FALSE; + src_class = device_get_devclass(src); + if (src_class == devclass_find("dmar")) { + unit = NULL; + if (is_dmar != NULL) + *is_dmar = TRUE; + } else if (src_class == devclass_find("hpet")) { + unit = dmar_find_hpet(src, rid); + } else { + unit = dmar_find(src); + if (unit != NULL && rid != NULL) + dmar_get_requester(src, rid); + } + return (unit); +} + +static void +dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low, + uint16_t rid) +{ + dmar_irte_t *irte; + uint64_t high; + + KASSERT(idx < unit->irte_cnt, + ("bad cookie %d %d", idx, unit->irte_cnt)); + irte = &(unit->irt[idx]); + high = DMAR_IRTE2_SVT_RID | DMAR_IRTE2_SQ_RID | + DMAR_IRTE2_SID_RID(rid); + device_printf(unit->dev, + "programming irte[%d] rid %#x high %#jx low %#jx\n", + idx, rid, (uintmax_t)high, (uintmax_t)low); + DMAR_LOCK(unit); + if ((irte->irte1 & DMAR_IRTE1_P) != 0) { + /* + * The rte is already valid. Assume that the request + * is to remap the interrupt for balancing. Only low + * word of rte needs to be changed. Assert that the + * high word contains expected value. + */ + KASSERT(irte->irte2 == high, + ("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2, + (uintmax_t)high)); + dmar_pte_update(&irte->irte1, low); + } else { + dmar_pte_store(&irte->irte2, high); + dmar_pte_store(&irte->irte1, low); + } + dmar_qi_invalidate_iec(unit, idx, 1); + DMAR_UNLOCK(unit); + +} + +static int +dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie) +{ + dmar_irte_t *irte; + + KASSERT(unit != NULL && unit->ir_enabled, + ("unmap: cookie %d unit %p", cookie, unit)); + KASSERT(cookie < unit->irte_cnt, + ("bad cookie %u %u", cookie, unit->irte_cnt)); + irte = &(unit->irt[cookie]); + dmar_pte_clear(&irte->irte1); + dmar_pte_clear(&irte->irte2); + DMAR_LOCK(unit); + dmar_qi_invalidate_iec(unit, cookie, 1); + DMAR_UNLOCK(unit); + vmem_free(unit->irtids, cookie, 1); + return (0); +} + +static u_int +clp2(u_int v) +{ + + return (powerof2(v) ? v : 1 << fls(v)); +} + +int +dmar_init_irt(struct dmar_unit *unit) +{ + + if ((unit->hw_ecap & DMAR_ECAP_IR) == 0) + return (0); + unit->ir_enabled = 1; + TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled); + if (!unit->ir_enabled) + return (0); + if (!unit->qi_enabled) { + unit->ir_enabled = 0; + if (bootverbose) + device_printf(unit->dev, + "QI disabled, disabling interrupt remapping\n"); + return (0); + } + unit->irte_cnt = clp2(NUM_IO_INTS); + unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig(kernel_arena, + unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO | M_WAITOK, 0, + dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ? + VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE); + if (unit->irt == NULL) + return (ENOMEM); + unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt); + unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0, + M_FIRSTFIT | M_NOWAIT); + DMAR_LOCK(unit); + dmar_load_irt_ptr(unit); + dmar_qi_invalidate_iec_glob(unit); + DMAR_UNLOCK(unit); + + /* + * Initialize mappings for already configured interrupt pins. + * Required, because otherwise the interrupts fault without + * irtes. + */ + intr_reprogram(); + + DMAR_LOCK(unit); + dmar_enable_ir(unit); + DMAR_UNLOCK(unit); + return (0); +} + +void +dmar_fini_irt(struct dmar_unit *unit) +{ + + unit->ir_enabled = 0; + if (unit->irt != NULL) { + dmar_disable_ir(unit); + dmar_qi_invalidate_iec_glob(unit); + vmem_destroy(unit->irtids); + kmem_free(kernel_arena, (vm_offset_t)unit->irt, + unit->irte_cnt * sizeof(dmar_irte_t)); + } +} Index: sys/x86/iommu/intel_qi.c =================================================================== --- sys/x86/iommu/intel_qi.c +++ sys/x86/iommu/intel_qi.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -194,13 +195,14 @@ } static void -dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq) +dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, + bool nowait) { DMAR_ASSERT_LOCKED(unit); unit->inv_seq_waiters++; while (!dmar_qi_seq_processed(unit, gseq)) { - if (cold) { + if (cold || nowait) { cpu_spinwait(); } else { msleep(&unit->inv_seq_waiters, &unit->lock, 0, @@ -246,7 +248,7 @@ dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); dmar_qi_emit_wait_seq(unit, &gseq); dmar_qi_advance_tail(unit); - dmar_qi_wait_for_seq(unit, &gseq); + dmar_qi_wait_for_seq(unit, &gseq, false); } void @@ -260,7 +262,60 @@ DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); dmar_qi_emit_wait_seq(unit, &gseq); dmar_qi_advance_tail(unit); - dmar_qi_wait_for_seq(unit, &gseq); + dmar_qi_wait_for_seq(unit, &gseq, false); +} + +void +dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) +{ + struct dmar_qi_genseq gseq; + + DMAR_ASSERT_LOCKED(unit); + dmar_qi_ensure(unit, 2); + dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); + dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_advance_tail(unit); + dmar_qi_wait_for_seq(unit, &gseq, false); +} + +void +dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) +{ + struct dmar_qi_genseq gseq; + u_int c, l; + + DMAR_ASSERT_LOCKED(unit); + KASSERT(start < unit->irte_cnt && start < start + cnt && + start + cnt <= unit->irte_cnt, + ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); + for (; cnt > 0; cnt -= c, start += c) { + l = ffs(start | cnt) - 1; + c = 1 << l; + dmar_qi_ensure(unit, 1); + dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | + DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | + DMAR_IQ_DESCR_IEC_IM(l), 0); + } + dmar_qi_ensure(unit, 1); + dmar_qi_emit_wait_seq(unit, &gseq); + dmar_qi_advance_tail(unit); + + /* + * The caller of the function, in particular, + * dmar_ir_program_irte(), may be called from the context + * where the sleeping is forbidden (in fact, the + * intr_table_lock mutex may be held, locked from + * intr_shuffle_irqs()). Wait for the invalidation completion + * using the busy wait. + * + * The impact on the interrupt input setup code is small, the + * expected overhead is comparable with the chipset register + * read. It is more harmful for the parallel DMA operations, + * since we own the dmar unit lock until whole invalidation + * queue is processed, which includes requests possibly issued + * before our request. + */ + dmar_qi_wait_for_seq(unit, &gseq, true); } int @@ -377,7 +432,7 @@ dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_seq(unit, &gseq); dmar_qi_advance_tail(unit); - dmar_qi_wait_for_seq(unit, &gseq); + dmar_qi_wait_for_seq(unit, &gseq, false); /* only after the quisce, disable queue */ dmar_disable_qi(unit); KASSERT(unit->inv_seq_waiters == 0, Index: sys/x86/iommu/intel_quirks.c =================================================================== --- sys/x86/iommu/intel_quirks.c +++ sys/x86/iommu/intel_quirks.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2013 The FreeBSD Foundation + * Copyright (c) 2013, 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -59,7 +60,7 @@ #include #include -typedef void (*dmar_quirk_fun)(struct dmar_unit *); +typedef void (*dmar_quirk_cpu_fun)(struct dmar_unit *); struct intel_dmar_quirk_cpu { u_int ext_family; @@ -67,17 +68,21 @@ u_int family_code; u_int model; u_int stepping; - dmar_quirk_fun quirk; + dmar_quirk_cpu_fun quirk; const char *descr; }; +typedef void (*dmar_quirk_nb_fun)(struct dmar_unit *, device_t nb); + struct intel_dmar_quirk_nb { u_int dev_id; u_int rev_no; - dmar_quirk_fun quirk; + dmar_quirk_nb_fun quirk; const char *descr; }; +#define QUIRK_NB_ALL_REV 0xffffffff + static void dmar_match_quirks(struct dmar_unit *dmar, const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len, @@ -99,13 +104,14 @@ for (i = 0; i < nb_quirks_len; i++) { nb_quirk = &nb_quirks[i]; if (nb_quirk->dev_id == dev_id && - nb_quirk->rev_no == rev_no) { + (nb_quirk->rev_no == rev_no || + nb_quirk->rev_no == QUIRK_NB_ALL_REV)) { if (bootverbose) { device_printf(dmar->dev, "NB IOMMU quirk %s\n", nb_quirk->descr); } - nb_quirk->quirk(dmar); + nb_quirk->quirk(dmar, nb); } } } else { @@ -139,12 +145,29 @@ } static void -nb_5400_no_low_high_prot_mem(struct dmar_unit *unit) +nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused) { unit->hw_cap &= ~(DMAR_CAP_PHMR | DMAR_CAP_PLMR); } +static void +nb_no_ir(struct dmar_unit *unit, device_t nb __unused) +{ + + unit->hw_ecap &= ~(DMAR_ECAP_IR | DMAR_ECAP_EIM); +} + +static void +nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb) +{ + u_int rev_no; + + rev_no = pci_get_revid(nb); + if (rev_no <= 0x13) + nb_no_ir(unit, nb); +} + static const struct intel_dmar_quirk_nb pre_use_nb[] = { { .dev_id = 0x4001, .rev_no = 0x20, @@ -156,6 +179,26 @@ .quirk = nb_5400_no_low_high_prot_mem, .descr = "5400 E23" /* no low/high protected memory */ }, + { + .dev_id = 0x3403, .rev_no = QUIRK_NB_ALL_REV, + .quirk = nb_5500_no_ir_rev13, + .descr = "5500 E47, E53" /* interrupt remapping does not work */ + }, + { + .dev_id = 0x3405, .rev_no = QUIRK_NB_ALL_REV, + .quirk = nb_5500_no_ir_rev13, + .descr = "5500 E47, E53" /* interrupt remapping does not work */ + }, + { + .dev_id = 0x3405, .rev_no = 0x22, + .quirk = nb_no_ir, + .descr = "5500 E47, E53" /* interrupt remapping does not work */ + }, + { + .dev_id = 0x3406, .rev_no = QUIRK_NB_ALL_REV, + .quirk = nb_5500_no_ir_rev13, + .descr = "5500 E47, E53" /* interrupt remapping does not work */ + }, }; static void Index: sys/x86/iommu/intel_reg.h =================================================================== --- sys/x86/iommu/intel_reg.h +++ sys/x86/iommu/intel_reg.h @@ -103,7 +103,7 @@ #define DMAR_IRTE1_IM_POSTED (1ULL << 15) /* Posted */ /* Delivery Mode */ #define DMAR_IRTE1_DLM_FM (0ULL << 5) -#define DMAR_IRTE1_DLM_LP (1ULL << 5 +#define DMAR_IRTE1_DLM_LP (1ULL << 5) #define DMAR_IRTE1_DLM_SMI (2ULL << 5) #define DMAR_IRTE1_DLM_NMI (4ULL << 5) #define DMAR_IRTE1_DLM_INIT (5ULL << 5) @@ -180,7 +180,7 @@ /* IOTLB Register Offset */ #define DMAR_ECAP_SC (1 << 7) /* Snoop Control */ #define DMAR_ECAP_PT (1 << 6) /* Pass Through */ -#define DMAR_ECAP_EIM (1 << 4) /* Extended Interrupt Mode */ +#define DMAR_ECAP_EIM (1 << 4) /* Extended Interrupt Mode (x2APIC) */ #define DMAR_ECAP_IR (1 << 3) /* Interrupt Remapping */ #define DMAR_ECAP_DI (1 << 2) /* Device IOTLB */ #define DMAR_ECAP_QI (1 << 1) /* Queued Invalidation */ @@ -320,8 +320,8 @@ #define DMAR_IQ_DESCR_SZ (1 << DMAR_IQ_DESCR_SZ_SHIFT) /* Descriptor size */ -#define DMAR_IQ_DESCR_CTX_INV 0x1 /* Context-cache Invalidate - Descriptor */ +/* Context-cache Invalidate Descriptor */ +#define DMAR_IQ_DESCR_CTX_INV 0x1 #define DMAR_IQ_DESCR_CTX_GLOB (0x1 << 4) /* Granularity: Global */ #define DMAR_IQ_DESCR_CTX_DOM (0x2 << 4) /* Granularity: Domain */ #define DMAR_IQ_DESCR_CTX_DEV (0x3 << 4) /* Granularity: Device */ @@ -329,7 +329,8 @@ #define DMAR_IQ_DESCR_CTX_SRC(x) (((uint64_t)(x)) << 32) /* Source Id */ #define DMAR_IQ_DESCR_CTX_FM(x) (((uint64_t)(x)) << 48) /* Function Mask */ -#define DMAR_IQ_DESCR_IOTLB_INV 0x2 /* IOTLB Invalidate Descriptor */ +/* IOTLB Invalidate Descriptor */ +#define DMAR_IQ_DESCR_IOTLB_INV 0x2 #define DMAR_IQ_DESCR_IOTLB_GLOB (0x1 << 4) /* Granularity: Global */ #define DMAR_IQ_DESCR_IOTLB_DOM (0x2 << 4) /* Granularity: Domain */ #define DMAR_IQ_DESCR_IOTLB_PAGE (0x3 << 4) /* Granularity: Page */ @@ -337,17 +338,31 @@ #define DMAR_IQ_DESCR_IOTLB_DR (1 << 7) /* Drain Reads */ #define DMAR_IQ_DESCR_IOTLB_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */ -#define DMAR_IQ_DESCR_IEC_INV 0x4 /* Invalidate Interrupt Entry Cache */ +/* Device-TLB Invalidate Descriptor */ +#define DMAR_IQ_DESCR_DTLB_INV 0x3 + +/* Invalidate Interrupt Entry Cache */ +#define DMAR_IQ_DESCR_IEC_INV 0x4 #define DMAR_IQ_DESCR_IEC_IDX (1 << 4) /* Index-Selective Invalidation */ #define DMAR_IQ_DESCR_IEC_IIDX(x) (((uint64_t)x) << 32) /* Interrupt Index */ #define DMAR_IQ_DESCR_IEC_IM(x) ((x) << 27) /* Index Mask */ -#define DMAR_IQ_DESCR_WAIT_ID 0x5 /* Invalidation Wait Descriptor */ +/* Invalidation Wait Descriptor */ +#define DMAR_IQ_DESCR_WAIT_ID 0x5 #define DMAR_IQ_DESCR_WAIT_IF (1 << 4) /* Interrupt Flag */ #define DMAR_IQ_DESCR_WAIT_SW (1 << 5) /* Status Write */ #define DMAR_IQ_DESCR_WAIT_FN (1 << 6) /* Fence */ #define DMAR_IQ_DESCR_WAIT_SD(x) (((uint64_t)(x)) << 32) /* Status Data */ +/* Extended IOTLB Invalidate Descriptor */ +#define DMAR_IQ_DESCR_EIOTLB_INV 0x6 + +/* PASID-Cache Invalidate Descriptor */ +#define DMAR_IQ_DESCR_PASIDC_INV 0x7 + +/* Extended Device-TLB Invalidate Descriptor */ +#define DMAR_IQ_DESCR_EDTLB_INV 0x8 + /* Invalidation Queue Head register */ #define DMAR_IQH_REG 0x80 #define DMAR_IQH_MASK 0x7fff0 /* Next cmd index mask */ Index: sys/x86/iommu/intel_utils.c =================================================================== --- sys/x86/iommu/intel_utils.c +++ sys/x86/iommu/intel_utils.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,8 @@ #include #include #include +#include +#include #include #include #include @@ -521,6 +524,55 @@ return (0); } +int +dmar_load_irt_ptr(struct dmar_unit *unit) +{ + uint64_t irta, s; + + DMAR_ASSERT_LOCKED(unit); + irta = unit->irt_phys; + if (DMAR_X2APIC(unit)) + irta |= DMAR_IRTA_EIME; + s = fls(unit->irte_cnt) - 2; + KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK && + powerof2(unit->irte_cnt), + ("IRTA_REG_S overflow %x", unit->irte_cnt)); + irta |= s; + dmar_write8(unit, DMAR_IRTA_REG, irta); + dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SIRTP); + /* XXXKIB should have a timeout */ + while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS) == 0) + cpu_spinwait(); + return (0); +} + +int +dmar_enable_ir(struct dmar_unit *unit) +{ + + DMAR_ASSERT_LOCKED(unit); + unit->hw_gcmd |= DMAR_GCMD_IRE; + unit->hw_gcmd &= ~DMAR_GCMD_CFI; + dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); + /* XXXKIB should have a timeout */ + while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) == 0) + cpu_spinwait(); + return (0); +} + +int +dmar_disable_ir(struct dmar_unit *unit) +{ + + DMAR_ASSERT_LOCKED(unit); + unit->hw_gcmd &= ~DMAR_GCMD_IRE; + dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); + /* XXXKIB should have a timeout */ + while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) != 0) + cpu_spinwait(); + return (0); +} + #define BARRIER_F \ u_int f_done, f_inproc, f_wakeup; \ \ Index: sys/x86/iommu/iommu_intrmap.h =================================================================== --- /dev/null +++ sys/x86/iommu/iommu_intrmap.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __X86_IOMMU_IOMMU_INTRMAP_H +#define __X86_IOMMU_IOMMU_INTRMAP_H + +int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count); +int iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, + uint64_t *addr, uint32_t *data); +int iommu_unmap_msi_intr(device_t src, u_int cookie); +int iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, + bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo); +int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie); + +#endif Index: sys/x86/x86/intr_machdep.c =================================================================== --- sys/x86/x86/intr_machdep.c +++ sys/x86/x86/intr_machdep.c @@ -423,6 +423,23 @@ return (0); } +void +intr_reprogram(void) +{ + struct intsrc *is; + int v; + + mtx_lock(&intr_table_lock); + for (v = 0; v < NUM_IO_INTS; v++) { + is = interrupt_sources[v]; + if (is == NULL) + continue; + if (is->is_pic->pic_reprogram_pin != NULL) + is->is_pic->pic_reprogram_pin(is); + } + mtx_unlock(&intr_table_lock); +} + #ifdef DDB /* * Dump data about interrupt handlers Index: sys/x86/x86/io_apic.c =================================================================== --- sys/x86/x86/io_apic.c +++ sys/x86/x86/io_apic.c @@ -27,6 +27,7 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_acpi.h" #include "opt_isa.h" #include @@ -51,17 +52,13 @@ #include #include #include +#include #define IOAPIC_ISA_INTS 16 #define IOAPIC_MEM_REGION 32 #define IOAPIC_REDTBL_LO(i) (IOAPIC_REDTBL + (i) * 2) #define IOAPIC_REDTBL_HI(i) (IOAPIC_REDTBL_LO(i) + 1) -#define IRQ_EXTINT (NUM_IO_INTS + 1) -#define IRQ_NMI (NUM_IO_INTS + 2) -#define IRQ_SMI (NUM_IO_INTS + 3) -#define IRQ_DISABLED (NUM_IO_INTS + 4) - static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures"); /* @@ -83,12 +80,13 @@ u_int io_irq; u_int io_intpin:8; u_int io_vector:8; - u_int io_cpu:8; + u_int io_cpu; u_int io_activehi:1; u_int io_edgetrigger:1; u_int io_masked:1; int io_bus:4; uint32_t io_lowreg; + u_int io_remap_cookie; }; struct ioapic { @@ -120,13 +118,23 @@ static void ioapic_resume(struct pic *pic, bool suspend_cancelled); static int ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id); static void ioapic_program_intpin(struct ioapic_intsrc *intpin); +static void ioapic_reprogram_intpin(struct intsrc *isrc); static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list); -struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source, - ioapic_eoi_source, ioapic_enable_intr, - ioapic_disable_intr, ioapic_vector, - ioapic_source_pending, NULL, ioapic_resume, - ioapic_config_intr, ioapic_assign_cpu }; +struct pic ioapic_template = { + .pic_enable_source = ioapic_enable_source, + .pic_disable_source = ioapic_disable_source, + .pic_eoi_source = ioapic_eoi_source, + .pic_enable_intr = ioapic_enable_intr, + .pic_disable_intr = ioapic_disable_intr, + .pic_vector = ioapic_vector, + .pic_source_pending = ioapic_source_pending, + .pic_suspend = NULL, + .pic_resume = ioapic_resume, + .pic_config_intr = ioapic_config_intr, + .pic_assign_cpu = ioapic_assign_cpu, + .pic_reprogram_pin = ioapic_reprogram_intpin, +}; static int next_ioapic_base; static u_int next_id; @@ -295,6 +303,9 @@ { struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; uint32_t low, high, value; +#ifdef ACPI_DMAR + int error; +#endif /* * If a pin is completely invalid or if it is valid but hasn't @@ -309,9 +320,34 @@ ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low | IOART_INTMSET); +#ifdef ACPI_DMAR + mtx_unlock_spin(&icu_lock); + iommu_unmap_ioapic_intr(io->io_apic_id, + &intpin->io_remap_cookie); + mtx_lock_spin(&icu_lock); +#endif return; } +#ifdef ACPI_DMAR + mtx_unlock_spin(&icu_lock); + error = iommu_map_ioapic_intr(io->io_apic_id, + intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger, + intpin->io_activehi, intpin->io_irq, &intpin->io_remap_cookie, + &high, &low); + mtx_lock_spin(&icu_lock); + if (error == 0) { + ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), + high); + intpin->io_lowreg = low; + ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), + low); + return; + } else if (error != EOPNOTSUPP) { + return; + } +#endif + /* Set the destination. */ low = IOART_DESTPHY; high = intpin->io_cpu << APIC_ID_SHIFT; @@ -358,6 +394,15 @@ ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low); } +static void +ioapic_reprogram_intpin(struct intsrc *isrc) +{ + + mtx_lock_spin(&icu_lock); + ioapic_program_intpin((struct ioapic_intsrc *)isrc); + mtx_unlock_spin(&icu_lock); +} + static int ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id) { @@ -643,6 +688,15 @@ intpin->io_cpu = PCPU_GET(apic_id); value = ioapic_read(apic, IOAPIC_REDTBL_LO(i)); ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET); +#ifdef ACPI_DMAR + /* dummy, but sets cookie */ + mtx_unlock_spin(&icu_lock); + iommu_map_ioapic_intr(io->io_apic_id, + intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger, + intpin->io_activehi, intpin->io_irq, + &intpin->io_remap_cookie, NULL, NULL); + mtx_lock_spin(&icu_lock); +#endif } mtx_unlock_spin(&icu_lock); Index: sys/x86/x86/msi.c =================================================================== --- sys/x86/x86/msi.c +++ sys/x86/x86/msi.c @@ -37,6 +37,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_acpi.h" + #include #include #include @@ -51,6 +53,7 @@ #include #include #include +#include #include #include @@ -111,10 +114,11 @@ u_int msi_irq; /* IRQ cookie. */ u_int msi_msix; /* MSI-X message. */ u_int msi_vector:8; /* IDT vector. */ - u_int msi_cpu:8; /* Local APIC ID. (g) */ + u_int msi_cpu; /* Local APIC ID. (g) */ u_int msi_count:8; /* Messages in this group. (g) */ u_int msi_maxcount:8; /* Alignment for this group. (g) */ int *msi_irqs; /* Group's IRQ list. (g) */ + u_int msi_remap_cookie; }; static void msi_create_source(void); @@ -129,10 +133,20 @@ enum intr_polarity pol); static int msi_assign_cpu(struct intsrc *isrc, u_int apic_id); -struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source, - msi_enable_intr, msi_disable_intr, msi_vector, - msi_source_pending, NULL, NULL, msi_config_intr, - msi_assign_cpu }; +struct pic msi_pic = { + .pic_enable_source = msi_enable_source, + .pic_disable_source = msi_disable_source, + .pic_eoi_source = msi_eoi_source, + .pic_enable_intr = msi_enable_intr, + .pic_disable_intr = msi_disable_intr, + .pic_vector = msi_vector, + .pic_source_pending = msi_source_pending, + .pic_suspend = NULL, + .pic_resume = NULL, + .pic_config_intr = msi_config_intr, + .pic_assign_cpu = msi_assign_cpu, + .pic_reprogram_pin = NULL, +}; static int msi_enabled; static int msi_last_irq; @@ -320,6 +334,10 @@ struct msi_intsrc *msi, *fsrc; u_int cpu; int cnt, i, *mirqs, vector; +#ifdef ACPI_DMAR + u_int cookies[count]; + int error; +#endif if (!msi_enabled) return (ENXIO); @@ -379,6 +397,24 @@ return (ENOSPC); } +#ifdef ACPI_DMAR + mtx_unlock(&msi_lock); + error = iommu_alloc_msi_intr(dev, cookies, count); + mtx_lock(&msi_lock); + if (error == EOPNOTSUPP) + error = 0; + if (error != 0) { + for (i = 0; i < count; i++) + apic_free_vector(cpu, vector + i, irqs[i]); + free(mirqs, M_MSI); + return (error); + } + for (i = 0; i < count; i++) { + msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); + msi->msi_remap_cookie = cookies[i]; + } +#endif + /* Assign IDT vectors and make these messages owned by 'dev'. */ fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); for (i = 0; i < count; i++) { @@ -400,7 +436,6 @@ bcopy(irqs, mirqs, count * sizeof(*mirqs)); fsrc->msi_irqs = mirqs; mtx_unlock(&msi_lock); - return (0); } @@ -444,6 +479,9 @@ msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); KASSERT(msi->msi_first == first, ("message not in group")); KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); +#ifdef ACPI_DMAR + iommu_unmap_msi_intr(first->msi_dev, msi->msi_remap_cookie); +#endif msi->msi_first = NULL; msi->msi_dev = NULL; apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); @@ -451,6 +489,11 @@ } /* Clear out the first message. */ +#ifdef ACPI_DMAR + mtx_unlock(&msi_lock); + iommu_unmap_msi_intr(first->msi_dev, first->msi_remap_cookie); + mtx_lock(&msi_lock); +#endif first->msi_first = NULL; first->msi_dev = NULL; apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq); @@ -468,6 +511,11 @@ msi_map(int irq, uint64_t *addr, uint32_t *data) { struct msi_intsrc *msi; + int error; +#ifdef ACPI_DMAR + struct msi_intsrc *msi1; + int i, k; +#endif mtx_lock(&msi_lock); msi = (struct msi_intsrc *)intr_lookup_source(irq); @@ -495,10 +543,36 @@ msi = msi->msi_first; } - *addr = INTEL_ADDR(msi); - *data = INTEL_DATA(msi); +#ifdef ACPI_DMAR + if (!msi->msi_msix) { + for (k = msi->msi_count - 1, i = FIRST_MSI_INT; k > 0 && + i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { + if (i == msi->msi_irq) + continue; + msi1 = (struct msi_intsrc *)intr_lookup_source(i); + if (!msi1->msi_msix && msi1->msi_first == msi) { + mtx_unlock(&msi_lock); + iommu_map_msi_intr(msi1->msi_dev, + msi1->msi_cpu, msi1->msi_vector, + msi1->msi_remap_cookie, NULL, NULL); + k--; + mtx_lock(&msi_lock); + } + } + } mtx_unlock(&msi_lock); - return (0); + error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu, + msi->msi_vector, msi->msi_remap_cookie, addr, data); +#else + mtx_unlock(&msi_lock); + error = EOPNOTSUPP; +#endif + if (error == EOPNOTSUPP) { + *addr = INTEL_ADDR(msi); + *data = INTEL_DATA(msi); + error = 0; + } + return (error); } int @@ -507,6 +581,10 @@ struct msi_intsrc *msi; u_int cpu; int i, vector; +#ifdef ACPI_DMAR + u_int cookie; + int error; +#endif if (!msi_enabled) return (ENXIO); @@ -548,13 +626,28 @@ mtx_unlock(&msi_lock); return (ENOSPC); } + + msi->msi_dev = dev; +#ifdef ACPI_DMAR + mtx_unlock(&msi_lock); + error = iommu_alloc_msi_intr(dev, &cookie, 1); + mtx_lock(&msi_lock); + if (error == EOPNOTSUPP) + error = 0; + if (error != 0) { + msi->msi_dev = NULL; + apic_free_vector(cpu, vector, i); + return (error); + } + msi->msi_remap_cookie = cookie; +#endif + if (bootverbose) printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n", msi->msi_irq, cpu, vector); /* Setup source. */ msi->msi_cpu = cpu; - msi->msi_dev = dev; msi->msi_first = msi; msi->msi_vector = vector; msi->msi_msix = 1; @@ -590,6 +683,11 @@ KASSERT(msi->msi_dev != NULL, ("unowned message")); /* Clear out the message. */ +#ifdef ACPI_DMAR + mtx_unlock(&msi_lock); + iommu_unmap_msi_intr(msi->msi_dev, msi->msi_remap_cookie); + mtx_lock(&msi_lock); +#endif msi->msi_first = NULL; msi->msi_dev = NULL; apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);