Index: sys/dev/iommu/busdma_iommu.h =================================================================== --- sys/dev/iommu/busdma_iommu.h +++ sys/dev/iommu/busdma_iommu.h @@ -35,6 +35,7 @@ #define __X86_IOMMU_BUSDMA_DMAR_H #include +#include struct bus_dma_tag_iommu { struct bus_dma_tag_common common; Index: sys/dev/iommu/iommu.h =================================================================== --- sys/dev/iommu/iommu.h +++ sys/dev/iommu/iommu.h @@ -65,18 +65,6 @@ struct iommu_qi_genseq gseq; }; -#define IOMMU_MAP_ENTRY_PLACE 0x0001 /* Fake entry */ -#define IOMMU_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by - dmamap_link */ -#define IOMMU_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by - dmamap_link */ -#define IOMMU_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */ -#define IOMMU_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */ -#define IOMMU_MAP_ENTRY_READ 0x1000 /* Read permitted */ -#define IOMMU_MAP_ENTRY_WRITE 0x2000 /* Write permitted */ -#define IOMMU_MAP_ENTRY_SNOOP 0x4000 /* Snoop */ -#define IOMMU_MAP_ENTRY_TM 0x8000 /* Transient */ - struct iommu_unit { struct mtx lock; int unit; @@ -123,6 +111,9 @@ iommu_gaddr_t end; /* (c) Highest address + 1 in the guest AS */ struct iommu_map_entry *first_place, *last_place; /* (d) */ + struct iommu_map_entry *msi_entry; /* (d) */ + iommu_gaddr_t msi_base; /* (d) */ + vm_paddr_t msi_phys; /* (d) */ u_int flags; /* (u) */ }; @@ -149,17 +140,6 @@ #define IOMMU_DOMAIN_RMRR 0x0020 /* Domain contains RMRR entry, cannot be turned off */ -/* Map flags */ -#define IOMMU_MF_CANWAIT 0x0001 -#define IOMMU_MF_CANSPLIT 0x0002 -#define IOMMU_MF_RMRR 0x0004 - -#define IOMMU_PGF_WAITOK 0x0001 -#define IOMMU_PGF_ZERO 0x0002 -#define IOMMU_PGF_ALLOC 0x0004 -#define IOMMU_PGF_NOALLOC 0x0008 -#define IOMMU_PGF_OBJL 0x0010 - #define IOMMU_LOCK(unit) mtx_lock(&(unit)->lock) #define IOMMU_UNLOCK(unit) mtx_unlock(&(unit)->lock) #define IOMMU_ASSERT_LOCKED(unit) mtx_assert(&(unit)->lock, MA_OWNED) @@ -230,8 +210,6 @@ vm_paddr_t start, vm_size_t length, int flags); bus_dma_tag_t iommu_get_dma_tag(device_t dev, device_t child); -struct iommu_ctx *iommu_get_dev_ctx(device_t dev); -struct iommu_domain *iommu_get_ctx_domain(struct iommu_ctx *ctx); SYSCTL_DECL(_hw_iommu); Index: sys/dev/iommu/iommu_gas.h =================================================================== --- sys/dev/iommu/iommu_gas.h +++ sys/dev/iommu/iommu_gas.h @@ -0,0 +1,60 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_IOMMU_IOMMU_GAS_H_ +#define _DEV_IOMMU_IOMMU_GAS_H_ + +/* Map flags */ +#define IOMMU_MF_CANWAIT 0x0001 +#define IOMMU_MF_CANSPLIT 0x0002 +#define IOMMU_MF_RMRR 0x0004 + +#define IOMMU_PGF_WAITOK 0x0001 +#define IOMMU_PGF_ZERO 0x0002 +#define IOMMU_PGF_ALLOC 0x0004 +#define IOMMU_PGF_NOALLOC 0x0008 +#define IOMMU_PGF_OBJL 0x0010 + +#define IOMMU_MAP_ENTRY_PLACE 0x0001 /* Fake entry */ +#define IOMMU_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by + dmamap_link */ +#define IOMMU_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by + dmamap_link */ +#define IOMMU_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */ +#define IOMMU_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */ +#define IOMMU_MAP_ENTRY_READ 0x1000 /* Read permitted */ +#define IOMMU_MAP_ENTRY_WRITE 0x2000 /* Write permitted */ +#define IOMMU_MAP_ENTRY_SNOOP 0x4000 /* Snoop */ +#define IOMMU_MAP_ENTRY_TM 0x8000 /* Transient */ + +#endif /* !_DEV_IOMMU_IOMMU_GAS_H_ */ Index: sys/dev/iommu/iommu_gas.c =================================================================== --- sys/dev/iommu/iommu_gas.c +++ sys/dev/iommu/iommu_gas.c @@ -62,6 +62,8 @@ #include #include #include +#include +#include #include #include #include @@ -721,6 +723,65 @@ ma, res); return (error); +} + +int +iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, + u_int eflags, u_int flags, vm_page_t *ma) +{ + struct iommu_domain *domain; + struct iommu_map_entry *entry; + int error; + + error = 0; + domain = ctx->domain; + + /* Check if there is already an MSI page allocated */ + IOMMU_DOMAIN_LOCK(domain); + entry = domain->msi_entry; + IOMMU_DOMAIN_UNLOCK(domain); + + if (entry == NULL) { + error = iommu_gas_map(domain, &ctx->tag->common, size, offset, + eflags, flags, ma, &entry); + IOMMU_DOMAIN_LOCK(domain); + if (error == 0) { + if (domain->msi_entry == NULL) { + MPASS(domain->msi_base == 0); + MPASS(domain->msi_phys == 0); + + domain->msi_entry = entry; + domain->msi_base = entry->start; + domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]); + } else { + /* + * We lost the race and already have an + * MSI page allocated. Free the unneeded entry. + */ + iommu_gas_free_entry(domain, entry); + } + } else if (domain->msi_entry != NULL) { + /* + * The allocation failed, but another succeeded. + * Return success as there is a valid MSI page. + */ + error = 0; + } + IOMMU_DOMAIN_UNLOCK(domain); + } + + return (error); +} + +void +iommu_translate_msi(struct iommu_domain *domain, uint64_t *addr) +{ + + KASSERT(*addr >= domain->msi_base, + ("%s: Address is below the MSI base address (%jx < %jx)", __func__, + (uintmax_t)*addr, (uintmax_t)domain->msi_base)); + + *addr = (*addr - domain->msi_phys) + domain->msi_base; } int Index: sys/dev/iommu/iommu_msi.h =================================================================== --- sys/dev/iommu/iommu_msi.h +++ sys/dev/iommu/iommu_msi.h @@ -0,0 +1,49 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_IOMMU_IOMMU_MSI_H_ +#define _DEV_IOMMU_IOMMU_MSI_H_ + +#include + +struct iommu_unit; +struct iommu_domain; +struct iommu_ctx; + +void iommu_translate_msi(struct iommu_domain *domain, uint64_t *addr); +int iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, + u_int eflags, u_int flags, vm_page_t *ma); +struct iommu_domain *iommu_get_ctx_domain(struct iommu_ctx *ctx); +struct iommu_ctx *iommu_get_dev_ctx(device_t dev); + +#endif /* !_DEV_IOMMU_IOMMU_MSI_H_ */ Index: sys/kern/msi_if.m =================================================================== --- sys/kern/msi_if.m +++ sys/kern/msi_if.m @@ -32,6 +32,12 @@ INTERFACE msi; HEADER { + #include "opt_iommu.h" +#ifdef IOMMU + #include + #include +#endif + struct intr_irqsrc; }; @@ -72,3 +78,10 @@ uint32_t *data; }; +#ifdef IOMMU +METHOD int iommu_init { + device_t dev; + device_t child; + struct iommu_domain **domain; +}; +#endif Index: sys/kern/subr_intr.c =================================================================== --- sys/kern/subr_intr.c +++ sys/kern/subr_intr.c @@ -38,6 +38,7 @@ #include "opt_ddb.h" #include "opt_hwpmc_hooks.h" +#include "opt_iommu.h" #include #include @@ -50,6 +51,8 @@ #include #include #include +#include +#include #include #include #include @@ -70,6 +73,10 @@ #include #endif +#ifdef IOMMU +#include +#endif + #include "pic_if.h" #include "msi_if.h" @@ -1290,6 +1297,9 @@ intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, int maxcount, int *irqs) { +#ifdef IOMMU + struct iommu_domain *domain; +#endif struct intr_irqsrc **isrc; struct intr_pic *pic; device_t pdev; @@ -1304,6 +1314,16 @@ ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); +#ifdef IOMMU + /* + * If this is the first time we have used this context ask the + * interrupt controller to map memory the msi source will need. + */ + err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); + if (err != 0) + return (err); +#endif + isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); if (err != 0) { @@ -1312,9 +1332,13 @@ } for (i = 0; i < count; i++) { +#ifdef IOMMU + isrc[i]->isrc_iommu = domain; +#endif msi = (struct intr_map_data_msi *)intr_alloc_map_data( INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); msi-> isrc = isrc[i]; + irqs[i] = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); } @@ -1365,6 +1389,9 @@ int intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) { +#ifdef IOMMU + struct iommu_domain *domain; +#endif struct intr_irqsrc *isrc; struct intr_pic *pic; device_t pdev; @@ -1379,10 +1406,23 @@ ("%s: Found a non-MSI controller: %s", __func__, device_get_name(pic->pic_dev))); +#ifdef IOMMU + /* + * If this is the first time we have used this context ask the + * interrupt controller to map memory the msi source will need. + */ + err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); + if (err != 0) + return (err); +#endif + err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); if (err != 0) return (err); +#ifdef IOMMU + isrc->isrc_iommu = domain; +#endif msi = (struct intr_map_data_msi *)intr_alloc_map_data( INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); msi->isrc = isrc; @@ -1444,6 +1484,12 @@ return (EINVAL); err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); + +#ifdef IOMMU + if (isrc->isrc_iommu != NULL) + iommu_translate_msi(isrc->isrc_iommu, addr); +#endif + return (err); } Index: sys/sys/intr.h =================================================================== --- sys/sys/intr.h +++ sys/sys/intr.h @@ -94,6 +94,8 @@ intr_irq_filter_t * isrc_filter; void * isrc_arg; #endif + /* Used by MSI interrupts to store the iommu details */ + void * isrc_iommu; }; /* Intr interface for PIC. */