Index: sys/conf/files.x86 =================================================================== --- sys/conf/files.x86 +++ sys/conf/files.x86 @@ -165,6 +165,7 @@ dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io +dev/iommu/busdma_iommu.c optional acpi acpi_dmar pci dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa Index: sys/dev/iommu/busdma_iommu.h =================================================================== --- /dev/null +++ sys/dev/iommu/busdma_iommu.h @@ -0,0 +1,91 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_IOMMU_BUSDMA_IOMMU_H_ +#define _DEV_IOMMU_BUSDMA_IOMMU_H_ + +struct iommu_map_entry; +TAILQ_HEAD(iommu_map_entries_tailq, iommu_map_entry); + +struct bus_dma_tag_iommu { + struct bus_dma_tag_common common; + struct iommu_device *device; + device_t owner; + int map_count; + bus_dma_segment_t *segments; +}; + +struct bus_dmamap_iommu { + struct bus_dma_tag_iommu *device_tag; + struct memdesc mem; + bus_dmamap_callback_t *callback; + void *callback_arg; + struct iommu_map_entries_tailq map_entries; + TAILQ_ENTRY(bus_dmamap_iommu) delay_link; + bool locked; + bool cansleep; + int flags; +}; + +#define BUS_DMAMAP_IOMMU_MALLOC 0x0001 +#define BUS_DMAMAP_IOMMU_KMEM_ALLOC 0x0002 + +extern struct bus_dma_impl bus_dma_iommu_impl; + +bus_dma_tag_t acpi_iommu_get_dma_tag(device_t dev, device_t child); + +struct iommu_device; +struct iommu_domain; +struct iommu_unit; + +int iommu_map(struct iommu_domain *domain, + const struct bus_dma_tag_common *common, bus_size_t size, int offset, + int eflags, int iommu_flags, vm_page_t *ma, + struct iommu_map_entry **entry); +int iommu_unmap(struct iommu_domain *domain, + struct iommu_map_entries_tailq *entries, bool free); + +struct iommu_device *iommu_get_device(struct iommu_unit *iommu, + device_t requester, uint16_t rid, bool disabled, bool rmrr); +int iommu_free_device(struct iommu_device *device); +int iommu_free_device_locked(struct iommu_unit *iommu, + struct iommu_device *device); + +struct iommu_unit *iommu_find(device_t dev, bool verbose); +int iommu_init_busdma(struct iommu_unit *unit); +void iommu_fini_busdma(struct iommu_unit *unit); +struct iommu_device *iommu_instantiate_device(struct iommu_unit *dmar, + device_t dev, bool rmrr); +device_t iommu_get_requester(device_t dev, uint16_t *rid); + +#endif /* !_DEV_IOMMU_BUSDMA_IOMMU_H_*/ Index: sys/dev/iommu/busdma_iommu.c =================================================================== --- /dev/null +++ sys/dev/iommu/busdma_iommu.c @@ -0,0 +1,959 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined(__amd64__) || defined(__i386__) +#include +#include +#include +#include +#else +#include +#include +#include +#endif + +/* + * busdma_iommu.c, the implementation of the busdma(9) interface using + * IOMMU units. + */ + +static bool +iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func) +{ + char str[128], *env; + int default_bounce; + bool ret; + static const char bounce_str[] = "bounce"; + static const char iommu_str[] = "iommu"; + static const char dmar_str[] = "dmar"; /* compatibility with old str */ + + default_bounce = 0; + env = kern_getenv("hw.busdma.default"); + if (env != NULL) { + if (strcmp(env, bounce_str) == 0) + default_bounce = 1; + else if (strcmp(env, iommu_str) == 0 || + strcmp(env, dmar_str) == 0) + default_bounce = 0; + freeenv(env); + } + + snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d", + domain, bus, slot, func); + env = kern_getenv(str); + if (env == NULL) + return (default_bounce != 0); + if (strcmp(env, bounce_str) == 0) + ret = true; + else if (strcmp(env, iommu_str) == 0) + ret = false; + else + ret = default_bounce != 0; + freeenv(env); + return (ret); +} + +/* + * Given original device, find the requester ID that will be seen by + * the IOMMU unit and used for page table lookup. PCI bridges may take + * ownership of transactions from downstream devices, so it may not be + * the same as the BSF of the target device. In those cases, all + * devices downstream of the bridge must share a single mapping + * domain, and must collectively be assigned to use either IOMMU or + * bounce mapping. + */ +device_t +iommu_get_requester(device_t dev, uint16_t *rid) +{ + devclass_t pci_class; + device_t l, pci, pcib, pcip, pcibp, requester; + int cap_offset; + uint16_t pcie_flags; + bool bridge_is_pcie; + + pci_class = devclass_find("pci"); + l = requester = dev; + + *rid = pci_get_rid(dev); + + /* + * Walk the bridge hierarchy from the target device to the + * host port to find the translating bridge nearest the IOMMU + * unit. + */ + for (;;) { + pci = device_get_parent(l); + KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent " + "for %s", device_get_name(dev), device_get_name(l))); + KASSERT(device_get_devclass(pci) == pci_class, + ("iommu_get_requester(%s): non-pci parent %s for %s", + device_get_name(dev), device_get_name(pci), + device_get_name(l))); + + pcib = device_get_parent(pci); + KASSERT(pcib != NULL, ("iommu_get_requester(%s): NULL bridge " + "for %s", device_get_name(dev), device_get_name(pci))); + + /* + * The parent of our "bridge" isn't another PCI bus, + * so pcib isn't a PCI->PCI bridge but rather a host + * port, and the requester ID won't be translated + * further. + */ + pcip = device_get_parent(pcib); + if (device_get_devclass(pcip) != pci_class) + break; + pcibp = device_get_parent(pcip); + + if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) { + /* + * Do not stop the loop even if the target + * device is PCIe, because it is possible (but + * unlikely) to have a PCI->PCIe bridge + * somewhere in the hierarchy. + */ + l = pcib; + } else { + /* + * Device is not PCIe, it cannot be seen as a + * requester by IOMMU unit. Check whether the + * bridge is PCIe. + */ + bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS, + &cap_offset) == 0; + requester = pcib; + + /* + * Check for a buggy PCIe/PCI bridge that + * doesn't report the express capability. If + * the bridge above it is express but isn't a + * PCI bridge, then we know pcib is actually a + * PCIe/PCI bridge. + */ + if (!bridge_is_pcie && pci_find_cap(pcibp, + PCIY_EXPRESS, &cap_offset) == 0) { + pcie_flags = pci_read_config(pcibp, + cap_offset + PCIER_FLAGS, 2); + if ((pcie_flags & PCIEM_FLAGS_TYPE) != + PCIEM_TYPE_PCI_BRIDGE) + bridge_is_pcie = true; + } + + if (bridge_is_pcie) { + /* + * The current device is not PCIe, but + * the bridge above it is. This is a + * PCIe->PCI bridge. Assume that the + * requester ID will be the secondary + * bus number with slot and function + * set to zero. + * + * XXX: Doesn't handle the case where + * the bridge is PCIe->PCI-X, and the + * bridge will only take ownership of + * requests in some cases. We should + * provide context entries with the + * same page tables for taken and + * non-taken transactions. + */ + *rid = PCI_RID(pci_get_bus(l), 0, 0); + l = pcibp; + } else { + /* + * Neither the device nor the bridge + * above it are PCIe. This is a + * conventional PCI->PCI bridge, which + * will use the bridge's BSF as the + * requester ID. + */ + *rid = pci_get_rid(pcib); + l = pcib; + } + } + } + return (requester); +} + +struct iommu_device * +iommu_instantiate_device(struct iommu_unit *iommu, device_t dev, bool rmrr) +{ + device_t requester; + struct iommu_device *device; + bool disabled; + uint16_t rid; + + requester = iommu_get_requester(dev, &rid); + + /* + * If the user requested the IOMMU disabled for the device, we + * cannot disable the IOMMU, due to possibility of other + * devices on the same IOMMU still requiring translation. + * Instead provide the identity mapping for the device + * context. + */ + disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester), + pci_get_bus(requester), pci_get_slot(requester), + pci_get_function(requester)); + device = iommu_get_device(iommu, requester, rid, disabled, rmrr); + if (device == NULL) + return (NULL); + if (disabled) { + /* + * Keep the first reference on context, release the + * later refs. + */ + IOMMU_LOCK(iommu); + if ((device->flags & IOMMU_DEVICE_DISABLED) == 0) { + device->flags |= IOMMU_DEVICE_DISABLED; + IOMMU_UNLOCK(iommu); + } else { + iommu_free_device_locked(iommu, device); + } + device = NULL; + } + return (device); +} + +bus_dma_tag_t +acpi_iommu_get_dma_tag(device_t dev, device_t child) +{ + struct iommu_unit *iommu; + struct iommu_device *device; + bus_dma_tag_t res; + + iommu = iommu_find(child, bootverbose); + /* Not in scope of any IOMMU ? */ + if (iommu == NULL) + return (NULL); + if (!iommu->dma_enabled) + return (NULL); +#if defined(__amd64__) || defined(__i386__) + dmar_quirks_pre_use(iommu); + dmar_instantiate_rmrr_ctxs(iommu); +#endif + + device = iommu_instantiate_device(iommu, child, false); + res = device == NULL ? NULL : (bus_dma_tag_t)&device->device_tag; + return (res); +} + +static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map"); + +static void iommu_bus_schedule_dmamap(struct iommu_unit *unit, + struct bus_dmamap_iommu *map); + +static int +iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, + bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, + bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, + int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, + void *lockfuncarg, bus_dma_tag_t *dmat) +{ + struct bus_dma_tag_iommu *newtag, *oldtag; + int error; + + *dmat = NULL; + error = common_bus_dma_tag_create(parent != NULL ? + &((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment, + boundary, lowaddr, highaddr, filter, filterarg, maxsize, + nsegments, maxsegsz, flags, lockfunc, lockfuncarg, + sizeof(struct bus_dma_tag_iommu), (void **)&newtag); + if (error != 0) + goto out; + + oldtag = (struct bus_dma_tag_iommu *)parent; + newtag->common.impl = &bus_dma_iommu_impl; + newtag->device = oldtag->device; + newtag->owner = oldtag->owner; + + *dmat = (bus_dma_tag_t)newtag; +out: + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), + error); + return (error); +} + +static int +iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat) +{ + + return (0); +} + +static int +iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1) +{ + struct bus_dma_tag_iommu *dmat, *dmat_copy, *parent; + int error; + + error = 0; + dmat_copy = dmat = (struct bus_dma_tag_iommu *)dmat1; + + if (dmat != NULL) { + if (dmat->map_count != 0) { + error = EBUSY; + goto out; + } + while (dmat != NULL) { + parent = (struct bus_dma_tag_iommu *) + dmat->common.parent; + if (atomic_fetchadd_int(&dmat->common.ref_count, -1) == + 1) { + if (dmat == &dmat->device->device_tag) + iommu_free_device(dmat->device); + free_domain(dmat->segments, M_IOMMU_DMAMAP); + free(dmat, M_DEVBUF); + dmat = parent; + } else + dmat = NULL; + } + } +out: + CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); + return (error); +} + +static bool +iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) +{ + + return (false); +} + +static int +iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP, + DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO); + if (map == NULL) { + *mapp = NULL; + return (ENOMEM); + } + if (tag->segments == NULL) { + tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) * + tag->common.nsegments, M_IOMMU_DMAMAP, + DOMAINSET_PREF(tag->common.domain), M_NOWAIT); + if (tag->segments == NULL) { + free_domain(map, M_IOMMU_DMAMAP); + *mapp = NULL; + return (ENOMEM); + } + } + TAILQ_INIT(&map->map_entries); + map->device_tag = tag; + map->locked = true; + map->cansleep = false; + tag->map_count++; + *mapp = (bus_dmamap_t)map; + + return (0); +} + +static int +iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + struct iommu_domain *domain; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + if (map != NULL) { + domain = tag->device->domain; + IOMMU_DOMAIN_LOCK(domain); + if (!TAILQ_EMPTY(&map->map_entries)) { + IOMMU_DOMAIN_UNLOCK(domain); + return (EBUSY); + } + IOMMU_DOMAIN_UNLOCK(domain); + free_domain(map, M_IOMMU_DMAMAP); + } + tag->map_count--; + return (0); +} + + +static int +iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, + bus_dmamap_t *mapp) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + int error, mflags; + vm_memattr_t attr; + + error = iommu_bus_dmamap_create(dmat, flags, mapp); + if (error != 0) + return (error); + + mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK; + mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0; + attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE : + VM_MEMATTR_DEFAULT; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)*mapp; + + if (tag->common.maxsize < PAGE_SIZE && + tag->common.alignment <= tag->common.maxsize && + attr == VM_MEMATTR_DEFAULT) { + *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF, + DOMAINSET_PREF(tag->common.domain), mflags); + map->flags |= BUS_DMAMAP_IOMMU_MALLOC; + } else { + *vaddr = (void *)kmem_alloc_attr_domainset( + DOMAINSET_PREF(tag->common.domain), tag->common.maxsize, + mflags, 0ul, BUS_SPACE_MAXADDR, attr); + map->flags |= BUS_DMAMAP_IOMMU_KMEM_ALLOC; + } + if (*vaddr == NULL) { + iommu_bus_dmamap_destroy(dmat, *mapp); + *mapp = NULL; + return (ENOMEM); + } + return (0); +} + +static void +iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + + if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) { + free_domain(vaddr, M_DEVBUF); + map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC; + } else { + KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0, + ("iommu_bus_dmamem_free for non alloced map %p", map)); + kmem_free((vm_offset_t)vaddr, tag->common.maxsize); + map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC; + } + + iommu_bus_dmamap_destroy(dmat, map1); +} + +static int +iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag, + struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen, + int flags, bus_dma_segment_t *segs, int *segp, + struct iommu_map_entries_tailq *unroll_list) +{ + struct iommu_device *device; + struct iommu_domain *domain; + struct iommu_map_entry *entry; + bus_size_t size; + bus_size_t buflen1; + int error, idx, iommu_flags, seg; + + KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset)); + if (segs == NULL) + segs = tag->segments; + device = tag->device; + domain = device->domain; + seg = *segp; + error = 0; + idx = 0; + while (buflen > 0) { + seg++; + if (seg >= tag->common.nsegments) { + error = EFBIG; + break; + } + buflen1 = buflen > tag->common.maxsegsz ? + tag->common.maxsegsz : buflen; + size = round_page(offset + buflen1); + + /* + * (Too) optimistically allow split if there are more + * then one segments left. + */ + iommu_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0; + if (seg + 1 < tag->common.nsegments) + iommu_flags |= IOMMU_MF_CANSPLIT; + + error = iommu_map(domain, &tag->common, size, offset, + IOMMU_MAP_ENTRY_READ | + ((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE: 0), + iommu_flags, ma + idx, &entry); + if (error != 0) + break; + if ((iommu_flags & IOMMU_MF_CANSPLIT) != 0) { + KASSERT(size >= entry->end - entry->start, + ("split increased entry size %jx %jx %jx", + (uintmax_t)size, (uintmax_t)entry->start, + (uintmax_t)entry->end)); + size = entry->end - entry->start; + if (buflen1 > size) + buflen1 = size; + } else { + KASSERT(entry->end - entry->start == size, + ("no split allowed %jx %jx %jx", + (uintmax_t)size, (uintmax_t)entry->start, + (uintmax_t)entry->end)); + } + if (offset + buflen1 > size) + buflen1 = size - offset; + if (buflen1 > tag->common.maxsegsz) + buflen1 = tag->common.maxsegsz; + + KASSERT(((entry->start + offset) & (tag->common.alignment - 1)) + == 0, + ("alignment failed: device %p start 0x%jx offset %x " + "align 0x%jx", device, (uintmax_t)entry->start, offset, + (uintmax_t)tag->common.alignment)); + KASSERT(entry->end <= tag->common.lowaddr || + entry->start >= tag->common.highaddr, + ("entry placement failed: device %p start 0x%jx end 0x%jx " + "lowaddr 0x%jx highaddr 0x%jx", device, + (uintmax_t)entry->start, (uintmax_t)entry->end, + (uintmax_t)tag->common.lowaddr, + (uintmax_t)tag->common.highaddr)); + KASSERT(iommu_test_boundary(entry->start + offset, buflen1, + tag->common.boundary), + ("boundary failed: device %p start 0x%jx end 0x%jx " + "boundary 0x%jx", device, (uintmax_t)entry->start, + (uintmax_t)entry->end, (uintmax_t)tag->common.boundary)); + KASSERT(buflen1 <= tag->common.maxsegsz, + ("segment too large: device %p start 0x%jx end 0x%jx " + "buflen1 0x%jx maxsegsz 0x%jx", device, + (uintmax_t)entry->start, (uintmax_t)entry->end, + (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz)); + + IOMMU_DOMAIN_LOCK(domain); + TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); + entry->flags |= IOMMU_MAP_ENTRY_MAP; + IOMMU_DOMAIN_UNLOCK(domain); + TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link); + + segs[seg].ds_addr = entry->start + offset; + segs[seg].ds_len = buflen1; + + idx += OFF_TO_IDX(trunc_page(offset + buflen1)); + offset += buflen1; + offset &= IOMMU_PAGE_MASK; + buflen -= buflen1; + } + if (error == 0) + *segp = seg; + return (error); +} + +static int +iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag, + struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen, + int flags, bus_dma_segment_t *segs, int *segp) +{ + struct iommu_device *device; + struct iommu_domain *domain; + struct iommu_map_entry *entry, *entry1; + struct iommu_map_entries_tailq unroll_list; + int error; + + device = tag->device; + domain = device->domain; + atomic_add_long(&device->loads, 1); + + TAILQ_INIT(&unroll_list); + error = iommu_bus_dmamap_load_something1(tag, map, ma, offset, + buflen, flags, segs, segp, &unroll_list); + if (error != 0) { + /* + * The busdma interface does not allow us to report + * partial buffer load, so unfortunately we have to + * revert all work done. + */ + IOMMU_DOMAIN_LOCK(domain); + TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link, + entry1) { + /* + * No entries other than what we have created + * during the failed run might have been + * inserted there in between, since we own device + * pglock. + */ + TAILQ_REMOVE(&map->map_entries, entry, dmamap_link); + TAILQ_REMOVE(&unroll_list, entry, unroll_link); + TAILQ_INSERT_TAIL(&domain->unload_entries, entry, + dmamap_link); + } + IOMMU_DOMAIN_UNLOCK(domain); + taskqueue_enqueue(domain->iommu->delayed_taskqueue, + &domain->unload_task); + } + + if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 && + !map->cansleep) + error = EINPROGRESS; + if (error == EINPROGRESS) + iommu_bus_schedule_dmamap(domain->iommu, map); + return (error); +} + +static int +iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1, + struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, + bus_dma_segment_t *segs, int *segp) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen, + flags, segs, segp)); +} + +static int +iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1, + vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, + int *segp) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + vm_page_t *ma, fma; + vm_paddr_t pstart, pend, paddr; + int error, i, ma_cnt, mflags, offset; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + pstart = trunc_page(buf); + pend = round_page(buf + buflen); + offset = buf & PAGE_MASK; + ma_cnt = OFF_TO_IDX(pend - pstart); + mflags = map->cansleep ? M_WAITOK : M_NOWAIT; + ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); + if (ma == NULL) + return (ENOMEM); + fma = NULL; + for (i = 0; i < ma_cnt; i++) { + paddr = pstart + ptoa(i); + ma[i] = PHYS_TO_VM_PAGE(paddr); + if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { + /* + * If PHYS_TO_VM_PAGE() returned NULL or the + * vm_page was not initialized we'll use a + * fake page. + */ + if (fma == NULL) { + fma = malloc(sizeof(struct vm_page) * ma_cnt, + M_DEVBUF, M_ZERO | mflags); + if (fma == NULL) { + free(ma, M_DEVBUF); + return (ENOMEM); + } + } + vm_page_initfake(&fma[i], pstart + ptoa(i), + VM_MEMATTR_DEFAULT); + ma[i] = &fma[i]; + } + } + error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen, + flags, segs, segp); + free(fma, M_DEVBUF); + free(ma, M_DEVBUF); + return (error); +} + +static int +iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf, + bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, + int *segp) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + vm_page_t *ma, fma; + vm_paddr_t pstart, pend, paddr; + int error, i, ma_cnt, mflags, offset; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + pstart = trunc_page((vm_offset_t)buf); + pend = round_page((vm_offset_t)buf + buflen); + offset = (vm_offset_t)buf & PAGE_MASK; + ma_cnt = OFF_TO_IDX(pend - pstart); + mflags = map->cansleep ? M_WAITOK : M_NOWAIT; + ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); + if (ma == NULL) + return (ENOMEM); + fma = NULL; + for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { + if (pmap == kernel_pmap) + paddr = pmap_kextract(pstart); + else + paddr = pmap_extract(pmap, pstart); + ma[i] = PHYS_TO_VM_PAGE(paddr); + if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { + /* + * If PHYS_TO_VM_PAGE() returned NULL or the + * vm_page was not initialized we'll use a + * fake page. + */ + if (fma == NULL) { + fma = malloc(sizeof(struct vm_page) * ma_cnt, + M_DEVBUF, M_ZERO | mflags); + if (fma == NULL) { + free(ma, M_DEVBUF); + return (ENOMEM); + } + } + vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT); + ma[i] = &fma[i]; + } + } + error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen, + flags, segs, segp); + free(ma, M_DEVBUF); + free(fma, M_DEVBUF); + return (error); +} + +static void +iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1, + struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) +{ + struct bus_dmamap_iommu *map; + + if (map1 == NULL) + return; + map = (struct bus_dmamap_iommu *)map1; + map->mem = *mem; + map->device_tag = (struct bus_dma_tag_iommu *)dmat; + map->callback = callback; + map->callback_arg = callback_arg; +} + +static bus_dma_segment_t * +iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1, + bus_dma_segment_t *segs, int nsegs, int error) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + + if (!map->locked) { + KASSERT(map->cansleep, + ("map not locked and not sleepable context %p", map)); + + /* + * We are called from the delayed context. Relock the + * driver. + */ + (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK); + map->locked = true; + } + + if (segs == NULL) + segs = tag->segments; + return (segs); +} + +/* + * The limitations of busdma KPI forces the iommu to perform the actual + * unload, consisting of the unmapping of the map entries page tables, + * from the delayed context on i386, since page table page mapping + * might require a sleep to be successfull. The unfortunate + * consequence is that the DMA requests can be served some time after + * the bus_dmamap_unload() call returned. + * + * On amd64, we assume that sf allocation cannot fail. + */ +static void +iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + struct iommu_device *device; + struct iommu_domain *domain; +#if defined(__amd64__) || defined(__aarch64__) + struct iommu_map_entries_tailq entries; +#endif + + tag = (struct bus_dma_tag_iommu *)dmat; + map = (struct bus_dmamap_iommu *)map1; + device = tag->device; + domain = device->domain; + atomic_add_long(&device->unloads, 1); + +#if defined(__i386__) + IOMMU_DOMAIN_LOCK(domain); + TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link); + IOMMU_DOMAIN_UNLOCK(domain); + taskqueue_enqueue(domain->iommu->delayed_taskqueue, + &domain->unload_task); +#else /* defined(__amd64__) || defined(__aarch64__) */ + TAILQ_INIT(&entries); + IOMMU_DOMAIN_LOCK(domain); + TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link); + IOMMU_DOMAIN_UNLOCK(domain); + THREAD_NO_SLEEPING(); + iommu_unmap(domain, &entries, false); + THREAD_SLEEPING_OK(); + KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_device_unload %p", device)); +#endif +} + +static void +iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dmasync_op_t op) +{ +} + +struct bus_dma_impl bus_dma_iommu_impl = { + .tag_create = iommu_bus_dma_tag_create, + .tag_destroy = iommu_bus_dma_tag_destroy, + .tag_set_domain = iommu_bus_dma_tag_set_domain, + .id_mapped = iommu_bus_dma_id_mapped, + .map_create = iommu_bus_dmamap_create, + .map_destroy = iommu_bus_dmamap_destroy, + .mem_alloc = iommu_bus_dmamem_alloc, + .mem_free = iommu_bus_dmamem_free, + .load_phys = iommu_bus_dmamap_load_phys, + .load_buffer = iommu_bus_dmamap_load_buffer, + .load_ma = iommu_bus_dmamap_load_ma, + .map_waitok = iommu_bus_dmamap_waitok, + .map_complete = iommu_bus_dmamap_complete, + .map_unload = iommu_bus_dmamap_unload, + .map_sync = iommu_bus_dmamap_sync, +}; + +static void +iommu_bus_task_dmamap(void *arg, int pending) +{ + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + struct iommu_unit *unit; + + unit = arg; + IOMMU_LOCK(unit); + while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) { + TAILQ_REMOVE(&unit->delayed_maps, map, delay_link); + IOMMU_UNLOCK(unit); + tag = map->device_tag; + map->cansleep = true; + map->locked = false; + bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map, + &map->mem, map->callback, map->callback_arg, + BUS_DMA_WAITOK); + map->cansleep = false; + if (map->locked) { + (tag->common.lockfunc)(tag->common.lockfuncarg, + BUS_DMA_UNLOCK); + } else + map->locked = true; + map->cansleep = false; + IOMMU_LOCK(unit); + } + IOMMU_UNLOCK(unit); +} + +static void +iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map) +{ + + map->locked = false; + IOMMU_LOCK(unit); + TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link); + IOMMU_UNLOCK(unit); + taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task); +} + +int +iommu_init_busdma(struct iommu_unit *unit) +{ + + unit->dma_enabled = 1; + TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled); + TAILQ_INIT(&unit->delayed_maps); + TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit); + unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK, + taskqueue_thread_enqueue, &unit->delayed_taskqueue); + taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK, + "iommu%d busdma taskq", unit->unit); + return (0); +} + +void +iommu_fini_busdma(struct iommu_unit *unit) +{ + + if (unit->delayed_taskqueue == NULL) + return; + + taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task); + taskqueue_free(unit->delayed_taskqueue); + unit->delayed_taskqueue = NULL; +} Index: sys/x86/iommu/busdma_dmar.h =================================================================== --- sys/x86/iommu/busdma_dmar.h +++ sys/x86/iommu/busdma_dmar.h @@ -34,34 +34,6 @@ #ifndef __X86_IOMMU_BUSDMA_DMAR_H #define __X86_IOMMU_BUSDMA_DMAR_H -struct dmar_map_entry; -TAILQ_HEAD(dmar_map_entries_tailq, dmar_map_entry); - -struct bus_dma_tag_dmar { - struct bus_dma_tag_common common; - struct dmar_ctx *ctx; - device_t owner; - int map_count; - bus_dma_segment_t *segments; -}; - -struct bus_dmamap_dmar { - struct bus_dma_tag_dmar *tag; - struct memdesc mem; - bus_dmamap_callback_t *callback; - void *callback_arg; - struct dmar_map_entries_tailq map_entries; - TAILQ_ENTRY(bus_dmamap_dmar) delay_link; - bool locked; - bool cansleep; - int flags; -}; - -#define BUS_DMAMAP_DMAR_MALLOC 0x0001 -#define BUS_DMAMAP_DMAR_KMEM_ALLOC 0x0002 - -extern struct bus_dma_impl bus_dma_dmar_impl; - -bus_dma_tag_t acpi_iommu_get_dma_tag(device_t dev, device_t child); +#include #endif Index: sys/x86/iommu/busdma_dmar.c =================================================================== --- sys/x86/iommu/busdma_dmar.c +++ sys/x86/iommu/busdma_dmar.c @@ -68,231 +68,10 @@ #include #include -/* - * busdma_dmar.c, the implementation of the busdma(9) interface using - * DMAR units from Intel VT-d. - */ - -static bool -dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func) -{ - char str[128], *env; - int default_bounce; - bool ret; - static const char bounce_str[] = "bounce"; - static const char dmar_str[] = "dmar"; - - default_bounce = 0; - env = kern_getenv("hw.busdma.default"); - if (env != NULL) { - if (strcmp(env, bounce_str) == 0) - default_bounce = 1; - else if (strcmp(env, dmar_str) == 0) - default_bounce = 0; - freeenv(env); - } - - snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d", - domain, bus, slot, func); - env = kern_getenv(str); - if (env == NULL) - return (default_bounce != 0); - if (strcmp(env, bounce_str) == 0) - ret = true; - else if (strcmp(env, dmar_str) == 0) - ret = false; - else - ret = default_bounce != 0; - freeenv(env); - return (ret); -} - -/* - * Given original device, find the requester ID that will be seen by - * the DMAR unit and used for page table lookup. PCI bridges may take - * ownership of transactions from downstream devices, so it may not be - * the same as the BSF of the target device. In those cases, all - * devices downstream of the bridge must share a single mapping - * domain, and must collectively be assigned to use either DMAR or - * bounce mapping. - */ -device_t -dmar_get_requester(device_t dev, uint16_t *rid) -{ - devclass_t pci_class; - device_t l, pci, pcib, pcip, pcibp, requester; - int cap_offset; - uint16_t pcie_flags; - bool bridge_is_pcie; - - pci_class = devclass_find("pci"); - l = requester = dev; - - *rid = pci_get_rid(dev); - - /* - * Walk the bridge hierarchy from the target device to the - * host port to find the translating bridge nearest the DMAR - * unit. - */ - for (;;) { - pci = device_get_parent(l); - KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent " - "for %s", device_get_name(dev), device_get_name(l))); - KASSERT(device_get_devclass(pci) == pci_class, - ("dmar_get_requester(%s): non-pci parent %s for %s", - device_get_name(dev), device_get_name(pci), - device_get_name(l))); - - pcib = device_get_parent(pci); - KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge " - "for %s", device_get_name(dev), device_get_name(pci))); - - /* - * The parent of our "bridge" isn't another PCI bus, - * so pcib isn't a PCI->PCI bridge but rather a host - * port, and the requester ID won't be translated - * further. - */ - pcip = device_get_parent(pcib); - if (device_get_devclass(pcip) != pci_class) - break; - pcibp = device_get_parent(pcip); - - if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) { - /* - * Do not stop the loop even if the target - * device is PCIe, because it is possible (but - * unlikely) to have a PCI->PCIe bridge - * somewhere in the hierarchy. - */ - l = pcib; - } else { - /* - * Device is not PCIe, it cannot be seen as a - * requester by DMAR unit. Check whether the - * bridge is PCIe. - */ - bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS, - &cap_offset) == 0; - requester = pcib; - - /* - * Check for a buggy PCIe/PCI bridge that - * doesn't report the express capability. If - * the bridge above it is express but isn't a - * PCI bridge, then we know pcib is actually a - * PCIe/PCI bridge. - */ - if (!bridge_is_pcie && pci_find_cap(pcibp, - PCIY_EXPRESS, &cap_offset) == 0) { - pcie_flags = pci_read_config(pcibp, - cap_offset + PCIER_FLAGS, 2); - if ((pcie_flags & PCIEM_FLAGS_TYPE) != - PCIEM_TYPE_PCI_BRIDGE) - bridge_is_pcie = true; - } - - if (bridge_is_pcie) { - /* - * The current device is not PCIe, but - * the bridge above it is. This is a - * PCIe->PCI bridge. Assume that the - * requester ID will be the secondary - * bus number with slot and function - * set to zero. - * - * XXX: Doesn't handle the case where - * the bridge is PCIe->PCI-X, and the - * bridge will only take ownership of - * requests in some cases. We should - * provide context entries with the - * same page tables for taken and - * non-taken transactions. - */ - *rid = PCI_RID(pci_get_bus(l), 0, 0); - l = pcibp; - } else { - /* - * Neither the device nor the bridge - * above it are PCIe. This is a - * conventional PCI->PCI bridge, which - * will use the bridge's BSF as the - * requester ID. - */ - *rid = pci_get_rid(pcib); - l = pcib; - } - } - } - return (requester); -} - -struct dmar_ctx * -dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr) -{ - device_t requester; - struct dmar_ctx *ctx; - bool disabled; - uint16_t rid; - - requester = dmar_get_requester(dev, &rid); - - /* - * If the user requested the IOMMU disabled for the device, we - * cannot disable the DMAR, due to possibility of other - * devices on the same DMAR still requiring translation. - * Instead provide the identity mapping for the device - * context. - */ - disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), - pci_get_bus(requester), pci_get_slot(requester), - pci_get_function(requester)); - ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr); - if (ctx == NULL) - return (NULL); - if (disabled) { - /* - * Keep the first reference on context, release the - * later refs. - */ - DMAR_LOCK(dmar); - if ((ctx->flags & DMAR_CTX_DISABLED) == 0) { - ctx->flags |= DMAR_CTX_DISABLED; - DMAR_UNLOCK(dmar); - } else { - dmar_free_ctx_locked(dmar, ctx); - } - ctx = NULL; - } - return (ctx); -} - -bus_dma_tag_t -acpi_iommu_get_dma_tag(device_t dev, device_t child) -{ - struct dmar_unit *dmar; - struct dmar_ctx *ctx; - bus_dma_tag_t res; - - dmar = dmar_find(child, bootverbose); - /* Not in scope of any DMAR ? */ - if (dmar == NULL) - return (NULL); - if (!dmar->dma_enabled) - return (NULL); - dmar_quirks_pre_use(dmar); - dmar_instantiate_rmrr_ctxs(dmar); - - ctx = dmar_instantiate_ctx(dmar, child, false); - res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag; - return (res); -} - bool bus_dma_dmar_set_buswide(device_t dev) { - struct dmar_unit *dmar; + struct iommu_unit *dmar; device_t parent; u_int busno, slot, func; @@ -317,673 +96,16 @@ return (true); } -static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map"); - -static void dmar_bus_schedule_dmamap(struct dmar_unit *unit, - struct bus_dmamap_dmar *map); - -static int -dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, - bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, - bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, - int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, - void *lockfuncarg, bus_dma_tag_t *dmat) -{ - struct bus_dma_tag_dmar *newtag, *oldtag; - int error; - - *dmat = NULL; - error = common_bus_dma_tag_create(parent != NULL ? - &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment, - boundary, lowaddr, highaddr, filter, filterarg, maxsize, - nsegments, maxsegsz, flags, lockfunc, lockfuncarg, - sizeof(struct bus_dma_tag_dmar), (void **)&newtag); - if (error != 0) - goto out; - - oldtag = (struct bus_dma_tag_dmar *)parent; - newtag->common.impl = &bus_dma_dmar_impl; - newtag->ctx = oldtag->ctx; - newtag->owner = oldtag->owner; - - *dmat = (bus_dma_tag_t)newtag; -out: - CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", - __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), - error); - return (error); -} - -static int -dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat) -{ - - return (0); -} - -static int -dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) -{ - struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent; - int error; - - error = 0; - dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1; - - if (dmat != NULL) { - if (dmat->map_count != 0) { - error = EBUSY; - goto out; - } - while (dmat != NULL) { - parent = (struct bus_dma_tag_dmar *)dmat->common.parent; - if (atomic_fetchadd_int(&dmat->common.ref_count, -1) == - 1) { - if (dmat == &dmat->ctx->ctx_tag) - dmar_free_ctx(dmat->ctx); - free_domain(dmat->segments, M_DMAR_DMAMAP); - free(dmat, M_DEVBUF); - dmat = parent; - } else - dmat = NULL; - } - } -out: - CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); - return (error); -} - -static bool -dmar_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) -{ - - return (false); -} - -static int -dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = malloc_domainset(sizeof(*map), M_DMAR_DMAMAP, - DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO); - if (map == NULL) { - *mapp = NULL; - return (ENOMEM); - } - if (tag->segments == NULL) { - tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) * - tag->common.nsegments, M_DMAR_DMAMAP, - DOMAINSET_PREF(tag->common.domain), M_NOWAIT); - if (tag->segments == NULL) { - free_domain(map, M_DMAR_DMAMAP); - *mapp = NULL; - return (ENOMEM); - } - } - TAILQ_INIT(&map->map_entries); - map->tag = tag; - map->locked = true; - map->cansleep = false; - tag->map_count++; - *mapp = (bus_dmamap_t)map; - - return (0); -} - -static int -dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - struct dmar_domain *domain; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - if (map != NULL) { - domain = tag->ctx->domain; - DMAR_DOMAIN_LOCK(domain); - if (!TAILQ_EMPTY(&map->map_entries)) { - DMAR_DOMAIN_UNLOCK(domain); - return (EBUSY); - } - DMAR_DOMAIN_UNLOCK(domain); - free_domain(map, M_DMAR_DMAMAP); - } - tag->map_count--; - return (0); -} - - -static int -dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, - bus_dmamap_t *mapp) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - int error, mflags; - vm_memattr_t attr; - - error = dmar_bus_dmamap_create(dmat, flags, mapp); - if (error != 0) - return (error); - - mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK; - mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0; - attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE : - VM_MEMATTR_DEFAULT; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)*mapp; - - if (tag->common.maxsize < PAGE_SIZE && - tag->common.alignment <= tag->common.maxsize && - attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF, - DOMAINSET_PREF(tag->common.domain), mflags); - map->flags |= BUS_DMAMAP_DMAR_MALLOC; - } else { - *vaddr = (void *)kmem_alloc_attr_domainset( - DOMAINSET_PREF(tag->common.domain), tag->common.maxsize, - mflags, 0ul, BUS_SPACE_MAXADDR, attr); - map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; - } - if (*vaddr == NULL) { - dmar_bus_dmamap_destroy(dmat, *mapp); - *mapp = NULL; - return (ENOMEM); - } - return (0); -} - -static void -dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - - if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) { - free_domain(vaddr, M_DEVBUF); - map->flags &= ~BUS_DMAMAP_DMAR_MALLOC; - } else { - KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0, - ("dmar_bus_dmamem_free for non alloced map %p", map)); - kmem_free((vm_offset_t)vaddr, tag->common.maxsize); - map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC; - } - - dmar_bus_dmamap_destroy(dmat, map1); -} - -static int -dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag, - struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, - int flags, bus_dma_segment_t *segs, int *segp, - struct dmar_map_entries_tailq *unroll_list) -{ - struct dmar_ctx *ctx; - struct dmar_domain *domain; - struct dmar_map_entry *entry; - dmar_gaddr_t size; - bus_size_t buflen1; - int error, idx, gas_flags, seg; - - KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset)); - if (segs == NULL) - segs = tag->segments; - ctx = tag->ctx; - domain = ctx->domain; - seg = *segp; - error = 0; - idx = 0; - while (buflen > 0) { - seg++; - if (seg >= tag->common.nsegments) { - error = EFBIG; - break; - } - buflen1 = buflen > tag->common.maxsegsz ? - tag->common.maxsegsz : buflen; - size = round_page(offset + buflen1); - - /* - * (Too) optimistically allow split if there are more - * then one segments left. - */ - gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0; - if (seg + 1 < tag->common.nsegments) - gas_flags |= DMAR_GM_CANSPLIT; - - error = dmar_gas_map(domain, &tag->common, size, offset, - DMAR_MAP_ENTRY_READ | - ((flags & BUS_DMA_NOWRITE) == 0 ? DMAR_MAP_ENTRY_WRITE : 0), - gas_flags, ma + idx, &entry); - if (error != 0) - break; - if ((gas_flags & DMAR_GM_CANSPLIT) != 0) { - KASSERT(size >= entry->end - entry->start, - ("split increased entry size %jx %jx %jx", - (uintmax_t)size, (uintmax_t)entry->start, - (uintmax_t)entry->end)); - size = entry->end - entry->start; - if (buflen1 > size) - buflen1 = size; - } else { - KASSERT(entry->end - entry->start == size, - ("no split allowed %jx %jx %jx", - (uintmax_t)size, (uintmax_t)entry->start, - (uintmax_t)entry->end)); - } - if (offset + buflen1 > size) - buflen1 = size - offset; - if (buflen1 > tag->common.maxsegsz) - buflen1 = tag->common.maxsegsz; - - KASSERT(((entry->start + offset) & (tag->common.alignment - 1)) - == 0, - ("alignment failed: ctx %p start 0x%jx offset %x " - "align 0x%jx", ctx, (uintmax_t)entry->start, offset, - (uintmax_t)tag->common.alignment)); - KASSERT(entry->end <= tag->common.lowaddr || - entry->start >= tag->common.highaddr, - ("entry placement failed: ctx %p start 0x%jx end 0x%jx " - "lowaddr 0x%jx highaddr 0x%jx", ctx, - (uintmax_t)entry->start, (uintmax_t)entry->end, - (uintmax_t)tag->common.lowaddr, - (uintmax_t)tag->common.highaddr)); - KASSERT(dmar_test_boundary(entry->start + offset, buflen1, - tag->common.boundary), - ("boundary failed: ctx %p start 0x%jx end 0x%jx " - "boundary 0x%jx", ctx, (uintmax_t)entry->start, - (uintmax_t)entry->end, (uintmax_t)tag->common.boundary)); - KASSERT(buflen1 <= tag->common.maxsegsz, - ("segment too large: ctx %p start 0x%jx end 0x%jx " - "buflen1 0x%jx maxsegsz 0x%jx", ctx, - (uintmax_t)entry->start, (uintmax_t)entry->end, - (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz)); - - DMAR_DOMAIN_LOCK(domain); - TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); - entry->flags |= DMAR_MAP_ENTRY_MAP; - DMAR_DOMAIN_UNLOCK(domain); - TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link); - - segs[seg].ds_addr = entry->start + offset; - segs[seg].ds_len = buflen1; - - idx += OFF_TO_IDX(trunc_page(offset + buflen1)); - offset += buflen1; - offset &= DMAR_PAGE_MASK; - buflen -= buflen1; - } - if (error == 0) - *segp = seg; - return (error); -} - -static int -dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag, - struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen, - int flags, bus_dma_segment_t *segs, int *segp) -{ - struct dmar_ctx *ctx; - struct dmar_domain *domain; - struct dmar_map_entry *entry, *entry1; - struct dmar_map_entries_tailq unroll_list; - int error; - - ctx = tag->ctx; - domain = ctx->domain; - atomic_add_long(&ctx->loads, 1); - - TAILQ_INIT(&unroll_list); - error = dmar_bus_dmamap_load_something1(tag, map, ma, offset, - buflen, flags, segs, segp, &unroll_list); - if (error != 0) { - /* - * The busdma interface does not allow us to report - * partial buffer load, so unfortunately we have to - * revert all work done. - */ - DMAR_DOMAIN_LOCK(domain); - TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link, - entry1) { - /* - * No entries other than what we have created - * during the failed run might have been - * inserted there in between, since we own ctx - * pglock. - */ - TAILQ_REMOVE(&map->map_entries, entry, dmamap_link); - TAILQ_REMOVE(&unroll_list, entry, unroll_link); - TAILQ_INSERT_TAIL(&domain->unload_entries, entry, - dmamap_link); - } - DMAR_DOMAIN_UNLOCK(domain); - taskqueue_enqueue(domain->dmar->delayed_taskqueue, - &domain->unload_task); - } - - if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 && - !map->cansleep) - error = EINPROGRESS; - if (error == EINPROGRESS) - dmar_bus_schedule_dmamap(domain->dmar, map); - return (error); -} - -static int -dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1, - struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, - bus_dma_segment_t *segs, int *segp) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen, - flags, segs, segp)); -} - -static int -dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1, - vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, - int *segp) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - vm_page_t *ma, fma; - vm_paddr_t pstart, pend, paddr; - int error, i, ma_cnt, mflags, offset; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - pstart = trunc_page(buf); - pend = round_page(buf + buflen); - offset = buf & PAGE_MASK; - ma_cnt = OFF_TO_IDX(pend - pstart); - mflags = map->cansleep ? M_WAITOK : M_NOWAIT; - ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); - if (ma == NULL) - return (ENOMEM); - fma = NULL; - for (i = 0; i < ma_cnt; i++) { - paddr = pstart + ptoa(i); - ma[i] = PHYS_TO_VM_PAGE(paddr); - if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { - /* - * If PHYS_TO_VM_PAGE() returned NULL or the - * vm_page was not initialized we'll use a - * fake page. - */ - if (fma == NULL) { - fma = malloc(sizeof(struct vm_page) * ma_cnt, - M_DEVBUF, M_ZERO | mflags); - if (fma == NULL) { - free(ma, M_DEVBUF); - return (ENOMEM); - } - } - vm_page_initfake(&fma[i], pstart + ptoa(i), - VM_MEMATTR_DEFAULT); - ma[i] = &fma[i]; - } - } - error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, - flags, segs, segp); - free(fma, M_DEVBUF); - free(ma, M_DEVBUF); - return (error); -} - -static int -dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf, - bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, - int *segp) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - vm_page_t *ma, fma; - vm_paddr_t pstart, pend, paddr; - int error, i, ma_cnt, mflags, offset; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - pstart = trunc_page((vm_offset_t)buf); - pend = round_page((vm_offset_t)buf + buflen); - offset = (vm_offset_t)buf & PAGE_MASK; - ma_cnt = OFF_TO_IDX(pend - pstart); - mflags = map->cansleep ? M_WAITOK : M_NOWAIT; - ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); - if (ma == NULL) - return (ENOMEM); - fma = NULL; - for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { - if (pmap == kernel_pmap) - paddr = pmap_kextract(pstart); - else - paddr = pmap_extract(pmap, pstart); - ma[i] = PHYS_TO_VM_PAGE(paddr); - if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { - /* - * If PHYS_TO_VM_PAGE() returned NULL or the - * vm_page was not initialized we'll use a - * fake page. - */ - if (fma == NULL) { - fma = malloc(sizeof(struct vm_page) * ma_cnt, - M_DEVBUF, M_ZERO | mflags); - if (fma == NULL) { - free(ma, M_DEVBUF); - return (ENOMEM); - } - } - vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT); - ma[i] = &fma[i]; - } - } - error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen, - flags, segs, segp); - free(ma, M_DEVBUF); - free(fma, M_DEVBUF); - return (error); -} - -static void -dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1, - struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) -{ - struct bus_dmamap_dmar *map; - - if (map1 == NULL) - return; - map = (struct bus_dmamap_dmar *)map1; - map->mem = *mem; - map->tag = (struct bus_dma_tag_dmar *)dmat; - map->callback = callback; - map->callback_arg = callback_arg; -} - -static bus_dma_segment_t * -dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1, - bus_dma_segment_t *segs, int nsegs, int error) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - - if (!map->locked) { - KASSERT(map->cansleep, - ("map not locked and not sleepable context %p", map)); - - /* - * We are called from the delayed context. Relock the - * driver. - */ - (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK); - map->locked = true; - } - - if (segs == NULL) - segs = tag->segments; - return (segs); -} - -/* - * The limitations of busdma KPI forces the dmar to perform the actual - * unload, consisting of the unmapping of the map entries page tables, - * from the delayed context on i386, since page table page mapping - * might require a sleep to be successfull. The unfortunate - * consequence is that the DMA requests can be served some time after - * the bus_dmamap_unload() call returned. - * - * On amd64, we assume that sf allocation cannot fail. - */ -static void -dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - struct dmar_ctx *ctx; - struct dmar_domain *domain; -#if defined(__amd64__) - struct dmar_map_entries_tailq entries; -#endif - - tag = (struct bus_dma_tag_dmar *)dmat; - map = (struct bus_dmamap_dmar *)map1; - ctx = tag->ctx; - domain = ctx->domain; - atomic_add_long(&ctx->unloads, 1); - -#if defined(__i386__) - DMAR_DOMAIN_LOCK(domain); - TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link); - DMAR_DOMAIN_UNLOCK(domain); - taskqueue_enqueue(domain->dmar->delayed_taskqueue, - &domain->unload_task); -#else /* defined(__amd64__) */ - TAILQ_INIT(&entries); - DMAR_DOMAIN_LOCK(domain); - TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link); - DMAR_DOMAIN_UNLOCK(domain); - THREAD_NO_SLEEPING(); - dmar_domain_unload(domain, &entries, false); - THREAD_SLEEPING_OK(); - KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx)); -#endif -} - -static void -dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, - bus_dmasync_op_t op) -{ -} - -struct bus_dma_impl bus_dma_dmar_impl = { - .tag_create = dmar_bus_dma_tag_create, - .tag_destroy = dmar_bus_dma_tag_destroy, - .tag_set_domain = dmar_bus_dma_tag_set_domain, - .id_mapped = dmar_bus_dma_id_mapped, - .map_create = dmar_bus_dmamap_create, - .map_destroy = dmar_bus_dmamap_destroy, - .mem_alloc = dmar_bus_dmamem_alloc, - .mem_free = dmar_bus_dmamem_free, - .load_phys = dmar_bus_dmamap_load_phys, - .load_buffer = dmar_bus_dmamap_load_buffer, - .load_ma = dmar_bus_dmamap_load_ma, - .map_waitok = dmar_bus_dmamap_waitok, - .map_complete = dmar_bus_dmamap_complete, - .map_unload = dmar_bus_dmamap_unload, - .map_sync = dmar_bus_dmamap_sync, -}; - -static void -dmar_bus_task_dmamap(void *arg, int pending) -{ - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - struct dmar_unit *unit; - - unit = arg; - DMAR_LOCK(unit); - while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) { - TAILQ_REMOVE(&unit->delayed_maps, map, delay_link); - DMAR_UNLOCK(unit); - tag = map->tag; - map->cansleep = true; - map->locked = false; - bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map, - &map->mem, map->callback, map->callback_arg, - BUS_DMA_WAITOK); - map->cansleep = false; - if (map->locked) { - (tag->common.lockfunc)(tag->common.lockfuncarg, - BUS_DMA_UNLOCK); - } else - map->locked = true; - map->cansleep = false; - DMAR_LOCK(unit); - } - DMAR_UNLOCK(unit); -} - -static void -dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map) -{ - - map->locked = false; - DMAR_LOCK(unit); - TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link); - DMAR_UNLOCK(unit); - taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task); -} - -int -dmar_init_busdma(struct dmar_unit *unit) -{ - - unit->dma_enabled = 1; - TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled); - TAILQ_INIT(&unit->delayed_maps); - TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit); - unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK, - taskqueue_thread_enqueue, &unit->delayed_taskqueue); - taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK, - "dmar%d busdma taskq", unit->unit); - return (0); -} - -void -dmar_fini_busdma(struct dmar_unit *unit) -{ - - if (unit->delayed_taskqueue == NULL) - return; - - taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task); - taskqueue_free(unit->delayed_taskqueue); - unit->delayed_taskqueue = NULL; -} - int bus_dma_dmar_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1, vm_paddr_t start, vm_size_t length, int flags) { struct bus_dma_tag_common *tc; - struct bus_dma_tag_dmar *tag; - struct bus_dmamap_dmar *map; - struct dmar_ctx *ctx; - struct dmar_domain *domain; - struct dmar_map_entry *entry; + struct bus_dma_tag_iommu *tag; + struct bus_dmamap_iommu *map; + struct iommu_device *ctx; + struct iommu_domain *domain; + struct iommu_map_entry *entry; vm_page_t *ma; vm_size_t i; int error; @@ -996,13 +118,13 @@ MPASS((flags & ~(BUS_DMA_NOWAIT | BUS_DMA_NOWRITE)) == 0); tc = (struct bus_dma_tag_common *)dmat; - if (tc->impl != &bus_dma_dmar_impl) + if (tc->impl != &bus_dma_iommu_impl) return (0); - tag = (struct bus_dma_tag_dmar *)dmat; - ctx = tag->ctx; + tag = (struct bus_dma_tag_iommu *)dmat; + ctx = tag->device; domain = ctx->domain; - map = (struct bus_dmamap_dmar *)map1; + map = (struct bus_dmamap_iommu *)map1; waitok = (flags & BUS_DMA_NOWAIT) != 0; entry = dmar_gas_alloc_entry(domain, waitok ? 0 : DMAR_PGF_WAITOK); @@ -1020,14 +142,14 @@ ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, VM_MEMATTR_DEFAULT); } - error = dmar_gas_map_region(domain, entry, DMAR_MAP_ENTRY_READ | - ((flags & BUS_DMA_NOWRITE) ? 0 : DMAR_MAP_ENTRY_WRITE), - waitok ? DMAR_GM_CANWAIT : 0, ma); + error = dmar_gas_map_region(domain, entry, IOMMU_MAP_ENTRY_READ | + ((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE), + waitok ? IOMMU_MF_CANWAIT : 0, ma); if (error == 0) { - DMAR_DOMAIN_LOCK(domain); + IOMMU_DOMAIN_LOCK(domain); TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); - entry->flags |= DMAR_MAP_ENTRY_MAP; - DMAR_DOMAIN_UNLOCK(domain); + entry->flags |= IOMMU_MAP_ENTRY_MAP; + IOMMU_DOMAIN_UNLOCK(domain); } else { dmar_domain_unload_entry(entry, true); } @@ -1036,3 +158,62 @@ free(ma, M_TEMP); return (error); } + +int +iommu_unmap(struct iommu_domain *domain, + struct iommu_map_entries_tailq *entries, bool cansleep) +{ + + dmar_domain_unload(domain, entries, cansleep); + + return (0); +} + +int +iommu_map(struct iommu_domain *domain, + const struct bus_dma_tag_common *common, + bus_size_t size, int offset, + int eflags, int iommu_flags, + vm_page_t *ma, struct iommu_map_entry **entry) +{ + int ret; + + ret = dmar_gas_map(domain, common, (dmar_gaddr_t)size, offset, + eflags, iommu_flags, ma, entry); + + return (ret); +} + +struct iommu_device * +iommu_get_device(struct iommu_unit *iommu, device_t dev, + uint16_t rid, bool id_mapped, bool rmrr_init) +{ + + return (dmar_get_ctx_for_dev(iommu, dev, rid, id_mapped, rmrr_init)); +} + + +int +iommu_free_device(struct iommu_device *device) +{ + + dmar_free_ctx(device); + + return (0); +} + +int +iommu_free_device_locked(struct iommu_unit *dmar, struct iommu_device *ctx) +{ + + dmar_free_ctx_locked(dmar, ctx); + + return (0); +} + +struct iommu_unit * +iommu_find(device_t dev, bool verbose) +{ + + return (dmar_find(dev, verbose)); +} Index: sys/x86/iommu/intel_ctx.c =================================================================== --- sys/x86/iommu/intel_ctx.c +++ sys/x86/iommu/intel_ctx.c @@ -71,16 +71,16 @@ #include #include -static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); +static MALLOC_DEFINE(M_DMAR_CTX, "iommu_device", "Intel DMAR Context"); static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); static void dmar_domain_unload_task(void *arg, int pending); -static void dmar_unref_domain_locked(struct dmar_unit *dmar, - struct dmar_domain *domain); -static void dmar_domain_destroy(struct dmar_domain *domain); +static void dmar_unref_domain_locked(struct iommu_unit *dmar, + struct iommu_domain *domain); +static void dmar_domain_destroy(struct iommu_domain *domain); static void -dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) +dmar_ensure_ctx_page(struct iommu_unit *dmar, int bus) { struct sf_buf *sf; dmar_root_entry_t *re; @@ -111,37 +111,37 @@ TD_PINNED_ASSERT; } -static dmar_ctx_entry_t * -dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) +static iommu_device_entry_t * +dmar_map_ctx_entry(struct iommu_device *ctx, struct sf_buf **sfp) { - dmar_ctx_entry_t *ctxp; + iommu_device_entry_t *ctxp; - ctxp = dmar_map_pgtbl(ctx->domain->dmar->ctx_obj, 1 + + ctxp = dmar_map_pgtbl(ctx->domain->iommu->ctx_obj, 1 + PCI_RID2BUS(ctx->rid), DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); ctxp += ctx->rid & 0xff; return (ctxp); } static void -ctx_tag_init(struct dmar_ctx *ctx, device_t dev) +device_tag_init(struct iommu_device *ctx, device_t dev) { bus_addr_t maxaddr; maxaddr = MIN(ctx->domain->end, BUS_SPACE_MAXADDR); - ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ - ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; - ctx->ctx_tag.common.boundary = 0; - ctx->ctx_tag.common.lowaddr = maxaddr; - ctx->ctx_tag.common.highaddr = maxaddr; - ctx->ctx_tag.common.maxsize = maxaddr; - ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; - ctx->ctx_tag.common.maxsegsz = maxaddr; - ctx->ctx_tag.ctx = ctx; - ctx->ctx_tag.owner = dev; + ctx->device_tag.common.ref_count = 1; /* Prevent free */ + ctx->device_tag.common.impl = &bus_dma_iommu_impl; + ctx->device_tag.common.boundary = 0; + ctx->device_tag.common.lowaddr = maxaddr; + ctx->device_tag.common.highaddr = maxaddr; + ctx->device_tag.common.maxsize = maxaddr; + ctx->device_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; + ctx->device_tag.common.maxsegsz = maxaddr; + ctx->device_tag.device = ctx; + ctx->device_tag.owner = dev; } static void -ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, +ctx_id_entry_init_one(iommu_device_entry_t *ctxp, struct iommu_domain *domain, vm_page_t ctx_root) { /* @@ -165,20 +165,20 @@ } static void -ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, - int busno) +ctx_id_entry_init(struct iommu_device *ctx, iommu_device_entry_t *ctxp, + bool move, int busno) { - struct dmar_unit *unit; - struct dmar_domain *domain; + struct iommu_unit *unit; + struct iommu_domain *domain; vm_page_t ctx_root; int i; domain = ctx->domain; - unit = domain->dmar; + unit = domain->iommu; KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", - unit->unit, busno, pci_get_slot(ctx->ctx_tag.owner), - pci_get_function(ctx->ctx_tag.owner), + unit->unit, busno, pci_get_slot(ctx->device_tag.owner), + pci_get_function(ctx->device_tag.owner), ctxp->ctx1, ctxp->ctx2)); if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 && @@ -202,7 +202,7 @@ } static int -dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) +dmar_flush_for_ctx_entry(struct iommu_unit *dmar, bool force) { int error; @@ -226,12 +226,12 @@ } static int -domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, +domain_init_rmrr(struct iommu_domain *domain, device_t dev, int bus, int slot, int func, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len) { - struct dmar_map_entries_tailq rmrr_entries; - struct dmar_map_entry *entry, *entry1; + struct iommu_map_entries_tailq rmrr_entries; + struct iommu_map_entry *entry, *entry1; vm_page_t *ma; dmar_gaddr_t start, end; vm_pindex_t size, i; @@ -255,7 +255,7 @@ end = entry->end; if (bootverbose) printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", - domain->dmar->unit, bus, slot, func, + domain->iommu->unit, bus, slot, func, (uintmax_t)start, (uintmax_t)end); entry->start = trunc_page(start); entry->end = round_page(end); @@ -267,7 +267,7 @@ printf("pci%d:%d:%d ", bus, slot, func); printf("BIOS bug: dmar%d RMRR " "region (%jx, %jx) corrected\n", - domain->dmar->unit, start, end); + domain->iommu->unit, start, end); } entry->end += DMAR_PAGE_SIZE * 0x20; } @@ -278,8 +278,8 @@ VM_MEMATTR_DEFAULT); } error1 = dmar_gas_map_region(domain, entry, - DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE, - DMAR_GM_CANWAIT | DMAR_GM_RMRR, ma); + IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, + IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); /* * Non-failed RMRR entries are owned by context rb * tree. Get rid of the failed entry, but do not stop @@ -287,10 +287,10 @@ * loaded and removed on the context destruction. */ if (error1 == 0 && entry->end != entry->start) { - DMAR_LOCK(domain->dmar); + IOMMU_LOCK(domain->iommu); domain->refs++; /* XXXKIB prevent free */ domain->flags |= DMAR_DOMAIN_RMRR; - DMAR_UNLOCK(domain->dmar); + IOMMU_UNLOCK(domain->iommu); } else { if (error1 != 0) { if (dev != NULL) @@ -298,7 +298,7 @@ printf("pci%d:%d:%d ", bus, slot, func); printf( "dmar%d failed to map RMRR region (%jx, %jx) %d\n", - domain->dmar->unit, start, end, + domain->iommu->unit, start, end, error1); error = error1; } @@ -312,10 +312,10 @@ return (error); } -static struct dmar_domain * -dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) +static struct iommu_domain * +dmar_domain_alloc(struct iommu_unit *dmar, bool id_mapped) { - struct dmar_domain *domain; + struct iommu_domain *domain; int error, id, mgaw; id = alloc_unr(dmar->domids); @@ -328,7 +328,7 @@ TAILQ_INIT(&domain->unload_entries); TASK_INIT(&domain->unload_task, 0, dmar_domain_unload_task, domain); mtx_init(&domain->lock, "dmardom", NULL, MTX_DEF); - domain->dmar = dmar; + domain->iommu = dmar; /* * For now, use the maximal usable physical address of the @@ -370,10 +370,10 @@ return (NULL); } -static struct dmar_ctx * -dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) +static struct iommu_device * +dmar_device_alloc(struct iommu_domain *domain, uint16_t rid) { - struct dmar_ctx *ctx; + struct iommu_device *ctx; ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); ctx->domain = domain; @@ -383,12 +383,12 @@ } static void -dmar_ctx_link(struct dmar_ctx *ctx) +dmar_device_link(struct iommu_device *ctx) { - struct dmar_domain *domain; + struct iommu_domain *domain; domain = ctx->domain; - DMAR_ASSERT_LOCKED(domain->dmar); + IOMMU_ASSERT_LOCKED(domain->iommu); KASSERT(domain->refs >= domain->ctx_cnt, ("dom %p ref underflow %d %d", domain, domain->refs, domain->ctx_cnt)); @@ -398,12 +398,12 @@ } static void -dmar_ctx_unlink(struct dmar_ctx *ctx) +dmar_device_unlink(struct iommu_device *ctx) { - struct dmar_domain *domain; + struct iommu_domain *domain; domain = ctx->domain; - DMAR_ASSERT_LOCKED(domain->dmar); + IOMMU_ASSERT_LOCKED(domain->iommu); KASSERT(domain->refs > 0, ("domain %p ctx dtr refs %d", domain, domain->refs)); KASSERT(domain->ctx_cnt >= domain->refs, @@ -415,7 +415,7 @@ } static void -dmar_domain_destroy(struct dmar_domain *domain) +dmar_domain_destroy(struct iommu_domain *domain) { KASSERT(TAILQ_EMPTY(&domain->unload_entries), @@ -427,9 +427,9 @@ KASSERT(domain->refs == 0, ("destroying dom %p with refs %d", domain, domain->refs)); if ((domain->flags & DMAR_DOMAIN_GAS_INITED) != 0) { - DMAR_DOMAIN_LOCK(domain); + IOMMU_DOMAIN_LOCK(domain); dmar_gas_fini_domain(domain); - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); } if ((domain->flags & DMAR_DOMAIN_PGTBL_INITED) != 0) { if (domain->pgtbl_obj != NULL) @@ -437,18 +437,18 @@ domain_free_pgtbl(domain); } mtx_destroy(&domain->lock); - free_unr(domain->dmar->domids, domain->domain); + free_unr(domain->iommu->domids, domain->domain); free(domain, M_DMAR_DOMAIN); } -static struct dmar_ctx * -dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, +static struct iommu_device * +dmar_get_ctx_for_dev1(struct iommu_unit *dmar, device_t dev, uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, bool id_mapped, bool rmrr_init) { - struct dmar_domain *domain, *domain1; - struct dmar_ctx *ctx, *ctx1; - dmar_ctx_entry_t *ctxp; + struct iommu_domain *domain, *domain1; + struct iommu_device *ctx, *ctx1; + iommu_device_entry_t *ctxp; struct sf_buf *sf; int bus, slot, func, error; bool enable; @@ -464,7 +464,7 @@ } enable = false; TD_PREP_PINNED_ASSERT; - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); KASSERT(!dmar_is_buswide_ctx(dmar, bus) || (slot == 0 && func == 0), ("dmar%d pci%d:%d:%d get_ctx for buswide", dmar->unit, bus, slot, func)); @@ -475,7 +475,7 @@ * Perform the allocations which require sleep or have * higher chance to succeed if the sleep is allowed. */ - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); domain1 = dmar_domain_alloc(dmar, id_mapped); if (domain1 == NULL) { @@ -492,9 +492,9 @@ return (NULL); } } - ctx1 = dmar_ctx_alloc(domain1, rid); + ctx1 = dmar_device_alloc(domain1, rid); ctxp = dmar_map_ctx_entry(ctx1, &sf); - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); /* * Recheck the contexts, other thread might have @@ -504,9 +504,9 @@ if (ctx == NULL) { domain = domain1; ctx = ctx1; - dmar_ctx_link(ctx); - ctx->ctx_tag.owner = dev; - ctx_tag_init(ctx, dev); + dmar_device_link(ctx); + ctx->device_tag.owner = dev; + device_tag_init(ctx, dev); /* * This is the first activated context for the @@ -536,8 +536,8 @@ } } else { domain = ctx->domain; - if (ctx->ctx_tag.owner == NULL) - ctx->ctx_tag.owner = dev; + if (ctx->device_tag.owner == NULL) + ctx->device_tag.owner = dev; ctx->refs++; /* tag referenced us */ } @@ -568,13 +568,13 @@ return (NULL); } } - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); TD_PINNED_ASSERT; return (ctx); } -struct dmar_ctx * -dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, +struct iommu_device * +dmar_get_ctx_for_dev(struct iommu_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init) { int dev_domain, dev_path_len, dev_busno; @@ -587,8 +587,8 @@ dev_path, dev_path_len, id_mapped, rmrr_init)); } -struct dmar_ctx * -dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, +struct iommu_device * +dmar_get_ctx_for_devpath(struct iommu_unit *dmar, uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, bool id_mapped, bool rmrr_init) @@ -599,28 +599,28 @@ } int -dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) +dmar_move_ctx_to_domain(struct iommu_domain *domain, struct iommu_device *ctx) { - struct dmar_unit *dmar; - struct dmar_domain *old_domain; - dmar_ctx_entry_t *ctxp; + struct iommu_unit *dmar; + struct iommu_domain *old_domain; + iommu_device_entry_t *ctxp; struct sf_buf *sf; int error; - dmar = domain->dmar; + dmar = domain->iommu; old_domain = ctx->domain; if (domain == old_domain) return (0); - KASSERT(old_domain->dmar == dmar, + KASSERT(old_domain->iommu == dmar, ("domain %p %u moving between dmars %u %u", domain, - domain->domain, old_domain->dmar->unit, domain->dmar->unit)); + domain->domain, old_domain->iommu->unit, domain->iommu->unit)); TD_PREP_PINNED_ASSERT; ctxp = dmar_map_ctx_entry(ctx, &sf); - DMAR_LOCK(dmar); - dmar_ctx_unlink(ctx); + IOMMU_LOCK(dmar); + dmar_device_unlink(ctx); ctx->domain = domain; - dmar_ctx_link(ctx); + dmar_device_link(ctx); ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); dmar_unmap_pgtbl(sf); error = dmar_flush_for_ctx_entry(dmar, true); @@ -634,10 +634,10 @@ } static void -dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) +dmar_unref_domain_locked(struct iommu_unit *dmar, struct iommu_domain *domain) { - DMAR_ASSERT_LOCKED(dmar); + IOMMU_ASSERT_LOCKED(dmar); KASSERT(domain->refs >= 1, ("dmar %d domain %p refs %u", dmar->unit, domain, domain->refs)); KASSERT(domain->refs > domain->ctx_cnt, @@ -646,7 +646,7 @@ if (domain->refs > 1) { domain->refs--; - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); return; } @@ -654,20 +654,20 @@ ("lost ref on RMRR domain %p", domain)); LIST_REMOVE(domain, link); - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); taskqueue_drain(dmar->delayed_taskqueue, &domain->unload_task); dmar_domain_destroy(domain); } void -dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) +dmar_free_ctx_locked(struct iommu_unit *dmar, struct iommu_device *ctx) { struct sf_buf *sf; - dmar_ctx_entry_t *ctxp; - struct dmar_domain *domain; + iommu_device_entry_t *ctxp; + struct iommu_domain *domain; - DMAR_ASSERT_LOCKED(dmar); + IOMMU_ASSERT_LOCKED(dmar); KASSERT(ctx->refs >= 1, ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); @@ -677,11 +677,11 @@ */ if (ctx->refs > 1) { ctx->refs--; - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); return; } - KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, + KASSERT((ctx->flags & IOMMU_DEVICE_DISABLED) == 0, ("lost ref on disabled ctx %p", ctx)); /* @@ -689,10 +689,10 @@ * page table is destroyed. The mapping of the context * entries page could require sleep, unlock the dmar. */ - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); TD_PREP_PINNED_ASSERT; ctxp = dmar_map_ctx_entry(ctx, &sf); - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); KASSERT(ctx->refs >= 1, ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); @@ -702,13 +702,13 @@ */ if (ctx->refs > 1) { ctx->refs--; - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; return; } - KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, + KASSERT((ctx->flags & IOMMU_DEVICE_DISABLED) == 0, ("lost ref on disabled ctx %p", ctx)); /* @@ -727,32 +727,32 @@ } dmar_unmap_pgtbl(sf); domain = ctx->domain; - dmar_ctx_unlink(ctx); + dmar_device_unlink(ctx); free(ctx, M_DMAR_CTX); dmar_unref_domain_locked(dmar, domain); TD_PINNED_ASSERT; } void -dmar_free_ctx(struct dmar_ctx *ctx) +dmar_free_ctx(struct iommu_device *ctx) { - struct dmar_unit *dmar; + struct iommu_unit *dmar; - dmar = ctx->domain->dmar; - DMAR_LOCK(dmar); + dmar = ctx->domain->iommu; + IOMMU_LOCK(dmar); dmar_free_ctx_locked(dmar, ctx); } /* * Returns with the domain locked. */ -struct dmar_ctx * -dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) +struct iommu_device * +dmar_find_ctx_locked(struct iommu_unit *dmar, uint16_t rid) { - struct dmar_domain *domain; - struct dmar_ctx *ctx; + struct iommu_domain *domain; + struct iommu_device *ctx; - DMAR_ASSERT_LOCKED(dmar); + IOMMU_ASSERT_LOCKED(dmar); LIST_FOREACH(domain, &dmar->domains, link) { LIST_FOREACH(ctx, &domain->contexts, link) { @@ -764,17 +764,17 @@ } void -dmar_domain_free_entry(struct dmar_map_entry *entry, bool free) +dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) { - struct dmar_domain *domain; + struct iommu_domain *domain; domain = entry->domain; - DMAR_DOMAIN_LOCK(domain); - if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) + IOMMU_DOMAIN_LOCK(domain); + if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) dmar_gas_free_region(domain, entry); else dmar_gas_free_space(domain, entry); - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); if (free) dmar_gas_free_entry(domain, entry); else @@ -782,19 +782,19 @@ } void -dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free) +dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free) { - struct dmar_unit *unit; + struct iommu_unit *unit; - unit = entry->domain->dmar; + unit = entry->domain->iommu; if (unit->qi_enabled) { - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_qi_invalidate_locked(entry->domain, entry->start, entry->end - entry->start, &entry->gseq, true); if (!free) - entry->flags |= DMAR_MAP_ENTRY_QI_NF; + entry->flags |= IOMMU_MAP_ENTRY_QI_NF; TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } else { domain_flush_iotlb_sync(entry->domain, entry->start, entry->end - entry->start); @@ -803,8 +803,8 @@ } static bool -dmar_domain_unload_emit_wait(struct dmar_domain *domain, - struct dmar_map_entry *entry) +dmar_domain_unload_emit_wait(struct iommu_domain *domain, + struct iommu_map_entry *entry) { if (TAILQ_NEXT(entry, dmamap_link) == NULL) @@ -813,17 +813,17 @@ } void -dmar_domain_unload(struct dmar_domain *domain, - struct dmar_map_entries_tailq *entries, bool cansleep) +dmar_domain_unload(struct iommu_domain *domain, + struct iommu_map_entries_tailq *entries, bool cansleep) { - struct dmar_unit *unit; - struct dmar_map_entry *entry, *entry1; + struct iommu_unit *unit; + struct iommu_map_entry *entry, *entry1; int error; - unit = domain->dmar; + unit = domain->iommu; TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { - KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, + KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, ("not mapped entry %p %p", domain, entry)); error = domain_unmap_buf(domain, entry->start, entry->end - entry->start, cansleep ? DMAR_PGF_WAITOK : 0); @@ -839,30 +839,30 @@ return; KASSERT(unit->qi_enabled, ("loaded entry left")); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); TAILQ_FOREACH(entry, entries, dmamap_link) { dmar_qi_invalidate_locked(domain, entry->start, entry->end - entry->start, &entry->gseq, dmar_domain_unload_emit_wait(domain, entry)); } TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } static void dmar_domain_unload_task(void *arg, int pending) { - struct dmar_domain *domain; - struct dmar_map_entries_tailq entries; + struct iommu_domain *domain; + struct iommu_map_entries_tailq entries; domain = arg; TAILQ_INIT(&entries); for (;;) { - DMAR_DOMAIN_LOCK(domain); - TAILQ_SWAP(&domain->unload_entries, &entries, dmar_map_entry, + IOMMU_DOMAIN_LOCK(domain); + TAILQ_SWAP(&domain->unload_entries, &entries, iommu_map_entry, dmamap_link); - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); if (TAILQ_EMPTY(&entries)) break; dmar_domain_unload(domain, &entries, true); Index: sys/x86/iommu/intel_dmar.h =================================================================== --- sys/x86/iommu/intel_dmar.h +++ sys/x86/iommu/intel_dmar.h @@ -44,7 +44,7 @@ uint32_t seq; }; -struct dmar_map_entry { +struct iommu_map_entry { dmar_gaddr_t start; dmar_gaddr_t end; dmar_gaddr_t first; /* Least start in subtree */ @@ -52,29 +52,29 @@ dmar_gaddr_t free_down; /* Max free space below the current R/B tree node */ u_int flags; - TAILQ_ENTRY(dmar_map_entry) dmamap_link; /* Link for dmamap entries */ - RB_ENTRY(dmar_map_entry) rb_entry; /* Links for domain entries */ - TAILQ_ENTRY(dmar_map_entry) unroll_link; /* Link for unroll after + TAILQ_ENTRY(iommu_map_entry) dmamap_link; /* Link for dmamap entries */ + RB_ENTRY(iommu_map_entry) rb_entry; /* Links for domain entries */ + TAILQ_ENTRY(iommu_map_entry) unroll_link; /* Link for unroll after dmamap_load failure */ - struct dmar_domain *domain; + struct iommu_domain *domain; struct dmar_qi_genseq gseq; }; -RB_HEAD(dmar_gas_entries_tree, dmar_map_entry); -RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, +RB_HEAD(dmar_gas_entries_tree, iommu_map_entry); +RB_PROTOTYPE(dmar_gas_entries_tree, iommu_map_entry, rb_entry, dmar_gas_cmp_entries); -#define DMAR_MAP_ENTRY_PLACE 0x0001 /* Fake entry */ -#define DMAR_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by +#define IOMMU_MAP_ENTRY_PLACE 0x0001 /* Fake entry */ +#define IOMMU_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by dmamap_link */ -#define DMAR_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by +#define IOMMU_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by dmamap_link */ -#define DMAR_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */ -#define DMAR_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */ -#define DMAR_MAP_ENTRY_READ 0x1000 /* Read permitted */ -#define DMAR_MAP_ENTRY_WRITE 0x2000 /* Write permitted */ -#define DMAR_MAP_ENTRY_SNOOP 0x4000 /* Snoop */ -#define DMAR_MAP_ENTRY_TM 0x8000 /* Transient */ +#define IOMMU_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */ +#define IOMMU_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */ +#define IOMMU_MAP_ENTRY_READ 0x1000 /* Read permitted */ +#define IOMMU_MAP_ENTRY_WRITE 0x2000 /* Write permitted */ +#define IOMMU_MAP_ENTRY_SNOOP 0x4000 /* Snoop */ +#define IOMMU_MAP_ENTRY_TM 0x8000 /* Transient */ /* * Locking annotations: @@ -94,7 +94,7 @@ * Page tables pages and pages content is protected by the vm object * lock pgtbl_obj, which contains the page tables pages. */ -struct dmar_domain { +struct iommu_domain { int domain; /* (c) DID, written in context entry */ int mgaw; /* (c) Real max address width */ int agaw; /* (c) Adjusted guest address width */ @@ -105,27 +105,27 @@ the guest AS */ u_int ctx_cnt; /* (u) Number of contexts owned */ u_int refs; /* (u) Refs, including ctx */ - struct dmar_unit *dmar; /* (c) */ + struct iommu_unit *iommu; /* (c) */ struct mtx lock; /* (c) */ - LIST_ENTRY(dmar_domain) link; /* (u) Member in the dmar list */ - LIST_HEAD(, dmar_ctx) contexts; /* (u) */ + LIST_ENTRY(iommu_domain) link; /* (u) Member in the dmar list */ + LIST_HEAD(, iommu_device) contexts; /* (u) */ vm_object_t pgtbl_obj; /* (c) Page table pages */ u_int flags; /* (u) */ u_int entries_cnt; /* (d) */ struct dmar_gas_entries_tree rb_root; /* (d) */ - struct dmar_map_entries_tailq unload_entries; /* (d) Entries to - unload */ - struct dmar_map_entry *first_place, *last_place; /* (d) */ + struct iommu_map_entries_tailq unload_entries; /* (d) Entries to + unload */ + struct iommu_map_entry *first_place, *last_place; /* (d) */ struct task unload_task; /* (c) */ u_int batch_no; }; -struct dmar_ctx { - struct bus_dma_tag_dmar ctx_tag; /* (c) Root tag */ +struct iommu_device { + struct bus_dma_tag_iommu device_tag; /* (c) Root tag */ uint16_t rid; /* (c) pci RID */ uint64_t last_fault_rec[2]; /* Last fault reported */ - struct dmar_domain *domain; /* (c) */ - LIST_ENTRY(dmar_ctx) link; /* (u) Member in the domain list */ + struct iommu_domain *domain; /* (c) */ + LIST_ENTRY(iommu_device) link; /* (u) Member in the domain list */ u_int refs; /* (u) References from tags */ u_int flags; /* (u) */ u_long loads; /* atomic updates, for stat only */ @@ -139,10 +139,10 @@ #define DMAR_DOMAIN_RMRR 0x0020 /* Domain contains RMRR entry, cannot be turned off */ -/* struct dmar_ctx flags */ -#define DMAR_CTX_FAULTED 0x0001 /* Fault was reported, +/* struct iommu_device flags */ +#define IOMMU_DEVICE_FAULTED 0x0001 /* Fault was reported, last_fault_rec is valid */ -#define DMAR_CTX_DISABLED 0x0002 /* Device is disabled, the +#define IOMMU_DEVICE_DISABLED 0x0002 /* Device is disabled, the ephemeral reference is kept to prevent context destruction */ @@ -152,9 +152,9 @@ #define DMAR_DOMAIN_ASSERT_PGLOCKED(dom) \ VM_OBJECT_ASSERT_WLOCKED((dom)->pgtbl_obj) -#define DMAR_DOMAIN_LOCK(dom) mtx_lock(&(dom)->lock) -#define DMAR_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->lock) -#define DMAR_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->lock, MA_OWNED) +#define IOMMU_DOMAIN_LOCK(dom) mtx_lock(&(dom)->lock) +#define IOMMU_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->lock) +#define IOMMU_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->lock, MA_OWNED) struct dmar_msi_data { int irq; @@ -165,8 +165,8 @@ int msi_data_reg; int msi_addr_reg; int msi_uaddr_reg; - void (*enable_intr)(struct dmar_unit *); - void (*disable_intr)(struct dmar_unit *); + void (*enable_intr)(struct iommu_unit *); + void (*disable_intr)(struct iommu_unit *); const char *name; }; @@ -174,7 +174,7 @@ #define DMAR_INTR_QI 1 #define DMAR_INTR_TOTAL 2 -struct dmar_unit { +struct iommu_unit { device_t dev; int unit; uint16_t segment; @@ -194,7 +194,7 @@ /* Data for being a dmar */ struct mtx lock; - LIST_HEAD(, dmar_domain) domains; + LIST_HEAD(, iommu_domain) domains; struct unrhdr *domids; vm_object_t ctx_obj; u_int barrier_flags; @@ -230,13 +230,13 @@ vmem_t *irtids; /* Delayed freeing of map entries queue processing */ - struct dmar_map_entries_tailq tlb_flush_entries; + struct iommu_map_entries_tailq tlb_flush_entries; struct task qi_task; struct taskqueue *qi_taskqueue; /* Busdma delayed map load */ struct task dmamap_load_task; - TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps; + TAILQ_HEAD(, bus_dmamap_iommu) delayed_maps; struct taskqueue *delayed_taskqueue; int dma_enabled; @@ -251,9 +251,9 @@ }; -#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock) -#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->lock) -#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED) +#define IOMMU_LOCK(dmar) mtx_lock(&(dmar)->lock) +#define IOMMU_UNLOCK(dmar) mtx_unlock(&(dmar)->lock) +#define IOMMU_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED) #define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock) #define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock) @@ -268,131 +268,127 @@ #define DMAR_BARRIER_RMRR 0 #define DMAR_BARRIER_USEQ 1 -struct dmar_unit *dmar_find(device_t dev, bool verbose); -struct dmar_unit *dmar_find_hpet(device_t dev, uint16_t *rid); -struct dmar_unit *dmar_find_ioapic(u_int apic_id, uint16_t *rid); +struct iommu_unit *dmar_find(device_t dev, bool verbose); +struct iommu_unit *dmar_find_hpet(device_t dev, uint16_t *rid); +struct iommu_unit *dmar_find_ioapic(u_int apic_id, uint16_t *rid); u_int dmar_nd2mask(u_int nd); -bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl); -int domain_set_agaw(struct dmar_domain *domain, int mgaw); -int dmar_maxaddr2mgaw(struct dmar_unit *unit, dmar_gaddr_t maxaddr, +bool dmar_pglvl_supported(struct iommu_unit *unit, int pglvl); +int domain_set_agaw(struct iommu_domain *domain, int mgaw); +int dmar_maxaddr2mgaw(struct iommu_unit *unit, dmar_gaddr_t maxaddr, bool allow_less); vm_pindex_t pglvl_max_pages(int pglvl); -int domain_is_sp_lvl(struct dmar_domain *domain, int lvl); +int domain_is_sp_lvl(struct iommu_domain *domain, int lvl); dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl); -dmar_gaddr_t domain_page_size(struct dmar_domain *domain, int lvl); -int calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, +dmar_gaddr_t domain_page_size(struct iommu_domain *domain, int lvl); +int calc_am(struct iommu_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, dmar_gaddr_t *isizep); struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags); void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags); void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, struct sf_buf **sf); void dmar_unmap_pgtbl(struct sf_buf *sf); -int dmar_load_root_entry_ptr(struct dmar_unit *unit); -int dmar_inv_ctx_glob(struct dmar_unit *unit); -int dmar_inv_iotlb_glob(struct dmar_unit *unit); -int dmar_flush_write_bufs(struct dmar_unit *unit); -void dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst); -void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst); -void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst); -int dmar_enable_translation(struct dmar_unit *unit); -int dmar_disable_translation(struct dmar_unit *unit); -int dmar_load_irt_ptr(struct dmar_unit *unit); -int dmar_enable_ir(struct dmar_unit *unit); -int dmar_disable_ir(struct dmar_unit *unit); -bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id); -void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id); +int dmar_load_root_entry_ptr(struct iommu_unit *unit); +int dmar_inv_ctx_glob(struct iommu_unit *unit); +int dmar_inv_iotlb_glob(struct iommu_unit *unit); +int dmar_flush_write_bufs(struct iommu_unit *unit); +void dmar_flush_pte_to_ram(struct iommu_unit *unit, dmar_pte_t *dst); +void dmar_flush_ctx_to_ram(struct iommu_unit *unit, iommu_device_entry_t *dst); +void dmar_flush_root_to_ram(struct iommu_unit *unit, dmar_root_entry_t *dst); +int dmar_enable_translation(struct iommu_unit *unit); +int dmar_disable_translation(struct iommu_unit *unit); +int dmar_load_irt_ptr(struct iommu_unit *unit); +int dmar_enable_ir(struct iommu_unit *unit); +int dmar_disable_ir(struct iommu_unit *unit); +bool dmar_barrier_enter(struct iommu_unit *dmar, u_int barrier_id); +void dmar_barrier_exit(struct iommu_unit *dmar, u_int barrier_id); uint64_t dmar_get_timeout(void); void dmar_update_timeout(uint64_t newval); int dmar_fault_intr(void *arg); -void dmar_enable_fault_intr(struct dmar_unit *unit); -void dmar_disable_fault_intr(struct dmar_unit *unit); -int dmar_init_fault_log(struct dmar_unit *unit); -void dmar_fini_fault_log(struct dmar_unit *unit); +void dmar_enable_fault_intr(struct iommu_unit *unit); +void dmar_disable_fault_intr(struct iommu_unit *unit); +int dmar_init_fault_log(struct iommu_unit *unit); +void dmar_fini_fault_log(struct iommu_unit *unit); int dmar_qi_intr(void *arg); -void dmar_enable_qi_intr(struct dmar_unit *unit); -void dmar_disable_qi_intr(struct dmar_unit *unit); -int dmar_init_qi(struct dmar_unit *unit); -void dmar_fini_qi(struct dmar_unit *unit); -void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start, +void dmar_enable_qi_intr(struct iommu_unit *unit); +void dmar_disable_qi_intr(struct iommu_unit *unit); +int dmar_init_qi(struct iommu_unit *unit); +void dmar_fini_qi(struct iommu_unit *unit); +void dmar_qi_invalidate_locked(struct iommu_domain *domain, dmar_gaddr_t start, dmar_gaddr_t size, struct dmar_qi_genseq *psec, bool emit_wait); -void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit); -void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit); -void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit); -void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt); +void dmar_qi_invalidate_ctx_glob_locked(struct iommu_unit *unit); +void dmar_qi_invalidate_iotlb_glob_locked(struct iommu_unit *unit); +void dmar_qi_invalidate_iec_glob(struct iommu_unit *unit); +void dmar_qi_invalidate_iec(struct iommu_unit *unit, u_int start, u_int cnt); -vm_object_t domain_get_idmap_pgtbl(struct dmar_domain *domain, +vm_object_t domain_get_idmap_pgtbl(struct iommu_domain *domain, dmar_gaddr_t maxaddr); void put_idmap_pgtbl(vm_object_t obj); -int domain_map_buf(struct dmar_domain *domain, dmar_gaddr_t base, +int domain_map_buf(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags); -int domain_unmap_buf(struct dmar_domain *domain, dmar_gaddr_t base, +int domain_unmap_buf(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, int flags); -void domain_flush_iotlb_sync(struct dmar_domain *domain, dmar_gaddr_t base, +void domain_flush_iotlb_sync(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size); -int domain_alloc_pgtbl(struct dmar_domain *domain); -void domain_free_pgtbl(struct dmar_domain *domain); +int domain_alloc_pgtbl(struct iommu_domain *domain); +void domain_free_pgtbl(struct iommu_domain *domain); int dmar_dev_depth(device_t child); void dmar_dev_path(device_t child, int *busno, void *path1, int depth); -struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, - bool rmrr); -struct dmar_ctx *dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, +struct iommu_device *dmar_get_ctx_for_dev(struct iommu_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init); -struct dmar_ctx *dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, - int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, - bool id_mapped, bool rmrr_init); -int dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx); -void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx); -void dmar_free_ctx(struct dmar_ctx *ctx); -struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid); -void dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free); -void dmar_domain_unload(struct dmar_domain *domain, - struct dmar_map_entries_tailq *entries, bool cansleep); -void dmar_domain_free_entry(struct dmar_map_entry *entry, bool free); - -int dmar_init_busdma(struct dmar_unit *unit); -void dmar_fini_busdma(struct dmar_unit *unit); -device_t dmar_get_requester(device_t dev, uint16_t *rid); - -void dmar_gas_init_domain(struct dmar_domain *domain); -void dmar_gas_fini_domain(struct dmar_domain *domain); -struct dmar_map_entry *dmar_gas_alloc_entry(struct dmar_domain *domain, +struct iommu_device *dmar_get_ctx_for_devpath(struct iommu_unit *dmar, + uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, + int dev_path_len, bool id_mapped, bool rmrr_init); +int dmar_move_ctx_to_domain(struct iommu_domain *domain, + struct iommu_device *ctx); +void dmar_free_ctx_locked(struct iommu_unit *dmar, struct iommu_device *ctx); +void dmar_free_ctx(struct iommu_device *ctx); +struct iommu_device *dmar_find_ctx_locked(struct iommu_unit *dmar, + uint16_t rid); +void dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free); +void dmar_domain_unload(struct iommu_domain *domain, + struct iommu_map_entries_tailq *entries, bool cansleep); +void dmar_domain_free_entry(struct iommu_map_entry *entry, bool free); + +void dmar_gas_init_domain(struct iommu_domain *domain); +void dmar_gas_fini_domain(struct iommu_domain *domain); +struct iommu_map_entry *dmar_gas_alloc_entry(struct iommu_domain *domain, u_int flags); -void dmar_gas_free_entry(struct dmar_domain *domain, - struct dmar_map_entry *entry); -void dmar_gas_free_space(struct dmar_domain *domain, - struct dmar_map_entry *entry); -int dmar_gas_map(struct dmar_domain *domain, +void dmar_gas_free_entry(struct iommu_domain *domain, + struct iommu_map_entry *entry); +void dmar_gas_free_space(struct iommu_domain *domain, + struct iommu_map_entry *entry); +int dmar_gas_map(struct iommu_domain *domain, const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, - u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res); -void dmar_gas_free_region(struct dmar_domain *domain, - struct dmar_map_entry *entry); -int dmar_gas_map_region(struct dmar_domain *domain, - struct dmar_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma); -int dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, + u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res); +void dmar_gas_free_region(struct iommu_domain *domain, + struct iommu_map_entry *entry); +int dmar_gas_map_region(struct iommu_domain *domain, + struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma); +int dmar_gas_reserve_region(struct iommu_domain *domain, dmar_gaddr_t start, dmar_gaddr_t end); -void dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain, +void dmar_dev_parse_rmrr(struct iommu_domain *domain, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, - struct dmar_map_entries_tailq *rmrr_entries); -int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar); + struct iommu_map_entries_tailq *rmrr_entries); +int dmar_instantiate_rmrr_ctxs(struct iommu_unit *dmar); -void dmar_quirks_post_ident(struct dmar_unit *dmar); -void dmar_quirks_pre_use(struct dmar_unit *dmar); +void dmar_quirks_post_ident(struct iommu_unit *dmar); +void dmar_quirks_pre_use(struct iommu_unit *dmar); -int dmar_init_irt(struct dmar_unit *unit); -void dmar_fini_irt(struct dmar_unit *unit); +int dmar_init_irt(struct iommu_unit *unit); +void dmar_fini_irt(struct iommu_unit *unit); -void dmar_set_buswide_ctx(struct dmar_unit *unit, u_int busno); -bool dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno); +void dmar_set_buswide_ctx(struct iommu_unit *unit, u_int busno); +bool dmar_is_buswide_ctx(struct iommu_unit *unit, u_int busno); -#define DMAR_GM_CANWAIT 0x0001 -#define DMAR_GM_CANSPLIT 0x0002 -#define DMAR_GM_RMRR 0x0004 +#define IOMMU_MF_CANWAIT 0x0001 +#define IOMMU_MF_CANSPLIT 0x0002 +#define IOMMU_MF_RMRR 0x0004 #define DMAR_PGF_WAITOK 0x0001 #define DMAR_PGF_ZERO 0x0002 @@ -407,14 +403,14 @@ extern int dmar_check_free; static inline uint32_t -dmar_read4(const struct dmar_unit *unit, int reg) +dmar_read4(const struct iommu_unit *unit, int reg) { return (bus_read_4(unit->regs, reg)); } static inline uint64_t -dmar_read8(const struct dmar_unit *unit, int reg) +dmar_read8(const struct iommu_unit *unit, int reg) { #ifdef __i386__ uint32_t high, low; @@ -428,7 +424,7 @@ } static inline void -dmar_write4(const struct dmar_unit *unit, int reg, uint32_t val) +dmar_write4(const struct iommu_unit *unit, int reg, uint32_t val) { KASSERT(reg != DMAR_GCMD_REG || (val & DMAR_GCMD_TE) == @@ -439,7 +435,7 @@ } static inline void -dmar_write8(const struct dmar_unit *unit, int reg, uint64_t val) +dmar_write8(const struct iommu_unit *unit, int reg, uint64_t val) { KASSERT(reg != DMAR_GCMD_REG, ("8byte GCMD write")); @@ -522,7 +518,7 @@ } static inline bool -dmar_test_boundary(dmar_gaddr_t start, dmar_gaddr_t size, +iommu_test_boundary(dmar_gaddr_t start, dmar_gaddr_t size, dmar_gaddr_t boundary) { Index: sys/x86/iommu/intel_drv.c =================================================================== --- sys/x86/iommu/intel_drv.c +++ sys/x86/iommu/intel_drv.c @@ -232,7 +232,7 @@ } static void -dmar_release_intr(device_t dev, struct dmar_unit *unit, int idx) +dmar_release_intr(device_t dev, struct iommu_unit *unit, int idx) { struct dmar_msi_data *dmd; @@ -248,11 +248,11 @@ } static void -dmar_release_resources(device_t dev, struct dmar_unit *unit) +dmar_release_resources(device_t dev, struct iommu_unit *unit) { int i; - dmar_fini_busdma(unit); + iommu_fini_busdma(unit); dmar_fini_irt(unit); dmar_fini_qi(unit); dmar_fini_fault_log(unit); @@ -276,7 +276,7 @@ } static int -dmar_alloc_irq(device_t dev, struct dmar_unit *unit, int idx) +dmar_alloc_irq(device_t dev, struct iommu_unit *unit, int idx) { device_t pcib; struct dmar_msi_data *dmd; @@ -344,7 +344,7 @@ static int dmar_remap_intr(device_t dev, device_t child, u_int irq) { - struct dmar_unit *unit; + struct iommu_unit *unit; struct dmar_msi_data *dmd; uint64_t msi_addr; uint32_t msi_data; @@ -359,13 +359,13 @@ dev, irq, &msi_addr, &msi_data); if (error != 0) return (error); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); (dmd->disable_intr)(unit); dmar_write4(unit, dmd->msi_data_reg, msi_data); dmar_write4(unit, dmd->msi_addr_reg, msi_addr); dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32); (dmd->enable_intr)(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); return (0); } } @@ -374,7 +374,7 @@ #endif static void -dmar_print_caps(device_t dev, struct dmar_unit *unit, +dmar_print_caps(device_t dev, struct iommu_unit *unit, ACPI_DMAR_HARDWARE_UNIT *dmaru) { uint32_t caphi, ecaphi; @@ -406,7 +406,7 @@ static int dmar_attach(device_t dev) { - struct dmar_unit *unit; + struct iommu_unit *unit; ACPI_DMAR_HARDWARE_UNIT *dmaru; uint64_t timeout; int i, error; @@ -493,29 +493,29 @@ * done. */ dmar_pgalloc(unit->ctx_obj, 0, DMAR_PGF_WAITOK | DMAR_PGF_ZERO); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); error = dmar_load_root_entry_ptr(unit); if (error != 0) { - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } error = dmar_inv_ctx_glob(unit); if (error != 0) { - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } if ((unit->hw_ecap & DMAR_ECAP_DI) != 0) { error = dmar_inv_iotlb_glob(unit); if (error != 0) { - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } } - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); error = dmar_init_fault_log(unit); if (error != 0) { dmar_release_resources(dev, unit); @@ -531,21 +531,21 @@ dmar_release_resources(dev, unit); return (error); } - error = dmar_init_busdma(unit); + error = iommu_init_busdma(unit); if (error != 0) { dmar_release_resources(dev, unit); return (error); } #ifdef NOTYET - DMAR_LOCK(unit); + IOMMU_LOCK(unit); error = dmar_enable_translation(unit); if (error != 0) { - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); #endif return (0); @@ -589,25 +589,25 @@ static driver_t dmar_driver = { "dmar", dmar_methods, - sizeof(struct dmar_unit), + sizeof(struct iommu_unit), }; DRIVER_MODULE(dmar, acpi, dmar_driver, dmar_devclass, 0, 0); MODULE_DEPEND(dmar, acpi, 1, 1, 1); void -dmar_set_buswide_ctx(struct dmar_unit *unit, u_int busno) +dmar_set_buswide_ctx(struct iommu_unit *unit, u_int busno) { MPASS(busno <= PCI_BUSMAX); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |= 1 << (busno % (NBBY * sizeof(uint32_t))); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } bool -dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno) +dmar_is_buswide_ctx(struct iommu_unit *unit, u_int busno) { MPASS(busno <= PCI_BUSMAX); @@ -728,7 +728,7 @@ } static bool -dmar_match_by_path(struct dmar_unit *unit, int dev_domain, int dev_busno, +dmar_match_by_path(struct iommu_unit *unit, int dev_domain, int dev_busno, const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len, const char **banner) { ACPI_DMAR_HARDWARE_UNIT *dmarh; @@ -764,11 +764,11 @@ return (false); } -static struct dmar_unit * +static struct iommu_unit * dmar_find_by_scope(int dev_domain, int dev_busno, const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len) { - struct dmar_unit *unit; + struct iommu_unit *unit; int i; for (i = 0; i < dmar_devcnt; i++) { @@ -782,11 +782,11 @@ return (NULL); } -struct dmar_unit * +struct iommu_unit * dmar_find(device_t dev, bool verbose) { device_t dmar_dev; - struct dmar_unit *unit; + struct iommu_unit *unit; const char *banner; int i, dev_domain, dev_busno, dev_path_len; @@ -826,11 +826,11 @@ return (unit); } -static struct dmar_unit * +static struct iommu_unit * dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid) { device_t dmar_dev; - struct dmar_unit *unit; + struct iommu_unit *unit; ACPI_DMAR_HARDWARE_UNIT *dmarh; ACPI_DMAR_DEVICE_SCOPE *devscope; ACPI_DMAR_PCI_PATH *path; @@ -844,7 +844,7 @@ dmar_dev = dmar_devs[i]; if (dmar_dev == NULL) continue; - unit = (struct dmar_unit *)device_get_softc(dmar_dev); + unit = (struct iommu_unit *)device_get_softc(dmar_dev); dmarh = dmar_find_by_index(i); if (dmarh == NULL) continue; @@ -890,7 +890,7 @@ } -struct dmar_unit * +struct iommu_unit * dmar_find_hpet(device_t dev, uint16_t *rid) { @@ -898,7 +898,7 @@ rid)); } -struct dmar_unit * +struct iommu_unit * dmar_find_ioapic(u_int apic_id, uint16_t *rid) { @@ -906,12 +906,12 @@ } struct rmrr_iter_args { - struct dmar_domain *domain; + struct iommu_domain *domain; int dev_domain; int dev_busno; const ACPI_DMAR_PCI_PATH *dev_path; int dev_path_len; - struct dmar_map_entries_tailq *rmrr_entries; + struct iommu_map_entries_tailq *rmrr_entries; }; static int @@ -920,7 +920,7 @@ struct rmrr_iter_args *ria; ACPI_DMAR_RESERVED_MEMORY *resmem; ACPI_DMAR_DEVICE_SCOPE *devscope; - struct dmar_map_entry *entry; + struct iommu_map_entry *entry; char *ptr, *ptrend; int match; @@ -956,9 +956,9 @@ } void -dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain, int dev_busno, +dmar_dev_parse_rmrr(struct iommu_domain *domain, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, - struct dmar_map_entries_tailq *rmrr_entries) + struct iommu_map_entries_tailq *rmrr_entries) { struct rmrr_iter_args ria; @@ -972,7 +972,7 @@ } struct inst_rmrr_iter_args { - struct dmar_unit *dmar; + struct iommu_unit *dmar; }; static device_t @@ -1004,7 +1004,7 @@ struct inst_rmrr_iter_args *iria; const char *ptr, *ptrend; device_t dev; - struct dmar_unit *unit; + struct iommu_unit *unit; int dev_path_len; uint16_t rid; @@ -1059,7 +1059,7 @@ unit = dmar_find(dev, false); if (iria->dmar != unit) continue; - dmar_instantiate_ctx(iria->dmar, dev, true); + iommu_instantiate_device(iria->dmar, dev, true); } } @@ -1071,7 +1071,7 @@ * Pre-create all contexts for the DMAR which have RMRR entries. */ int -dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar) +dmar_instantiate_rmrr_ctxs(struct iommu_unit *dmar) { struct inst_rmrr_iter_args iria; int error; @@ -1082,7 +1082,7 @@ error = 0; iria.dmar = dmar; dmar_iterate_tbl(dmar_inst_rmrr_iter, &iria); - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); if (!LIST_EMPTY(&dmar->domains)) { KASSERT((dmar->hw_gcmd & DMAR_GCMD_TE) == 0, ("dmar%d: RMRR not handled but translation is already enabled", @@ -1107,9 +1107,9 @@ #include static void -dmar_print_domain_entry(const struct dmar_map_entry *entry) +dmar_print_domain_entry(const struct iommu_map_entry *entry) { - struct dmar_map_entry *l, *r; + struct iommu_map_entry *l, *r; db_printf( " start %jx end %jx first %jx last %jx free_down %jx flags %x ", @@ -1131,22 +1131,22 @@ } static void -dmar_print_ctx(struct dmar_ctx *ctx) +dmar_print_ctx(struct iommu_device *ctx) { db_printf( " @%p pci%d:%d:%d refs %d flags %x loads %lu unloads %lu\n", - ctx, pci_get_bus(ctx->ctx_tag.owner), - pci_get_slot(ctx->ctx_tag.owner), - pci_get_function(ctx->ctx_tag.owner), ctx->refs, ctx->flags, + ctx, pci_get_bus(ctx->device_tag.owner), + pci_get_slot(ctx->device_tag.owner), + pci_get_function(ctx->device_tag.owner), ctx->refs, ctx->flags, ctx->loads, ctx->unloads); } static void -dmar_print_domain(struct dmar_domain *domain, bool show_mappings) +dmar_print_domain(struct iommu_domain *domain, bool show_mappings) { - struct dmar_map_entry *entry; - struct dmar_ctx *ctx; + struct iommu_map_entry *entry; + struct iommu_device *ctx; db_printf( " @%p dom %d mgaw %d agaw %d pglvl %d end %jx refs %d\n" @@ -1177,11 +1177,11 @@ } } -DB_FUNC(dmar_domain, db_dmar_print_domain, db_show_table, CS_OWN, NULL) +DB_FUNC(iommu_domain, db_dmar_print_domain, db_show_table, CS_OWN, NULL) { - struct dmar_unit *unit; - struct dmar_domain *domain; - struct dmar_ctx *ctx; + struct iommu_unit *unit; + struct iommu_domain *domain; + struct iommu_device *ctx; bool show_mappings, valid; int pci_domain, bus, device, function, i, t; db_expr_t radix; @@ -1222,7 +1222,7 @@ db_radix = radix; db_skip_to_eol(); if (!valid) { - db_printf("usage: show dmar_domain [/m] " + db_printf("usage: show iommu_domain [/m] " " \n"); return; } @@ -1231,11 +1231,11 @@ LIST_FOREACH(domain, &unit->domains, link) { LIST_FOREACH(ctx, &domain->contexts, link) { if (pci_domain == unit->segment && - bus == pci_get_bus(ctx->ctx_tag.owner) && + bus == pci_get_bus(ctx->device_tag.owner) && device == - pci_get_slot(ctx->ctx_tag.owner) && + pci_get_slot(ctx->device_tag.owner) && function == - pci_get_function(ctx->ctx_tag.owner)) { + pci_get_function(ctx->device_tag.owner)) { dmar_print_domain(domain, show_mappings); goto out; @@ -1249,8 +1249,8 @@ static void dmar_print_one(int idx, bool show_domains, bool show_mappings) { - struct dmar_unit *unit; - struct dmar_domain *domain; + struct iommu_unit *unit; + struct iommu_domain *domain; int i, frir; unit = device_get_softc(dmar_devs[idx]); Index: sys/x86/iommu/intel_fault.c =================================================================== --- sys/x86/iommu/intel_fault.c +++ sys/x86/iommu/intel_fault.c @@ -80,7 +80,7 @@ */ static int -dmar_fault_next(struct dmar_unit *unit, int faultp) +dmar_fault_next(struct iommu_unit *unit, int faultp) { faultp += 2; @@ -90,7 +90,7 @@ } static void -dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) +dmar_fault_intr_clear(struct iommu_unit *unit, uint32_t fsts) { uint32_t clear; @@ -124,7 +124,7 @@ int dmar_fault_intr(void *arg) { - struct dmar_unit *unit; + struct iommu_unit *unit; uint64_t fault_rec[2]; uint32_t fsts; int fri, frir, faultp; @@ -190,8 +190,8 @@ static void dmar_fault_task(void *arg, int pending __unused) { - struct dmar_unit *unit; - struct dmar_ctx *ctx; + struct iommu_unit *unit; + struct iommu_device *ctx; uint64_t fault_rec[2]; int sid, bus, slot, func, faultp; @@ -209,7 +209,7 @@ sid = DMAR_FRCD2_SID(fault_rec[1]); printf("DMAR%d: ", unit->unit); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); ctx = dmar_find_ctx_locked(unit, sid); if (ctx == NULL) { printf(":"); @@ -223,15 +223,15 @@ slot = PCI_RID2SLOT(sid); func = PCI_RID2FUNC(sid); } else { - ctx->flags |= DMAR_CTX_FAULTED; + ctx->flags |= IOMMU_DEVICE_FAULTED; ctx->last_fault_rec[0] = fault_rec[0]; ctx->last_fault_rec[1] = fault_rec[1]; - device_print_prettyname(ctx->ctx_tag.owner); - bus = pci_get_bus(ctx->ctx_tag.owner); - slot = pci_get_slot(ctx->ctx_tag.owner); - func = pci_get_function(ctx->ctx_tag.owner); + device_print_prettyname(ctx->device_tag.owner); + bus = pci_get_bus(ctx->device_tag.owner); + slot = pci_get_slot(ctx->device_tag.owner); + func = pci_get_function(ctx->device_tag.owner); } - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); printf( "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x " "addr %jx\n", @@ -244,7 +244,7 @@ } static void -dmar_clear_faults(struct dmar_unit *unit) +dmar_clear_faults(struct iommu_unit *unit) { uint32_t frec, frir, fsts; int i; @@ -261,7 +261,7 @@ } int -dmar_init_fault_log(struct dmar_unit *unit) +dmar_init_fault_log(struct iommu_unit *unit) { mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); @@ -278,25 +278,25 @@ taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, "dmar%d fault taskq", unit->unit); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_disable_fault_intr(unit); dmar_clear_faults(unit); dmar_enable_fault_intr(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); return (0); } void -dmar_fini_fault_log(struct dmar_unit *unit) +dmar_fini_fault_log(struct iommu_unit *unit) { if (unit->fault_taskqueue == NULL) return; - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_disable_fault_intr(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); taskqueue_free(unit->fault_taskqueue); @@ -309,22 +309,22 @@ } void -dmar_enable_fault_intr(struct dmar_unit *unit) +dmar_enable_fault_intr(struct iommu_unit *unit) { uint32_t fectl; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); fectl = dmar_read4(unit, DMAR_FECTL_REG); fectl &= ~DMAR_FECTL_IM; dmar_write4(unit, DMAR_FECTL_REG, fectl); } void -dmar_disable_fault_intr(struct dmar_unit *unit) +dmar_disable_fault_intr(struct iommu_unit *unit) { uint32_t fectl; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); fectl = dmar_read4(unit, DMAR_FECTL_REG); dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); } Index: sys/x86/iommu/intel_gas.c =================================================================== --- sys/x86/iommu/intel_gas.c +++ sys/x86/iommu/intel_gas.c @@ -74,27 +74,27 @@ * Guest Address Space management. */ -static uma_zone_t dmar_map_entry_zone; +static uma_zone_t iommu_map_entry_zone; static void intel_gas_init(void) { - dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", - sizeof(struct dmar_map_entry), NULL, NULL, + iommu_map_entry_zone = uma_zcreate("IOMMU_MAP_ENTRY", + sizeof(struct iommu_map_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); } SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); -struct dmar_map_entry * -dmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) +struct iommu_map_entry * +dmar_gas_alloc_entry(struct iommu_domain *domain, u_int flags) { - struct dmar_map_entry *res; + struct iommu_map_entry *res; KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, ("unsupported flags %x", flags)); - res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != + res = uma_zalloc(iommu_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); if (res != NULL) { res->domain = domain; @@ -104,18 +104,18 @@ } void -dmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) +dmar_gas_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) { KASSERT(domain == entry->domain, ("mismatched free domain %p entry %p entry->domain %p", domain, entry, entry->domain)); atomic_subtract_int(&domain->entries_cnt, 1); - uma_zfree(dmar_map_entry_zone, entry); + uma_zfree(iommu_map_entry_zone, entry); } static int -dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) +dmar_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b) { /* Last entry have zero size, so <= */ @@ -137,9 +137,9 @@ } static void -dmar_gas_augment_entry(struct dmar_map_entry *entry) +dmar_gas_augment_entry(struct iommu_map_entry *entry) { - struct dmar_map_entry *child; + struct iommu_map_entry *child; dmar_gaddr_t free_down; free_down = 0; @@ -159,14 +159,14 @@ entry->free_down = free_down; } -RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, +RB_GENERATE(dmar_gas_entries_tree, iommu_map_entry, rb_entry, dmar_gas_cmp_entries); #ifdef INVARIANTS static void -dmar_gas_check_free(struct dmar_domain *domain) +dmar_gas_check_free(struct iommu_domain *domain) { - struct dmar_map_entry *entry, *l, *r; + struct iommu_map_entry *entry, *l, *r; dmar_gaddr_t v; RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { @@ -190,61 +190,61 @@ #endif static bool -dmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) +dmar_gas_rb_insert(struct iommu_domain *domain, struct iommu_map_entry *entry) { - struct dmar_map_entry *found; + struct iommu_map_entry *found; found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); return (found == NULL); } static void -dmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) +dmar_gas_rb_remove(struct iommu_domain *domain, struct iommu_map_entry *entry) { RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); } void -dmar_gas_init_domain(struct dmar_domain *domain) +dmar_gas_init_domain(struct iommu_domain *domain) { - struct dmar_map_entry *begin, *end; + struct iommu_map_entry *begin, *end; begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); - DMAR_DOMAIN_LOCK(domain); + IOMMU_DOMAIN_LOCK(domain); KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); begin->start = 0; begin->end = DMAR_PAGE_SIZE; - begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; + begin->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; dmar_gas_rb_insert(domain, begin); end->start = domain->end; end->end = domain->end; - end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; + end->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; dmar_gas_rb_insert(domain, end); domain->first_place = begin; domain->last_place = end; domain->flags |= DMAR_DOMAIN_GAS_INITED; - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); } void -dmar_gas_fini_domain(struct dmar_domain *domain) +dmar_gas_fini_domain(struct iommu_domain *domain) { - struct dmar_map_entry *entry, *entry1; + struct iommu_map_entry *entry, *entry1; - DMAR_DOMAIN_ASSERT_LOCKED(domain); + IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); KASSERT(entry->start == 0, ("start entry start %p", domain)); KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); - KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, + KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE, ("start entry flags %p", domain)); RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); dmar_gas_free_entry(domain, entry); @@ -252,14 +252,14 @@ entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); KASSERT(entry->start == domain->end, ("end entry start %p", domain)); KASSERT(entry->end == domain->end, ("end entry end %p", domain)); - KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, + KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE, ("end entry flags %p", domain)); RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); dmar_gas_free_entry(domain, entry); RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, entry1) { - KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, + KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0, ("non-RMRR entry left %p", domain)); RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); dmar_gas_free_entry(domain, entry); @@ -267,16 +267,16 @@ } struct dmar_gas_match_args { - struct dmar_domain *domain; + struct iommu_domain *domain; dmar_gaddr_t size; int offset; const struct bus_dma_tag_common *common; u_int gas_flags; - struct dmar_map_entry *entry; + struct iommu_map_entry *entry; }; /* - * The interval [beg, end) is a free interval between two dmar_map_entries. + * The interval [beg, end) is a free interval between two iommu_map_entries. * maxaddr is an upper bound on addresses that can be allocated. Try to * allocate space in the free interval, subject to the conditions expressed * by a, and return 'true' if and only if the allocation attempt succeeds. @@ -298,7 +298,7 @@ return (false); /* No boundary crossing. */ - if (dmar_test_boundary(a->entry->start + a->offset, a->size, + if (iommu_test_boundary(a->entry->start + a->offset, a->size, a->common->boundary)) return (true); @@ -313,7 +313,7 @@ /* DMAR_PAGE_SIZE to create gap after new entry. */ if (start + a->offset + a->size + DMAR_PAGE_SIZE <= end && start + a->offset + a->size <= maxaddr && - dmar_test_boundary(start + a->offset, a->size, + iommu_test_boundary(start + a->offset, a->size, a->common->boundary)) { a->entry->start = start; return (true); @@ -327,7 +327,7 @@ * XXXKIB. It is possible that bs is exactly at the start of * the next entry, then we do not have gap. Ignore for now. */ - if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { + if ((a->gas_flags & IOMMU_MF_CANSPLIT) != 0) { a->size = bs - a->entry->start; return (true); } @@ -353,13 +353,14 @@ found = dmar_gas_rb_insert(a->domain, a->entry); KASSERT(found, ("found dup %p start %jx size %jx", a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); - a->entry->flags = DMAR_MAP_ENTRY_MAP; + a->entry->flags = IOMMU_MAP_ENTRY_MAP; } static int -dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *entry) +dmar_gas_lowermatch(struct dmar_gas_match_args *a, + struct iommu_map_entry *entry) { - struct dmar_map_entry *child; + struct iommu_map_entry *child; child = RB_RIGHT(entry, rb_entry); if (child != NULL && entry->end < a->common->lowaddr && @@ -388,9 +389,10 @@ } static int -dmar_gas_uppermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *entry) +dmar_gas_uppermatch(struct dmar_gas_match_args *a, + struct iommu_map_entry *entry) { - struct dmar_map_entry *child; + struct iommu_map_entry *child; if (entry->free_down < a->size + a->offset + DMAR_PAGE_SIZE) return (ENOMEM); @@ -418,14 +420,14 @@ } static int -dmar_gas_find_space(struct dmar_domain *domain, +dmar_gas_find_space(struct iommu_domain *domain, const struct bus_dma_tag_common *common, dmar_gaddr_t size, - int offset, u_int flags, struct dmar_map_entry *entry) + int offset, u_int flags, struct iommu_map_entry *entry) { struct dmar_gas_match_args a; int error; - DMAR_DOMAIN_ASSERT_LOCKED(domain); + IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); @@ -454,13 +456,13 @@ } static int -dmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, - u_int flags) +dmar_gas_alloc_region(struct iommu_domain *domain, + struct iommu_map_entry *entry, u_int flags) { - struct dmar_map_entry *next, *prev; + struct iommu_map_entry *next, *prev; bool found; - DMAR_DOMAIN_ASSERT_LOCKED(domain); + IOMMU_DOMAIN_ASSERT_LOCKED(domain); if ((entry->start & DMAR_PAGE_MASK) != 0 || (entry->end & DMAR_PAGE_MASK) != 0) @@ -485,16 +487,16 @@ * extends both ways. */ if (prev != NULL && prev->end > entry->start && - (prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { - if ((flags & DMAR_GM_RMRR) == 0 || - (prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) + (prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { + if ((flags & IOMMU_MF_RMRR) == 0 || + (prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0) return (EBUSY); entry->start = prev->end; } if (next->start < entry->end && - (next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { - if ((flags & DMAR_GM_RMRR) == 0 || - (next->flags & DMAR_MAP_ENTRY_RMRR) == 0) + (next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { + if ((flags & IOMMU_MF_RMRR) == 0 || + (next->flags & IOMMU_MAP_ENTRY_RMRR) == 0) return (EBUSY); entry->end = next->start; } @@ -514,11 +516,11 @@ found = dmar_gas_rb_insert(domain, entry); KASSERT(found, ("found RMRR dup %p start %jx end %jx", domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); - if ((flags & DMAR_GM_RMRR) != 0) - entry->flags = DMAR_MAP_ENTRY_RMRR; + if ((flags & IOMMU_MF_RMRR) != 0) + entry->flags = IOMMU_MAP_ENTRY_RMRR; #ifdef INVARIANTS - struct dmar_map_entry *ip, *in; + struct iommu_map_entry *ip, *in; ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); KASSERT(prev == NULL || ip == prev, @@ -537,16 +539,16 @@ } void -dmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) +dmar_gas_free_space(struct iommu_domain *domain, struct iommu_map_entry *entry) { - DMAR_DOMAIN_ASSERT_LOCKED(domain); - KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | - DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, + IOMMU_DOMAIN_ASSERT_LOCKED(domain); + KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | + IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP, ("permanent entry %p %p", domain, entry)); dmar_gas_rb_remove(domain, entry); - entry->flags &= ~DMAR_MAP_ENTRY_MAP; + entry->flags &= ~IOMMU_MAP_ENTRY_MAP; #ifdef INVARIANTS if (dmar_check_free) dmar_gas_check_free(domain); @@ -554,19 +556,19 @@ } void -dmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) +dmar_gas_free_region(struct iommu_domain *domain, struct iommu_map_entry *entry) { - struct dmar_map_entry *next, *prev; + struct iommu_map_entry *next, *prev; - DMAR_DOMAIN_ASSERT_LOCKED(domain); - KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | - DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, + IOMMU_DOMAIN_ASSERT_LOCKED(domain); + KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | + IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR, ("non-RMRR entry %p %p", domain, entry)); prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); dmar_gas_rb_remove(domain, entry); - entry->flags &= ~DMAR_MAP_ENTRY_RMRR; + entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; if (prev == NULL) dmar_gas_rb_insert(domain, domain->first_place); @@ -575,25 +577,25 @@ } int -dmar_gas_map(struct dmar_domain *domain, +dmar_gas_map(struct iommu_domain *domain, const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, - u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) + u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) { - struct dmar_map_entry *entry; + struct iommu_map_entry *entry; int error; - KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, + KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_CANSPLIT)) == 0, ("invalid flags 0x%x", flags)); - entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? + entry = dmar_gas_alloc_entry(domain, (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); if (entry == NULL) return (ENOMEM); - DMAR_DOMAIN_LOCK(domain); + IOMMU_DOMAIN_LOCK(domain); error = dmar_gas_find_space(domain, common, size, offset, flags, entry); if (error == ENOMEM) { - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); dmar_gas_free_entry(domain, entry); return (error); } @@ -606,15 +608,15 @@ KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", (uintmax_t)entry->end, (uintmax_t)domain->end)); entry->flags |= eflags; - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); error = domain_map_buf(domain, entry->start, entry->end - entry->start, ma, - ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | - ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | - ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | - ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), - (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); + ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | + ((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | + ((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | + ((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), + (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); if (error == ENOMEM) { dmar_domain_unload_entry(entry, true); return (error); @@ -627,7 +629,7 @@ } int -dmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, +dmar_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma) { dmar_gaddr_t start; @@ -635,28 +637,28 @@ KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, entry, entry->flags)); - KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_RMRR)) == 0, + KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0, ("invalid flags 0x%x", flags)); start = entry->start; - DMAR_DOMAIN_LOCK(domain); + IOMMU_DOMAIN_LOCK(domain); error = dmar_gas_alloc_region(domain, entry, flags); if (error != 0) { - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); return (error); } entry->flags |= eflags; - DMAR_DOMAIN_UNLOCK(domain); + IOMMU_DOMAIN_UNLOCK(domain); if (entry->end == entry->start) return (0); error = domain_map_buf(domain, entry->start, entry->end - entry->start, ma + OFF_TO_IDX(start - entry->start), - ((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | - ((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | - ((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | - ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), - (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); + ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | + ((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | + ((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | + ((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), + (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); if (error == ENOMEM) { dmar_domain_unload_entry(entry, false); return (error); @@ -668,20 +670,20 @@ } int -dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, +dmar_gas_reserve_region(struct iommu_domain *domain, dmar_gaddr_t start, dmar_gaddr_t end) { - struct dmar_map_entry *entry; + struct iommu_map_entry *entry; int error; entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); entry->start = start; entry->end = end; - DMAR_DOMAIN_LOCK(domain); - error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); + IOMMU_DOMAIN_LOCK(domain); + error = dmar_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT); if (error == 0) - entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; - DMAR_DOMAIN_UNLOCK(domain); + entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; + IOMMU_DOMAIN_UNLOCK(domain); if (error != 0) dmar_gas_free_entry(domain, entry); return (error); Index: sys/x86/iommu/intel_idpgtbl.c =================================================================== --- sys/x86/iommu/intel_idpgtbl.c +++ sys/x86/iommu/intel_idpgtbl.c @@ -69,7 +69,7 @@ #include #include -static int domain_unmap_buf_locked(struct dmar_domain *domain, +static int domain_unmap_buf_locked(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, int flags); /* @@ -163,9 +163,9 @@ * maxaddr is typically mapped. */ vm_object_t -domain_get_idmap_pgtbl(struct dmar_domain *domain, dmar_gaddr_t maxaddr) +domain_get_idmap_pgtbl(struct iommu_domain *domain, dmar_gaddr_t maxaddr) { - struct dmar_unit *unit; + struct iommu_unit *unit; struct idpgtbl *tbl; vm_object_t res; vm_page_t m; @@ -194,7 +194,7 @@ sx_slock(&idpgtbl_lock); LIST_FOREACH(tbl, &idpgtbls, link) { if (tbl->maxaddr >= maxaddr && - dmar_pglvl_supported(domain->dmar, tbl->pglvl) && + dmar_pglvl_supported(domain->iommu, tbl->pglvl) && tbl->leaf == leaf) { res = tbl->pgtbl_obj; vm_object_reference(res); @@ -213,7 +213,7 @@ sx_xlock(&idpgtbl_lock); LIST_FOREACH(tbl, &idpgtbls, link) { if (tbl->maxaddr >= maxaddr && - dmar_pglvl_supported(domain->dmar, tbl->pglvl) && + dmar_pglvl_supported(domain->iommu, tbl->pglvl) && tbl->leaf == leaf) { res = tbl->pgtbl_obj; vm_object_reference(res); @@ -254,7 +254,7 @@ * If DMAR cannot look into the chipset write buffer, flush it * as well. */ - unit = domain->dmar; + unit = domain->iommu; if (!DMAR_IS_COHERENT(unit)) { VM_OBJECT_WLOCK(res); for (m = vm_page_lookup(res, 0); m != NULL; @@ -263,9 +263,9 @@ VM_OBJECT_WUNLOCK(res); } if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) { - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_flush_write_bufs(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } return (res); @@ -323,7 +323,7 @@ * the level lvl. */ static int -domain_pgtbl_pte_off(struct dmar_domain *domain, dmar_gaddr_t base, int lvl) +domain_pgtbl_pte_off(struct iommu_domain *domain, dmar_gaddr_t base, int lvl) { base >>= DMAR_PAGE_SHIFT + (domain->pglvl - lvl - 1) * @@ -337,7 +337,7 @@ * lvl. */ static vm_pindex_t -domain_pgtbl_get_pindex(struct dmar_domain *domain, dmar_gaddr_t base, int lvl) +domain_pgtbl_get_pindex(struct iommu_domain *domain, dmar_gaddr_t base, int lvl) { vm_pindex_t idx, pidx; int i; @@ -353,7 +353,7 @@ } static dmar_pte_t * -domain_pgtbl_map_pte(struct dmar_domain *domain, dmar_gaddr_t base, int lvl, +domain_pgtbl_map_pte(struct iommu_domain *domain, dmar_gaddr_t base, int lvl, int flags, vm_pindex_t *idxp, struct sf_buf **sf) { vm_page_t m; @@ -408,7 +408,7 @@ } dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | VM_PAGE_TO_PHYS(m)); - dmar_flush_pte_to_ram(domain->dmar, ptep); + dmar_flush_pte_to_ram(domain->iommu, ptep); sf_buf_page(sfp)->ref_count += 1; m->ref_count--; dmar_unmap_pgtbl(sfp); @@ -421,7 +421,7 @@ } static int -domain_map_buf_locked(struct dmar_domain *domain, dmar_gaddr_t base, +domain_map_buf_locked(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags) { dmar_pte_t *pte; @@ -489,7 +489,7 @@ } dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | (superpage ? DMAR_PTE_SP : 0)); - dmar_flush_pte_to_ram(domain->dmar, pte); + dmar_flush_pte_to_ram(domain->iommu, pte); sf_buf_page(sf)->ref_count += 1; } if (sf != NULL) @@ -499,13 +499,13 @@ } int -domain_map_buf(struct dmar_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, - vm_page_t *ma, uint64_t pflags, int flags) +domain_map_buf(struct iommu_domain *domain, dmar_gaddr_t base, + dmar_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags) { - struct dmar_unit *unit; + struct iommu_unit *unit; int error; - unit = domain->dmar; + unit = domain->iommu; KASSERT((domain->flags & DMAR_DOMAIN_IDMAP) == 0, ("modifying idmap pagetable domain %p", domain)); @@ -551,19 +551,19 @@ domain_flush_iotlb_sync(domain, base, size); else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) { /* See 11.1 Write Buffer Flushing. */ - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_flush_write_bufs(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } return (0); } -static void domain_unmap_clear_pte(struct dmar_domain *domain, +static void domain_unmap_clear_pte(struct iommu_domain *domain, dmar_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_fs); static void -domain_free_pgtbl_pde(struct dmar_domain *domain, dmar_gaddr_t base, +domain_free_pgtbl_pde(struct iommu_domain *domain, dmar_gaddr_t base, int lvl, int flags) { struct sf_buf *sf; @@ -576,13 +576,13 @@ } static void -domain_unmap_clear_pte(struct dmar_domain *domain, dmar_gaddr_t base, int lvl, +domain_unmap_clear_pte(struct iommu_domain *domain, dmar_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) { vm_page_t m; dmar_pte_clear(&pte->pte); - dmar_flush_pte_to_ram(domain->dmar, pte); + dmar_flush_pte_to_ram(domain->iommu, pte); m = sf_buf_page(*sf); if (free_sf) { dmar_unmap_pgtbl(*sf); @@ -605,7 +605,7 @@ * Assumes that the unmap is never partial. */ static int -domain_unmap_buf_locked(struct dmar_domain *domain, dmar_gaddr_t base, +domain_unmap_buf_locked(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, int flags) { dmar_pte_t *pte; @@ -677,7 +677,7 @@ } int -domain_unmap_buf(struct dmar_domain *domain, dmar_gaddr_t base, +domain_unmap_buf(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, int flags) { int error; @@ -689,7 +689,7 @@ } int -domain_alloc_pgtbl(struct dmar_domain *domain) +domain_alloc_pgtbl(struct iommu_domain *domain) { vm_page_t m; @@ -704,21 +704,21 @@ /* No implicit free of the top level page table page. */ m->ref_count = 1; DMAR_DOMAIN_PGUNLOCK(domain); - DMAR_LOCK(domain->dmar); + IOMMU_LOCK(domain->iommu); domain->flags |= DMAR_DOMAIN_PGTBL_INITED; - DMAR_UNLOCK(domain->dmar); + IOMMU_UNLOCK(domain->iommu); return (0); } void -domain_free_pgtbl(struct dmar_domain *domain) +domain_free_pgtbl(struct iommu_domain *domain) { vm_object_t obj; vm_page_t m; obj = domain->pgtbl_obj; if (obj == NULL) { - KASSERT((domain->dmar->hw_ecap & DMAR_ECAP_PT) != 0 && + KASSERT((domain->iommu->hw_ecap & DMAR_ECAP_PT) != 0 && (domain->flags & DMAR_DOMAIN_IDMAP) != 0, ("lost pagetable object domain %p", domain)); return; @@ -741,7 +741,7 @@ } static inline uint64_t -domain_wait_iotlb_flush(struct dmar_unit *unit, uint64_t wt, int iro) +domain_wait_iotlb_flush(struct iommu_unit *unit, uint64_t wt, int iro) { uint64_t iotlbr; @@ -757,19 +757,19 @@ } void -domain_flush_iotlb_sync(struct dmar_domain *domain, dmar_gaddr_t base, +domain_flush_iotlb_sync(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size) { - struct dmar_unit *unit; + struct iommu_unit *unit; dmar_gaddr_t isize; uint64_t iotlbr; int am, iro; - unit = domain->dmar; + unit = domain->iommu; KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call", unit->unit)); iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16; - DMAR_LOCK(unit); + IOMMU_LOCK(unit); if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) { iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM | DMAR_IOTLB_DID(domain->domain), iro); @@ -799,5 +799,5 @@ break; } } - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } Index: sys/x86/iommu/intel_intrmap.c =================================================================== --- sys/x86/iommu/intel_intrmap.c +++ sys/x86/iommu/intel_intrmap.c @@ -60,16 +60,16 @@ #include #include -static struct dmar_unit *dmar_ir_find(device_t src, uint16_t *rid, +static struct iommu_unit *dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar); -static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, +static void dmar_ir_program_irte(struct iommu_unit *unit, u_int idx, uint64_t low, uint16_t rid); -static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie); +static int dmar_ir_free_irte(struct iommu_unit *unit, u_int cookie); int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) { - struct dmar_unit *unit; + struct iommu_unit *unit; vmem_addr_t vmem_res; u_int idx, i; int error; @@ -98,7 +98,7 @@ iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, uint64_t *addr, uint32_t *data) { - struct dmar_unit *unit; + struct iommu_unit *unit; uint64_t low; uint16_t rid; int is_dmar; @@ -143,7 +143,7 @@ int iommu_unmap_msi_intr(device_t src, u_int cookie) { - struct dmar_unit *unit; + struct iommu_unit *unit; if (cookie == -1) return (0); @@ -155,7 +155,7 @@ iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo) { - struct dmar_unit *unit; + struct iommu_unit *unit; vmem_addr_t vmem_res; uint64_t low, iorte; u_int idx; @@ -217,7 +217,7 @@ int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) { - struct dmar_unit *unit; + struct iommu_unit *unit; u_int idx; idx = *cookie; @@ -230,11 +230,11 @@ return (dmar_ir_free_irte(unit, idx)); } -static struct dmar_unit * +static struct iommu_unit * dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar) { devclass_t src_class; - struct dmar_unit *unit; + struct iommu_unit *unit; /* * We need to determine if the interrupt source generates FSB @@ -255,13 +255,13 @@ } else { unit = dmar_find(src, bootverbose); if (unit != NULL && rid != NULL) - dmar_get_requester(src, rid); + iommu_get_requester(src, rid); } return (unit); } static void -dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low, +dmar_ir_program_irte(struct iommu_unit *unit, u_int idx, uint64_t low, uint16_t rid) { dmar_irte_t *irte; @@ -277,7 +277,7 @@ "programming irte[%d] rid %#x high %#jx low %#jx\n", idx, rid, (uintmax_t)high, (uintmax_t)low); } - DMAR_LOCK(unit); + IOMMU_LOCK(unit); if ((irte->irte1 & DMAR_IRTE1_P) != 0) { /* * The rte is already valid. Assume that the request @@ -294,12 +294,12 @@ dmar_pte_store(&irte->irte1, low); } dmar_qi_invalidate_iec(unit, idx, 1); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } static int -dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie) +dmar_ir_free_irte(struct iommu_unit *unit, u_int cookie) { dmar_irte_t *irte; @@ -310,9 +310,9 @@ irte = &(unit->irt[cookie]); dmar_pte_clear(&irte->irte1); dmar_pte_clear(&irte->irte2); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_qi_invalidate_iec(unit, cookie, 1); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); vmem_free(unit->irtids, cookie, 1); return (0); } @@ -325,7 +325,7 @@ } int -dmar_init_irt(struct dmar_unit *unit) +dmar_init_irt(struct iommu_unit *unit) { if ((unit->hw_ecap & DMAR_ECAP_IR) == 0) @@ -351,10 +351,10 @@ unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt); unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0, M_FIRSTFIT | M_NOWAIT); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_load_irt_ptr(unit); dmar_qi_invalidate_iec_glob(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); /* * Initialize mappings for already configured interrupt pins. @@ -363,14 +363,14 @@ */ intr_reprogram(); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_enable_ir(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); return (0); } void -dmar_fini_irt(struct dmar_unit *unit) +dmar_fini_irt(struct iommu_unit *unit) { unit->ir_enabled = 0; Index: sys/x86/iommu/intel_qi.c =================================================================== --- sys/x86/iommu/intel_qi.c +++ sys/x86/iommu/intel_qi.c @@ -62,7 +62,7 @@ #include static bool -dmar_qi_seq_processed(const struct dmar_unit *unit, +dmar_qi_seq_processed(const struct iommu_unit *unit, const struct dmar_qi_genseq *pseq) { @@ -72,11 +72,11 @@ } static int -dmar_enable_qi(struct dmar_unit *unit) +dmar_enable_qi(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_QIE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) @@ -85,11 +85,11 @@ } static int -dmar_disable_qi(struct dmar_unit *unit) +dmar_disable_qi(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_QIE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) @@ -98,20 +98,20 @@ } static void -dmar_qi_advance_tail(struct dmar_unit *unit) +dmar_qi_advance_tail(struct iommu_unit *unit) { - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); } static void -dmar_qi_ensure(struct dmar_unit *unit, int descr_count) +dmar_qi_ensure(struct iommu_unit *unit, int descr_count) { uint32_t head; int bytes; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; for (;;) { if (bytes <= unit->inv_queue_avail) @@ -141,10 +141,10 @@ } static void -dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) +dmar_qi_emit(struct iommu_unit *unit, uint64_t data1, uint64_t data2) { - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, @@ -160,11 +160,11 @@ } static void -dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, +dmar_qi_emit_wait_descr(struct iommu_unit *unit, uint32_t seq, bool intr, bool memw, bool fence) { - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | @@ -174,14 +174,14 @@ } static void -dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq, +dmar_qi_emit_wait_seq(struct iommu_unit *unit, struct dmar_qi_genseq *pseq, bool emit_wait) { struct dmar_qi_genseq gsec; uint32_t seq; KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); if (unit->inv_waitd_seq == 0xffffffff) { gsec.gen = unit->inv_waitd_gen; gsec.seq = unit->inv_waitd_seq; @@ -203,11 +203,11 @@ } static void -dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, +dmar_qi_wait_for_seq(struct iommu_unit *unit, const struct dmar_qi_genseq *gseq, bool nowait) { - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->inv_seq_waiters++; while (!dmar_qi_seq_processed(unit, gseq)) { if (cold || nowait) { @@ -221,15 +221,15 @@ } void -dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, +dmar_qi_invalidate_locked(struct iommu_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait) { - struct dmar_unit *unit; + struct iommu_unit *unit; dmar_gaddr_t isize; int am; - unit = domain->dmar; - DMAR_ASSERT_LOCKED(unit); + unit = domain->iommu; + IOMMU_ASSERT_LOCKED(unit); for (; size > 0; base += isize, size -= isize) { am = calc_am(unit, base, size, &isize); dmar_qi_ensure(unit, 1); @@ -244,11 +244,11 @@ } void -dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) +dmar_qi_invalidate_ctx_glob_locked(struct iommu_unit *unit) { struct dmar_qi_genseq gseq; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); dmar_qi_emit_wait_seq(unit, &gseq, true); @@ -257,11 +257,11 @@ } void -dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) +dmar_qi_invalidate_iotlb_glob_locked(struct iommu_unit *unit) { struct dmar_qi_genseq gseq; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); @@ -271,11 +271,11 @@ } void -dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) +dmar_qi_invalidate_iec_glob(struct iommu_unit *unit) { struct dmar_qi_genseq gseq; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); dmar_qi_emit_wait_seq(unit, &gseq, true); @@ -284,12 +284,12 @@ } void -dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) +dmar_qi_invalidate_iec(struct iommu_unit *unit, u_int start, u_int cnt) { struct dmar_qi_genseq gseq; u_int c, l; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); KASSERT(start < unit->irte_cnt && start < start + cnt && start + cnt <= unit->irte_cnt, ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); @@ -326,7 +326,7 @@ int dmar_qi_intr(void *arg) { - struct dmar_unit *unit; + struct iommu_unit *unit; unit = arg; KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit)); @@ -337,13 +337,13 @@ static void dmar_qi_task(void *arg, int pending __unused) { - struct dmar_unit *unit; - struct dmar_map_entry *entry; + struct iommu_unit *unit; + struct iommu_map_entry *entry; uint32_t ics; unit = arg; - DMAR_LOCK(unit); + IOMMU_LOCK(unit); for (;;) { entry = TAILQ_FIRST(&unit->tlb_flush_entries); if (entry == NULL) @@ -351,10 +351,10 @@ if (!dmar_qi_seq_processed(unit, &entry->gseq)) break; TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); dmar_domain_free_entry(entry, (entry->flags & - DMAR_MAP_ENTRY_QI_NF) == 0); - DMAR_LOCK(unit); + IOMMU_MAP_ENTRY_QI_NF) == 0); + IOMMU_LOCK(unit); } ics = dmar_read4(unit, DMAR_ICS_REG); if ((ics & DMAR_ICS_IWC) != 0) { @@ -363,11 +363,11 @@ } if (unit->inv_seq_waiters > 0) wakeup(&unit->inv_seq_waiters); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); } int -dmar_init_qi(struct dmar_unit *unit) +dmar_init_qi(struct iommu_unit *unit) { uint64_t iqa; uint32_t ics; @@ -404,7 +404,7 @@ unit->inv_waitd_seq_hw_phys = pmap_kextract( (vm_offset_t)&unit->inv_waitd_seq_hw); - DMAR_LOCK(unit); + IOMMU_LOCK(unit); dmar_write8(unit, DMAR_IQT_REG, 0); iqa = pmap_kextract(unit->inv_queue); iqa |= qi_sz; @@ -416,13 +416,13 @@ dmar_write4(unit, DMAR_ICS_REG, ics); } dmar_enable_qi_intr(unit); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); return (0); } void -dmar_fini_qi(struct dmar_unit *unit) +dmar_fini_qi(struct iommu_unit *unit) { struct dmar_qi_genseq gseq; @@ -432,7 +432,7 @@ taskqueue_free(unit->qi_taskqueue); unit->qi_taskqueue = NULL; - DMAR_LOCK(unit); + IOMMU_LOCK(unit); /* quisce */ dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_seq(unit, &gseq, true); @@ -443,7 +443,7 @@ dmar_disable_qi(unit); KASSERT(unit->inv_seq_waiters == 0, ("dmar%d: waiters on disabled queue", unit->unit)); - DMAR_UNLOCK(unit); + IOMMU_UNLOCK(unit); kmem_free(unit->inv_queue, unit->inv_queue_size); unit->inv_queue = 0; @@ -452,11 +452,11 @@ } void -dmar_enable_qi_intr(struct dmar_unit *unit) +dmar_enable_qi_intr(struct iommu_unit *unit) { uint32_t iectl; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); iectl = dmar_read4(unit, DMAR_IECTL_REG); iectl &= ~DMAR_IECTL_IM; @@ -464,11 +464,11 @@ } void -dmar_disable_qi_intr(struct dmar_unit *unit) +dmar_disable_qi_intr(struct iommu_unit *unit) { uint32_t iectl; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); iectl = dmar_read4(unit, DMAR_IECTL_REG); dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); Index: sys/x86/iommu/intel_quirks.c =================================================================== --- sys/x86/iommu/intel_quirks.c +++ sys/x86/iommu/intel_quirks.c @@ -64,7 +64,7 @@ #include #include -typedef void (*dmar_quirk_cpu_fun)(struct dmar_unit *); +typedef void (*dmar_quirk_cpu_fun)(struct iommu_unit *); struct intel_dmar_quirk_cpu { u_int ext_family; @@ -76,7 +76,7 @@ const char *descr; }; -typedef void (*dmar_quirk_nb_fun)(struct dmar_unit *, device_t nb); +typedef void (*dmar_quirk_nb_fun)(struct iommu_unit *, device_t nb); struct intel_dmar_quirk_nb { u_int dev_id; @@ -88,7 +88,7 @@ #define QUIRK_NB_ALL_REV 0xffffffff static void -dmar_match_quirks(struct dmar_unit *dmar, +dmar_match_quirks(struct iommu_unit *dmar, const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len, const struct intel_dmar_quirk_cpu *cpu_quirks, int cpu_quirks_len) { @@ -149,21 +149,21 @@ } static void -nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused) +nb_5400_no_low_high_prot_mem(struct iommu_unit *unit, device_t nb __unused) { unit->hw_cap &= ~(DMAR_CAP_PHMR | DMAR_CAP_PLMR); } static void -nb_no_ir(struct dmar_unit *unit, device_t nb __unused) +nb_no_ir(struct iommu_unit *unit, device_t nb __unused) { unit->hw_ecap &= ~(DMAR_ECAP_IR | DMAR_ECAP_EIM); } static void -nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb) +nb_5500_no_ir_rev13(struct iommu_unit *unit, device_t nb) { u_int rev_no; @@ -206,7 +206,7 @@ }; static void -cpu_e5_am9(struct dmar_unit *unit) +cpu_e5_am9(struct iommu_unit *unit) { unit->hw_cap &= ~(0x3fULL << 48); @@ -222,19 +222,19 @@ }; void -dmar_quirks_pre_use(struct dmar_unit *dmar) +dmar_quirks_pre_use(struct iommu_unit *dmar) { if (!dmar_barrier_enter(dmar, DMAR_BARRIER_USEQ)) return; - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); dmar_match_quirks(dmar, pre_use_nb, nitems(pre_use_nb), NULL, 0); dmar_barrier_exit(dmar, DMAR_BARRIER_USEQ); } void -dmar_quirks_post_ident(struct dmar_unit *dmar) +dmar_quirks_post_ident(struct iommu_unit *dmar) { dmar_match_quirks(dmar, NULL, 0, post_ident_cpu, Index: sys/x86/iommu/intel_reg.h =================================================================== --- sys/x86/iommu/intel_reg.h +++ sys/x86/iommu/intel_reg.h @@ -34,12 +34,16 @@ #ifndef __X86_IOMMU_INTEL_REG_H #define __X86_IOMMU_INTEL_REG_H -#define DMAR_PAGE_SIZE PAGE_SIZE -#define DMAR_PAGE_MASK (DMAR_PAGE_SIZE - 1) -#define DMAR_PAGE_SHIFT PAGE_SHIFT -#define DMAR_NPTEPG (DMAR_PAGE_SIZE / sizeof(dmar_pte_t)) -#define DMAR_NPTEPGSHIFT 9 -#define DMAR_PTEMASK (DMAR_NPTEPG - 1) +#define DMAR_PAGE_SIZE PAGE_SIZE +#define DMAR_PAGE_MASK (DMAR_PAGE_SIZE - 1) +#define DMAR_PAGE_SHIFT PAGE_SHIFT +#define DMAR_NPTEPG (DMAR_PAGE_SIZE / sizeof(dmar_pte_t)) +#define DMAR_NPTEPGSHIFT 9 +#define DMAR_PTEMASK (DMAR_NPTEPG - 1) + +/* For the iommu generic busdma backend. */ +#define IOMMU_PAGE_SIZE DMAR_PAGE_SIZE +#define IOMMU_PAGE_MASK DMAR_PAGE_MASK typedef struct dmar_root_entry { uint64_t r1; @@ -51,10 +55,10 @@ #define DMAR_CTX_CNT (DMAR_PAGE_SIZE / sizeof(dmar_root_entry_t)) -typedef struct dmar_ctx_entry { +typedef struct iommu_device_entry { uint64_t ctx1; uint64_t ctx2; -} dmar_ctx_entry_t; +} iommu_device_entry_t; #define DMAR_CTX1_P 1 /* Present */ #define DMAR_CTX1_FPD 2 /* Fault Processing Disable */ /* Translation Type: */ Index: sys/x86/iommu/intel_utils.c =================================================================== --- sys/x86/iommu/intel_utils.c +++ sys/x86/iommu/intel_utils.c @@ -106,7 +106,7 @@ }; bool -dmar_pglvl_supported(struct dmar_unit *unit, int pglvl) +dmar_pglvl_supported(struct iommu_unit *unit, int pglvl) { int i; @@ -120,12 +120,12 @@ } int -domain_set_agaw(struct dmar_domain *domain, int mgaw) +domain_set_agaw(struct iommu_domain *domain, int mgaw) { int sagaw, i; domain->mgaw = mgaw; - sagaw = DMAR_CAP_SAGAW(domain->dmar->hw_cap); + sagaw = DMAR_CAP_SAGAW(domain->iommu->hw_cap); for (i = 0; i < nitems(sagaw_bits); i++) { if (sagaw_bits[i].agaw >= mgaw) { domain->agaw = sagaw_bits[i].agaw; @@ -134,7 +134,7 @@ return (0); } } - device_printf(domain->dmar->dev, + device_printf(domain->iommu->dev, "context request mgaw %d: no agaw found, sagaw %x\n", mgaw, sagaw); return (EINVAL); @@ -148,7 +148,8 @@ * address space, accept the biggest sagaw, whatever is it. */ int -dmar_maxaddr2mgaw(struct dmar_unit *unit, dmar_gaddr_t maxaddr, bool allow_less) +dmar_maxaddr2mgaw(struct iommu_unit *unit, dmar_gaddr_t maxaddr, + bool allow_less) { int i; @@ -192,7 +193,7 @@ * the context ctx. */ int -domain_is_sp_lvl(struct dmar_domain *domain, int lvl) +domain_is_sp_lvl(struct iommu_domain *domain, int lvl) { int alvl, cap_sps; static const int sagaw_sp[] = { @@ -203,7 +204,7 @@ }; alvl = domain->pglvl - lvl - 1; - cap_sps = DMAR_CAP_SPS(domain->dmar->hw_cap); + cap_sps = DMAR_CAP_SPS(domain->iommu->hw_cap); return (alvl < nitems(sagaw_sp) && (sagaw_sp[alvl] & cap_sps) != 0); } @@ -228,14 +229,14 @@ } dmar_gaddr_t -domain_page_size(struct dmar_domain *domain, int lvl) +domain_page_size(struct iommu_domain *domain, int lvl) { return (pglvl_page_size(domain->pglvl, lvl)); } int -calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, +calc_am(struct iommu_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, dmar_gaddr_t *isizep) { dmar_gaddr_t isize; @@ -360,7 +361,7 @@ } static void -dmar_flush_transl_to_ram(struct dmar_unit *unit, void *dst, size_t sz) +dmar_flush_transl_to_ram(struct iommu_unit *unit, void *dst, size_t sz) { if (DMAR_IS_COHERENT(unit)) @@ -373,21 +374,21 @@ } void -dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst) +dmar_flush_pte_to_ram(struct iommu_unit *unit, dmar_pte_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void -dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst) +dmar_flush_ctx_to_ram(struct iommu_unit *unit, iommu_device_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void -dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst) +dmar_flush_root_to_ram(struct iommu_unit *unit, dmar_root_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); @@ -398,7 +399,7 @@ * the completion. */ int -dmar_load_root_entry_ptr(struct dmar_unit *unit) +dmar_load_root_entry_ptr(struct iommu_unit *unit) { vm_page_t root_entry; int error; @@ -407,7 +408,7 @@ * Access to the GCMD register must be serialized while the * command is submitted. */ - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); VM_OBJECT_RLOCK(unit->ctx_obj); root_entry = vm_page_lookup(unit->ctx_obj, 0); @@ -424,7 +425,7 @@ * the completion. */ int -dmar_inv_ctx_glob(struct dmar_unit *unit) +dmar_inv_ctx_glob(struct iommu_unit *unit) { int error; @@ -432,7 +433,7 @@ * Access to the CCMD register must be serialized while the * command is submitted. */ - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); /* @@ -451,11 +452,11 @@ * Globally invalidate the IOTLB, busily waiting for the completion. */ int -dmar_inv_iotlb_glob(struct dmar_unit *unit) +dmar_inv_iotlb_glob(struct iommu_unit *unit) { int error, reg; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap); @@ -472,11 +473,11 @@ * in the architecture specification. */ int -dmar_flush_write_bufs(struct dmar_unit *unit) +dmar_flush_write_bufs(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); /* * DMAR_GCMD_WBF is only valid when CAP_RWBF is reported. @@ -491,11 +492,11 @@ } int -dmar_enable_translation(struct dmar_unit *unit) +dmar_enable_translation(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) @@ -504,11 +505,11 @@ } int -dmar_disable_translation(struct dmar_unit *unit) +dmar_disable_translation(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) @@ -517,12 +518,12 @@ } int -dmar_load_irt_ptr(struct dmar_unit *unit) +dmar_load_irt_ptr(struct iommu_unit *unit) { uint64_t irta, s; int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); irta = unit->irt_phys; if (DMAR_X2APIC(unit)) irta |= DMAR_IRTA_EIME; @@ -539,11 +540,11 @@ } int -dmar_enable_ir(struct dmar_unit *unit) +dmar_enable_ir(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_IRE; unit->hw_gcmd &= ~DMAR_GCMD_CFI; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); @@ -553,11 +554,11 @@ } int -dmar_disable_ir(struct dmar_unit *unit) +dmar_disable_ir(struct iommu_unit *unit) { int error; - DMAR_ASSERT_LOCKED(unit); + IOMMU_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_IRE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) @@ -573,13 +574,13 @@ f_wakeup = 1 << (barrier_id * 3 + 2) bool -dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id) +dmar_barrier_enter(struct iommu_unit *dmar, u_int barrier_id) { BARRIER_F; - DMAR_LOCK(dmar); + IOMMU_LOCK(dmar); if ((dmar->barrier_flags & f_done) != 0) { - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); return (false); } @@ -591,28 +592,28 @@ } KASSERT((dmar->barrier_flags & f_done) != 0, ("dmar%d barrier %d missing done", dmar->unit, barrier_id)); - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); return (false); } dmar->barrier_flags |= f_inproc; - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); return (true); } void -dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id) +dmar_barrier_exit(struct iommu_unit *dmar, u_int barrier_id) { BARRIER_F; - DMAR_ASSERT_LOCKED(dmar); + IOMMU_ASSERT_LOCKED(dmar); KASSERT((dmar->barrier_flags & (f_done | f_inproc)) == f_inproc, ("dmar%d barrier %d missed entry", dmar->unit, barrier_id)); dmar->barrier_flags |= f_done; if ((dmar->barrier_flags & f_wakeup) != 0) wakeup(&dmar->barrier_flags); dmar->barrier_flags &= ~(f_inproc | f_wakeup); - DMAR_UNLOCK(dmar); + IOMMU_UNLOCK(dmar); } int dmar_batch_coalesce = 100;