diff --git a/sys/arm64/iommu/iommu.c b/sys/arm64/iommu/iommu.c index 3451a002bef6..c786d50a43e1 100644 --- a/sys/arm64/iommu/iommu.c +++ b/sys/arm64/iommu/iommu.c @@ -1,523 +1,525 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Ruslan Bukin * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Portions of this work was supported by Innovate UK project 105694, * "Digital Security by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include "iommu.h" #include "iommu_if.h" static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework"); #define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx) #define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx) #define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED) #define dprintf(fmt, ...) static struct sx iommu_sx; struct iommu_entry { struct iommu_unit *iommu; LIST_ENTRY(iommu_entry) next; }; static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list); static int iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, int flags) { struct iommu_unit *iommu; int error; iommu = iodom->iommu; error = IOMMU_UNMAP(iommu->dev, iodom, base, size); return (error); } static int iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags) { struct iommu_unit *iommu; vm_prot_t prot; vm_offset_t va; int error; dprintf("%s: base %lx, size %lx\n", __func__, base, size); prot = 0; if (eflags & IOMMU_MAP_ENTRY_READ) prot |= VM_PROT_READ; if (eflags & IOMMU_MAP_ENTRY_WRITE) prot |= VM_PROT_WRITE; va = base; iommu = iodom->iommu; error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot); return (error); } static const struct iommu_domain_map_ops domain_map_ops = { .map = iommu_domain_map_buf, .unmap = iommu_domain_unmap_buf, }; static struct iommu_domain * iommu_domain_alloc(struct iommu_unit *iommu) { struct iommu_domain *iodom; iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu); if (iodom == NULL) return (NULL); KASSERT(iodom->end != 0, ("domain end is not set")); iommu_domain_init(iommu, iodom, &domain_map_ops); iodom->iommu = iommu; iommu_gas_init_domain(iodom); return (iodom); } static int iommu_domain_free(struct iommu_domain *iodom) { struct iommu_unit *iommu; iommu = iodom->iommu; IOMMU_LOCK(iommu); if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) { IOMMU_DOMAIN_LOCK(iodom); iommu_gas_fini_domain(iodom); IOMMU_DOMAIN_UNLOCK(iodom); } iommu_domain_fini(iodom); IOMMU_DOMAIN_FREE(iommu->dev, iodom); IOMMU_UNLOCK(iommu); return (0); } static void iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t) { bus_addr_t maxaddr; maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR); t->common.ref_count = 0; t->common.impl = &bus_dma_iommu_impl; t->common.alignment = 1; t->common.boundary = 0; t->common.lowaddr = maxaddr; t->common.highaddr = maxaddr; t->common.maxsize = maxaddr; t->common.nsegments = BUS_SPACE_UNRESTRICTED; t->common.maxsegsz = maxaddr; } static struct iommu_ctx * iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled) { struct iommu_unit *iommu; struct iommu_ctx *ioctx; iommu = iodom->iommu; ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled); if (ioctx == NULL) return (NULL); ioctx->domain = iodom; return (ioctx); } static int iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx) { struct bus_dma_tag_iommu *tag; struct iommu_domain *iodom; struct iommu_unit *iommu; int error; iodom = ioctx->domain; iommu = iodom->iommu; error = IOMMU_CTX_INIT(iommu->dev, ioctx); if (error) return (error); tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu), M_IOMMU, M_WAITOK | M_ZERO); tag->owner = requester; tag->ctx = ioctx; tag->ctx->domain = iodom; iommu_tag_init(iodom, tag); return (error); } static struct iommu_unit * iommu_lookup(device_t dev) { struct iommu_entry *entry; struct iommu_unit *iommu; IOMMU_LIST_LOCK(); LIST_FOREACH(entry, &iommu_list, next) { iommu = entry->iommu; if (iommu->dev == dev) { IOMMU_LIST_UNLOCK(); return (iommu); } } IOMMU_LIST_UNLOCK(); return (NULL); } +#ifdef FDT struct iommu_ctx * iommu_get_ctx_ofw(device_t dev, int channel) { struct iommu_domain *iodom; struct iommu_unit *iommu; struct iommu_ctx *ioctx; phandle_t node, parent; device_t iommu_dev; pcell_t *cells; int niommus; int ncells; int error; node = ofw_bus_get_node(dev); if (node <= 0) { device_printf(dev, "%s called on not ofw based device.\n", __func__); return (NULL); } error = ofw_bus_parse_xref_list_get_length(node, "iommus", "#iommu-cells", &niommus); if (error) { device_printf(dev, "%s can't get iommu list.\n", __func__); return (NULL); } if (niommus == 0) { device_printf(dev, "%s iommu list is empty.\n", __func__); return (NULL); } error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells", channel, &parent, &ncells, &cells); if (error != 0) { device_printf(dev, "%s can't get iommu device xref.\n", __func__); return (NULL); } iommu_dev = OF_device_from_xref(parent); if (iommu_dev == NULL) { device_printf(dev, "%s can't get iommu device.\n", __func__); return (NULL); } iommu = iommu_lookup(iommu_dev); if (iommu == NULL) { device_printf(dev, "%s can't lookup iommu.\n", __func__); return (NULL); } /* * In our current configuration we have a domain per each ctx, * so allocate a domain first. */ iodom = iommu_domain_alloc(iommu); if (iodom == NULL) { device_printf(dev, "%s can't allocate domain.\n", __func__); return (NULL); } ioctx = iommu_ctx_alloc(dev, iodom, false); if (ioctx == NULL) { iommu_domain_free(iodom); return (NULL); } ioctx->domain = iodom; error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells); if (error) { device_printf(dev, "%s can't set MD data\n", __func__); return (NULL); } error = iommu_ctx_init(dev, ioctx); if (error) { IOMMU_CTX_FREE(iommu->dev, ioctx); iommu_domain_free(iodom); return (NULL); } return (ioctx); } +#endif struct iommu_ctx * iommu_get_ctx(struct iommu_unit *iommu, device_t requester, uint16_t rid, bool disabled, bool rmrr) { struct iommu_domain *iodom; struct iommu_ctx *ioctx; int error; IOMMU_LOCK(iommu); ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester); if (ioctx) { IOMMU_UNLOCK(iommu); return (ioctx); } IOMMU_UNLOCK(iommu); /* * In our current configuration we have a domain per each ctx. * So allocate a domain first. */ iodom = iommu_domain_alloc(iommu); if (iodom == NULL) return (NULL); ioctx = iommu_ctx_alloc(requester, iodom, disabled); if (ioctx == NULL) { iommu_domain_free(iodom); return (NULL); } error = iommu_ctx_init(requester, ioctx); if (error) { IOMMU_CTX_FREE(iommu->dev, ioctx); iommu_domain_free(iodom); return (NULL); } return (ioctx); } void iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx) { struct bus_dma_tag_iommu *tag; IOMMU_ASSERT_LOCKED(iommu); tag = ioctx->tag; IOMMU_CTX_FREE(iommu->dev, ioctx); free(tag, M_IOMMU); } void iommu_free_ctx(struct iommu_ctx *ioctx) { struct iommu_unit *iommu; struct iommu_domain *iodom; int error; iodom = ioctx->domain; iommu = iodom->iommu; IOMMU_LOCK(iommu); iommu_free_ctx_locked(iommu, ioctx); IOMMU_UNLOCK(iommu); /* Since we have a domain per each ctx, remove the domain too. */ error = iommu_domain_free(iodom); if (error) device_printf(iommu->dev, "Could not free a domain\n"); } static void iommu_domain_free_entry(struct iommu_map_entry *entry, bool free) { iommu_gas_free_space(entry); if (free) iommu_gas_free_entry(entry); else entry->flags = 0; } void iommu_domain_unload(struct iommu_domain *iodom, struct iommu_map_entries_tailq *entries, bool cansleep) { struct iommu_map_entry *entry, *entry1; int error __diagused; TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, ("not mapped entry %p %p", iodom, entry)); error = iodom->ops->unmap(iodom, entry->start, entry->end - entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); KASSERT(error == 0, ("unmap %p error %d", iodom, error)); TAILQ_REMOVE(entries, entry, dmamap_link); iommu_domain_free_entry(entry, true); } if (TAILQ_EMPTY(entries)) return; panic("entries map is not empty"); } int iommu_register(struct iommu_unit *iommu) { struct iommu_entry *entry; mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF); entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO); entry->iommu = iommu; IOMMU_LIST_LOCK(); LIST_INSERT_HEAD(&iommu_list, entry, next); IOMMU_LIST_UNLOCK(); iommu_init_busdma(iommu); return (0); } int iommu_unregister(struct iommu_unit *iommu) { struct iommu_entry *entry, *tmp; IOMMU_LIST_LOCK(); LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) { if (entry->iommu == iommu) { LIST_REMOVE(entry, next); free(entry, M_IOMMU); } } IOMMU_LIST_UNLOCK(); iommu_fini_busdma(iommu); mtx_destroy(&iommu->lock); return (0); } struct iommu_unit * iommu_find(device_t dev, bool verbose) { struct iommu_entry *entry; struct iommu_unit *iommu; int error; IOMMU_LIST_LOCK(); LIST_FOREACH(entry, &iommu_list, next) { iommu = entry->iommu; error = IOMMU_FIND(iommu->dev, dev); if (error == 0) { IOMMU_LIST_UNLOCK(); return (entry->iommu); } } IOMMU_LIST_UNLOCK(); return (NULL); } void iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free, bool cansleep __unused) { dprintf("%s\n", __func__); iommu_domain_free_entry(entry, free); } static void iommu_init(void) { sx_init(&iommu_sx, "IOMMU list"); } SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL); diff --git a/sys/arm64/iommu/iommu_if.m b/sys/arm64/iommu/iommu_if.m index 6d4f963c5ff1..a3aa8001d00c 100644 --- a/sys/arm64/iommu/iommu_if.m +++ b/sys/arm64/iommu/iommu_if.m @@ -1,149 +1,153 @@ #- # SPDX-License-Identifier: BSD-2-Clause #: # Copyright (c) 2020 Ruslan Bukin # # This software was developed by SRI International and the University of # Cambridge Computer Laboratory (Department of Computer Science and # Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the # DARPA SSITH research programme. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # +#include "opt_platform.h" + #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif INTERFACE iommu; # # Check if the iommu controller dev is responsible to serve traffic # for a given child. # METHOD int find { device_t dev; device_t child; }; # # Map a virtual address VA to a physical address PA. # METHOD int map { device_t dev; struct iommu_domain *iodom; vm_offset_t va; vm_page_t *ma; bus_size_t size; vm_prot_t prot; }; # # Unmap a virtual address VA. # METHOD int unmap { device_t dev; struct iommu_domain *iodom; vm_offset_t va; bus_size_t size; }; # # Allocate an IOMMU domain. # METHOD struct iommu_domain * domain_alloc { device_t dev; struct iommu_unit *iommu; }; # # Release all the resources held by IOMMU domain. # METHOD void domain_free { device_t dev; struct iommu_domain *iodom; }; # # Find a domain allocated for a dev. # METHOD struct iommu_domain * domain_lookup { device_t dev; }; # # Find an allocated context for a device. # METHOD struct iommu_ctx * ctx_lookup { device_t dev; device_t child; }; # # Allocate a new iommu context. # METHOD struct iommu_ctx * ctx_alloc { device_t dev; struct iommu_domain *iodom; device_t child; bool disabled; }; # # Initialize the new iommu context. # METHOD int ctx_init { device_t dev; struct iommu_ctx *ioctx; }; # # Free the iommu context. # METHOD void ctx_free { device_t dev; struct iommu_ctx *ioctx; }; +#ifdef FDT # # Notify controller we have machine-dependent data. # METHOD int ofw_md_data { device_t dev; struct iommu_ctx *ioctx; pcell_t *cells; int ncells; }; +#endif diff --git a/sys/arm64/iommu/smmu_fdt.c b/sys/arm64/iommu/smmu_fdt.c index e5541b50058f..b33f30e9166a 100644 --- a/sys/arm64/iommu/smmu_fdt.c +++ b/sys/arm64/iommu/smmu_fdt.c @@ -1,208 +1,206 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Ruslan Bukin * * This work was supported by Innovate UK project 105694, "Digital Security * by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "smmuvar.h" static struct ofw_compat_data compat_data[] = { { "arm,smmu-v3", 1 }, { NULL, 0 } }; static int smmu_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) return (ENXIO); device_set_desc(dev, "ARM System MMU (SMMU) v3"); return (BUS_PROBE_DEFAULT); } static int smmu_fdt_attach(device_t dev) { struct smmu_softc *sc; struct smmu_unit *unit; struct iommu_unit *iommu; phandle_t node; int err; int rid; sc = device_get_softc(dev); sc->dev = dev; node = ofw_bus_get_node(dev); rid = 0; sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res[0] == NULL) { device_printf(dev, "Can't allocate memory resource.\n"); err = ENXIO; goto error; } /* * Interrupt lines are "eventq", "priq", "cmdq-sync", "gerror". */ err = ofw_bus_find_string_index(node, "interrupt-names", "eventq", &rid); if (err != 0) { device_printf(dev, "Can't get eventq IRQ.\n"); err = ENXIO; goto error; } sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->res[1] == NULL) { device_printf(dev, "Can't allocate eventq IRQ resource.\n"); err = ENXIO; goto error; } /* * sc->res[2] is reserved for priq IRQ. It is optional and not used * by the SMMU driver. This IRQ line may or may not be provided by * hardware. */ err = ofw_bus_find_string_index(node, "interrupt-names", "cmdq-sync", &rid); if (err != 0) { device_printf(dev, "Can't get cmdq-sync IRQ.\n"); err = ENXIO; goto error; } sc->res[3] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->res[3] == NULL) { device_printf(dev, "Can't allocate cmdq-sync IRQ resource.\n"); err = ENXIO; goto error; } err = ofw_bus_find_string_index(node, "interrupt-names", "gerror", &rid); if (err != 0) { device_printf(dev, "Can't get gerror IRQ.\n"); err = ENXIO; goto error; } sc->res[4] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->res[4] == NULL) { device_printf(dev, "Can't allocate gerror IRQ resource.\n"); err = ENXIO; goto error; } err = smmu_attach(dev); if (err != 0) goto error; unit = &sc->unit; unit->dev = dev; iommu = &unit->iommu; iommu->dev = dev; LIST_INIT(&unit->domain_list); /* Use memory start address as an xref. */ sc->xref = bus_get_resource_start(dev, SYS_RES_MEMORY, 0); err = iommu_register(iommu); if (err) { device_printf(dev, "Failed to register SMMU.\n"); return (ENXIO); } OF_device_register_xref(OF_xref_from_node(node), dev); return (0); error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } /* Failure so free resources. */ smmu_detach(dev); return (err); } static device_method_t smmu_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, smmu_fdt_probe), DEVMETHOD(device_attach, smmu_fdt_attach), DEVMETHOD_END }; DEFINE_CLASS_1(smmu, smmu_fdt_driver, smmu_fdt_methods, sizeof(struct smmu_softc), smmu_driver); -static devclass_t smmu_fdt_devclass; - -EARLY_DRIVER_MODULE(smmu, simplebus, smmu_fdt_driver, smmu_fdt_devclass, - 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); +EARLY_DRIVER_MODULE(smmu, simplebus, smmu_fdt_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);