Page MenuHomeFreeBSD
Files F107143227

D24618.id75304.diff
No OneTemporary
Actions

D24618.id75304.diff
View Options

Index: sys/arm64/arm64/pmap.c
===================================================================
--- sys/arm64/arm64/pmap.c
+++ sys/arm64/arm64/pmap.c
@@ -3402,6 +3402,230 @@
}
#endif /* VM_NRESERVLEVEL > 0 */
+/*
+ * Preallocate l1, l2 page directories for a specific VA range.
+ * This is optional and not in use currently.
+ */
+int
+pmap_bootstrap_smmu(pmap_t pmap, vm_offset_t sva, int count)
+{
+ struct rwlock *lock;
+ pd_entry_t *pde;
+ vm_page_t mpte;
+ vm_offset_t va;
+ int lvl;
+ int i;
+
+ lock = NULL;
+ PMAP_LOCK(pmap);
+
+ va = sva;
+ for (i = 0; i < count; i++) {
+ pde = pmap_pde(pmap, va, &lvl);
+ if (pde != NULL && lvl == 2)
+ return (EEXIST);
+ mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), &lock);
+ if (mpte == NULL)
+ return (ENOMEM);
+ va += L2_SIZE;
+ }
+
+ if (lock != NULL)
+ rw_wunlock(lock);
+ PMAP_UNLOCK(pmap);
+
+ return (0);
+}
+
+/*
+ * Add a single SMMU entry. This function does not sleep.
+ */
+int
+pmap_senter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ vm_prot_t prot, u_int flags)
+{
+ pd_entry_t *pde;
+ pt_entry_t new_l3, orig_l3;
+ pt_entry_t *l3;
+ vm_page_t mpte;
+ int lvl;
+ int rv;
+
+ PMAP_ASSERT_STAGE1(pmap);
+ KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space"));
+
+ va = trunc_page(va);
+ new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT |
+ ATTR_S1_IDX(VM_MEMATTR_DEVICE) | L3_PAGE);
+ if ((prot & VM_PROT_WRITE) == 0)
+ new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO);
+ new_l3 |= ATTR_S1_XN; /* Execute never. */
+ new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER);
+ new_l3 |= ATTR_S1_nG; /* Non global. */
+
+ CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa);
+
+ PMAP_LOCK(pmap);
+
+ /*
+ * In the case that a page table page is not
+ * resident, we are creating it here.
+ */
+retry:
+ pde = pmap_pde(pmap, va, &lvl);
+ if (pde != NULL && lvl == 2) {
+ l3 = pmap_l2_to_l3(pde, va);
+ } else {
+ mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), NULL);
+ if (mpte == NULL) {
+ CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
+ rv = KERN_RESOURCE_SHORTAGE;
+ goto out;
+ }
+ goto retry;
+ }
+
+ orig_l3 = pmap_load(l3);
+ KASSERT(!pmap_l3_valid(orig_l3), ("l3 is valid"));
+
+ /* New mapping */
+ pmap_store(l3, new_l3);
+ dsb(ishst);
+
+ rv = KERN_SUCCESS;
+out:
+ PMAP_UNLOCK(pmap);
+
+ return (rv);
+}
+
+/*
+ * Remove a single SMMU entry.
+ */
+int
+pmap_sremove(pmap_t pmap, vm_offset_t va)
+{
+ pt_entry_t *pte;
+ int lvl;
+ int rc;
+
+ PMAP_LOCK(pmap);
+
+ pte = pmap_pte(pmap, va, &lvl);
+ KASSERT(lvl == 3,
+ ("Invalid SMMU pagetable level: %d != 3", lvl));
+
+ if (pte != NULL) {
+ pmap_clear(pte);
+ rc = KERN_FAILURE;
+ } else
+ rc = KERN_SUCCESS;
+
+ PMAP_UNLOCK(pmap);
+
+ return (rc);
+}
+
+/*
+ * Remove all the allocated L1, L2 pages from SMMU pmap.
+ * All the L3 entires must be cleared in advance, otherwise
+ * this function returns error.
+ */
+int
+pmap_sremove_all(pmap_t pmap)
+{
+ pd_entry_t l0e, *l1, l1e, *l2, l2e;
+ pt_entry_t *l3, l3e;
+ vm_offset_t sva;
+ vm_paddr_t pa;
+ vm_paddr_t pa0;
+ vm_paddr_t pa1;
+ int i, j, k, l;
+ vm_page_t m;
+ vm_page_t m0;
+ vm_page_t m1;
+ int rc;
+
+ PMAP_LOCK(pmap);
+
+ for (sva = VM_MINUSER_ADDRESS, i = pmap_l0_index(sva); i < Ln_ENTRIES;
+ i++) {
+ l0e = pmap->pm_l0[i];
+ if ((l0e & ATTR_DESCR_VALID) == 0) {
+ sva += L0_SIZE;
+ continue;
+ }
+ pa0 = l0e & ~ATTR_MASK;
+ m0 = PHYS_TO_VM_PAGE(pa0);
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0);
+
+ for (j = pmap_l1_index(sva); j < Ln_ENTRIES; j++) {
+ l1e = l1[j];
+ if ((l1e & ATTR_DESCR_VALID) == 0) {
+ sva += L1_SIZE;
+ continue;
+ }
+ if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) {
+ sva += L1_SIZE;
+ continue;
+ }
+ pa1 = l1e & ~ATTR_MASK;
+ m1 = PHYS_TO_VM_PAGE(pa1);
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1);
+
+ for (k = pmap_l2_index(sva); k < Ln_ENTRIES; k++) {
+ l2e = l2[k];
+ if ((l2e & ATTR_DESCR_VALID) == 0) {
+ sva += L2_SIZE;
+ continue;
+ }
+ if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) {
+ sva += L2_SIZE;
+ continue;
+ }
+ pa = l2e & ~ATTR_MASK;
+ m = PHYS_TO_VM_PAGE(pa);
+ l3 = (pt_entry_t *)PHYS_TO_DMAP(pa);
+
+ for (l = pmap_l3_index(sva); l < Ln_ENTRIES;
+ l++, sva += L3_SIZE) {
+ l3e = l3[l];
+ if ((l3e & ATTR_DESCR_VALID) == 0)
+ continue;
+ printf("%s: l3e found for va %jx\n",
+ __func__, sva);
+ rc = KERN_FAILURE;
+ goto out;
+ }
+
+ vm_page_unwire_noq(m1);
+ vm_page_unwire_noq(m);
+ pmap_resident_count_dec(pmap, 1);
+ vm_page_free(m);
+ pmap_clear(&l2[k]);
+ }
+
+ vm_page_unwire_noq(m0);
+ pmap_resident_count_dec(pmap, 1);
+ vm_page_free(m1);
+ pmap_clear(&l1[j]);
+ }
+
+ pmap_resident_count_dec(pmap, 1);
+ vm_page_free(m0);
+ pmap_clear(&pmap->pm_l0[i]);
+ }
+
+ KASSERT(pmap->pm_stats.resident_count == 0,
+ ("Invalid resident count %jd", pmap->pm_stats.resident_count));
+
+ rc = KERN_SUCCESS;
+out:
+ PMAP_UNLOCK(pmap);
+
+ return (rc);
+}
+
/*
* Insert the given physical page (p) at
* the specified virtual address (v) in the
Index: sys/arm64/include/bus_dma_impl.h
===================================================================
--- sys/arm64/include/bus_dma_impl.h
+++ sys/arm64/include/bus_dma_impl.h
@@ -48,6 +48,7 @@
bus_dma_lock_t *lockfunc;
void *lockfuncarg;
int ref_count;
+ int domain;
};
struct bus_dma_impl {
@@ -59,6 +60,7 @@
void *lockfuncarg, bus_dma_tag_t *dmat);
int (*tag_destroy)(bus_dma_tag_t dmat);
bool (*id_mapped)(bus_dma_tag_t, vm_paddr_t, bus_size_t);
+ int (*tag_set_domain)(bus_dma_tag_t);
int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map);
int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags,
Index: sys/arm64/include/pmap.h
===================================================================
--- sys/arm64/include/pmap.h
+++ sys/arm64/include/pmap.h
@@ -187,6 +187,13 @@
int pmap_fault(pmap_t, uint64_t, uint64_t);
+/* System MMU (SMMU). */
+int pmap_senter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ vm_prot_t prot, u_int flags);
+int pmap_sremove(pmap_t pmap, vm_offset_t va);
+int pmap_sremove_all(pmap_t pmap);
+int pmap_bootstrap_smmu(pmap_t pmap, vm_offset_t sva, int count);
+
struct pcb *pmap_switch(struct thread *, struct thread *);
extern void (*pmap_clean_stage2_tlbi)(void);
Index: sys/arm64/iommu/iommu.h
===================================================================
--- /dev/null
+++ sys/arm64/iommu/iommu.h
@@ -0,0 +1,84 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _DEV_IOMMU_IOMMU_H_
+#define _DEV_IOMMU_IOMMU_H_
+
+#define IOMMU_PAGE_SIZE 4096
+#define IOMMU_PAGE_MASK (IOMMU_PAGE_SIZE - 1)
+
+#define IOMMU_MF_CANWAIT 0x0001
+#define IOMMU_MF_CANSPLIT 0x0002
+
+#define IOMMU_PGF_WAITOK 0x0001
+#define IOMMU_PGF_ZERO 0x0002
+#define IOMMU_PGF_ALLOC 0x0004
+#define IOMMU_PGF_NOALLOC 0x0008
+#define IOMMU_PGF_OBJL 0x0010
+
+struct smmu_unit {
+ struct iommu_unit unit;
+ LIST_HEAD(, smmu_domain) domain_list;
+ LIST_ENTRY(smmu_unit) next;
+ device_t dev;
+ intptr_t xref;
+};
+
+struct smmu_domain {
+ struct iommu_domain domain;
+ LIST_HEAD(, smmu_ctx) ctx_list;
+ LIST_ENTRY(smmu_domain) next;
+ u_int entries_cnt;
+ struct smmu_cd *cd;
+ struct pmap p;
+ uint16_t asid;
+};
+
+struct smmu_ctx {
+ struct iommu_ctx ctx;
+ struct smmu_domain *domain;
+ LIST_ENTRY(smmu_ctx) next;
+ device_t dev;
+ uint16_t rid;
+ bool bypass;
+ int sid;
+};
+
+int iommu_unregister(device_t dev);
+int iommu_register(device_t dev, struct smmu_unit *unit, intptr_t xref);
+int iommu_map_page(struct smmu_domain *domain, vm_offset_t va, vm_paddr_t pa,
+ vm_prot_t prot);
+int iommu_unmap_page(struct smmu_domain *domain, vm_offset_t va);
+
+#endif /* _DEV_IOMMU_IOMMU_H_ */
Index: sys/arm64/iommu/iommu.c
===================================================================
--- /dev/null
+++ sys/arm64/iommu/iommu.c
@@ -0,0 +1,592 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_acpi.h"
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/module.h>
+#include <sys/queue.h>
+#include <sys/rman.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <vm/vm.h>
+#include <vm/uma.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+#ifdef DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <dev/acpica/acpivar.h>
+#include <dev/acpica/acpi_pcibvar.h>
+#endif
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/busdma_iommu.h>
+
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/intr.h>
+
+#include "iommu.h"
+#include "iommu_if.h"
+
+static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
+static MALLOC_DEFINE(M_BUSDMA, "SMMU", "ARM64 busdma SMMU");
+
+#define IOMMU_LIST_LOCK() mtx_lock(&iommu_mtx)
+#define IOMMU_LIST_UNLOCK() mtx_unlock(&iommu_mtx)
+#define IOMMU_LIST_ASSERT_LOCKED() mtx_assert(&iommu_mtx, MA_OWNED)
+
+#define dprintf(fmt, ...)
+
+#define GICV3_ITS_PAGE 0x300b0000
+
+static struct mtx iommu_mtx;
+static LIST_HEAD(, smmu_unit) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
+
+static void
+smmu_domain_unload_task(void *arg, int pending)
+{
+ struct iommu_domain *iodom;
+ struct smmu_domain *domain;
+ struct iommu_map_entries_tailq entries;
+
+ domain = arg;
+ iodom = (struct iommu_domain *)domain;
+ TAILQ_INIT(&entries);
+
+ for (;;) {
+ IOMMU_DOMAIN_LOCK(iodom);
+ TAILQ_SWAP(&domain->domain.unload_entries, &entries,
+ iommu_map_entry, dmamap_link);
+ IOMMU_DOMAIN_UNLOCK(iodom);
+ if (TAILQ_EMPTY(&entries))
+ break;
+ iommu_domain_unload(iodom, &entries, true);
+ }
+}
+
+static int
+domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
+ iommu_gaddr_t size, int flags)
+{
+ struct smmu_domain *domain;
+ struct smmu_unit *unit;
+ int error;
+
+ unit = (struct smmu_unit *)iodom->iommu;
+ domain = (struct smmu_domain *)iodom;
+
+ error = IOMMU_UNMAP(unit->dev, domain, base, size);
+
+ return (error);
+}
+
+static int
+domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
+ iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
+{
+ struct smmu_unit *unit;
+ struct smmu_domain *domain;
+ vm_prot_t prot;
+ vm_offset_t va;
+ vm_paddr_t pa;
+ int error;
+
+ domain = (struct smmu_domain *)iodom;
+
+ dprintf("%s: base %lx, size %lx\n", __func__, base, size);
+
+ prot = 0;
+ if (eflags & IOMMU_MAP_ENTRY_READ)
+ prot |= VM_PROT_READ;
+ if (eflags & IOMMU_MAP_ENTRY_WRITE)
+ prot |= VM_PROT_WRITE;
+
+ va = base;
+ pa = VM_PAGE_TO_PHYS(ma[0]);
+
+ unit = (struct smmu_unit *)iodom->iommu;
+ error = IOMMU_MAP(unit->dev, domain, va, pa, size, prot);
+
+ return (0);
+}
+
+static const struct iommu_domain_map_ops smmu_domain_map_ops = {
+ .map = domain_map_buf,
+ .unmap = domain_unmap_buf,
+};
+
+static struct smmu_domain *
+smmu_domain_alloc(struct iommu_unit *unit)
+{
+ struct smmu_unit *iommu;
+ struct smmu_domain *domain;
+
+ iommu = (struct smmu_unit *)unit;
+
+ domain = IOMMU_DOMAIN_ALLOC(iommu->dev);
+ if (domain == NULL)
+ return (NULL);
+
+ LIST_INIT(&domain->ctx_list);
+
+ RB_INIT(&domain->domain.rb_root);
+ TAILQ_INIT(&domain->domain.unload_entries);
+ TASK_INIT(&domain->domain.unload_task, 0, smmu_domain_unload_task,
+ domain);
+ mtx_init(&domain->domain.lock, "IOMMU domain", NULL, MTX_DEF);
+
+ domain->domain.iommu = unit;
+ domain->domain.end = BUS_SPACE_MAXADDR;
+ iommu_gas_init_domain(&domain->domain);
+ domain->domain.ops = &smmu_domain_map_ops;
+
+ IOMMU_LOCK(unit);
+ LIST_INSERT_HEAD(&iommu->domain_list, domain, next);
+ IOMMU_UNLOCK(unit);
+
+ return (domain);
+}
+
+static int
+smmu_domain_free(struct smmu_domain *domain)
+{
+ struct iommu_unit *unit;
+ struct smmu_unit *iommu;
+ int error;
+
+ unit = domain->domain.iommu;
+ iommu = (struct smmu_unit *)unit;
+
+ IOMMU_LOCK(unit);
+ LIST_REMOVE(domain, next);
+ error = IOMMU_DOMAIN_FREE(iommu->dev, domain);
+ if (error) {
+ LIST_INSERT_HEAD(&iommu->domain_list, domain, next);
+ IOMMU_UNLOCK(unit);
+ return (error);
+ }
+
+ IOMMU_UNLOCK(unit);
+
+ return (0);
+}
+
+static struct smmu_ctx *
+smmu_ctx_lookup(device_t dev)
+{
+ struct smmu_domain *domain;
+ struct smmu_ctx *ctx;
+ struct smmu_unit *iommu;
+
+ LIST_FOREACH(iommu, &iommu_list, next) {
+ LIST_FOREACH(domain, &iommu->domain_list, next) {
+ LIST_FOREACH(ctx, &domain->ctx_list, next) {
+ if (ctx->dev == dev)
+ return (ctx);
+ }
+ }
+ }
+
+ return (NULL);
+}
+
+static void
+smmu_tag_init(struct bus_dma_tag_iommu *t)
+{
+ bus_addr_t maxaddr;
+
+ maxaddr = BUS_SPACE_MAXADDR;
+
+ t->common.ref_count = 0;
+ t->common.impl = &bus_dma_iommu_impl;
+ t->common.boundary = 0;
+ t->common.lowaddr = maxaddr;
+ t->common.highaddr = maxaddr;
+ t->common.maxsize = maxaddr;
+ t->common.nsegments = BUS_SPACE_UNRESTRICTED;
+ t->common.maxsegsz = maxaddr;
+}
+
+static struct smmu_ctx *
+smmu_ctx_alloc(device_t dev)
+{
+ struct smmu_ctx *ctx;
+
+ ctx = malloc(sizeof(struct smmu_ctx), M_IOMMU, M_WAITOK | M_ZERO);
+ ctx->rid = pci_get_rid(dev);
+ ctx->dev = dev;
+
+ return (ctx);
+}
+/*
+ * Attach a consumer device to a domain.
+ */
+static int
+smmu_ctx_attach(struct smmu_domain *domain, struct smmu_ctx *ctx)
+{
+ struct iommu_domain *iodom;
+ struct smmu_unit *iommu;
+ int error;
+
+ iommu = (struct smmu_unit *)domain->domain.iommu;
+
+ error = IOMMU_CTX_ATTACH(iommu->dev, domain, ctx);
+ if (error) {
+ device_printf(iommu->dev, "Failed to add ctx\n");
+ return (error);
+ }
+
+ ctx->domain = domain;
+
+ iodom = (struct iommu_domain *)domain;
+
+ IOMMU_DOMAIN_LOCK(iodom);
+ LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
+ IOMMU_DOMAIN_UNLOCK(iodom);
+
+ return (error);
+}
+
+
+struct iommu_ctx *
+iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
+ uint16_t rid, bool disabled, bool rmrr)
+{
+ struct smmu_ctx *ctx;
+ struct smmu_domain *domain;
+ struct bus_dma_tag_iommu *tag;
+ int error;
+
+ ctx = smmu_ctx_lookup(requester);
+ if (ctx)
+ return (&ctx->ctx);
+
+ ctx = smmu_ctx_alloc(requester);
+ if (ctx == NULL)
+ return (NULL);
+
+ if (disabled)
+ ctx->bypass = true;
+
+ /* In our current configuration we have a domain per each ctx. */
+ domain = smmu_domain_alloc(iommu);
+ if (domain == NULL)
+ return (NULL);
+
+ tag = ctx->ctx.tag = malloc(sizeof(struct bus_dma_tag_iommu),
+ M_IOMMU, M_WAITOK | M_ZERO);
+ tag->owner = requester;
+ tag->ctx = (struct iommu_ctx *)ctx;
+ tag->ctx->domain = (struct iommu_domain *)domain;
+
+ smmu_tag_init(tag);
+
+ ctx->domain = domain;
+
+ /* Reserve the GIC page */
+ error = iommu_gas_reserve_region(&domain->domain, GICV3_ITS_PAGE,
+ GICV3_ITS_PAGE + PAGE_SIZE);
+ if (error != 0) {
+ smmu_domain_free(domain);
+ return (NULL);
+ }
+
+ /* Map the GICv3 ITS page so the device could send MSI interrupts. */
+ iommu_map_page(domain, GICV3_ITS_PAGE, GICV3_ITS_PAGE, VM_PROT_WRITE);
+
+ error = smmu_ctx_attach(domain, ctx);
+ if (error) {
+ smmu_domain_free(domain);
+ return (NULL);
+ }
+
+ return (&ctx->ctx);
+}
+
+void
+iommu_free_ctx_locked(struct iommu_unit *unit, struct iommu_ctx *ctx)
+{
+ struct smmu_domain *domain;
+ struct smmu_unit *iommu;
+ int error;
+
+ IOMMU_ASSERT_LOCKED(unit);
+
+ domain = (struct smmu_domain *)ctx->domain;
+ iommu = (struct smmu_unit *)unit;
+
+ error = IOMMU_CTX_DETACH(iommu->dev, (struct smmu_ctx *)ctx);
+ if (error) {
+ device_printf(iommu->dev, "Failed to remove device\n");
+ return;
+ }
+
+ LIST_REMOVE((struct smmu_ctx *)ctx, next);
+ free(ctx->tag, M_IOMMU);
+
+ IOMMU_UNLOCK(unit);
+
+ /* Since we have a domain per each ctx, remove the domain too. */
+ iommu_unmap_page(domain, GICV3_ITS_PAGE);
+ error = smmu_domain_free(domain);
+ if (error)
+ device_printf(iommu->dev, "Could not free a domain\n");
+}
+
+void
+iommu_free_ctx(struct iommu_ctx *ctx)
+{
+ struct iommu_unit *iommu;
+ struct iommu_domain *domain;
+
+ domain = ctx->domain;
+ iommu = domain->iommu;
+
+ IOMMU_LOCK(iommu);
+ iommu_free_ctx_locked(iommu, ctx);
+}
+
+int
+iommu_map_page(struct smmu_domain *domain,
+ vm_offset_t va, vm_paddr_t pa, vm_prot_t prot)
+{
+ struct smmu_unit *iommu;
+ int error;
+
+ iommu = (struct smmu_unit *)domain->domain.iommu;
+
+ error = IOMMU_MAP(iommu->dev, domain, va, pa, PAGE_SIZE, prot);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+int
+iommu_unmap_page(struct smmu_domain *domain, vm_offset_t va)
+{
+ struct smmu_unit *iommu;
+ int error;
+
+ iommu = (struct smmu_unit *)domain->domain.iommu;
+
+ error = IOMMU_UNMAP(iommu->dev, domain, va, PAGE_SIZE);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+static void
+smmu_domain_free_entry(struct iommu_map_entry *entry, bool free)
+{
+ struct iommu_domain *domain;
+
+ domain = entry->domain;
+
+ IOMMU_DOMAIN_LOCK(domain);
+ iommu_gas_free_space(domain, entry);
+ IOMMU_DOMAIN_UNLOCK(domain);
+
+ if (free)
+ iommu_gas_free_entry(domain, entry);
+ else
+ entry->flags = 0;
+}
+
+void
+iommu_domain_unload(struct iommu_domain *domain,
+ struct iommu_map_entries_tailq *entries, bool cansleep)
+{
+ struct smmu_unit *unit;
+ struct iommu_map_entry *entry, *entry1;
+ int error;
+
+ unit = (struct smmu_unit *)domain->iommu;
+
+ TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
+ KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
+ ("not mapped entry %p %p", domain, entry));
+ error = domain->ops->unmap(domain, entry->start, entry->end -
+ entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
+ KASSERT(error == 0, ("unmap %p error %d", domain, error));
+ TAILQ_REMOVE(entries, entry, dmamap_link);
+ smmu_domain_free_entry(entry, true);
+ }
+
+ if (TAILQ_EMPTY(entries))
+ return;
+
+ panic("here");
+}
+
+int
+iommu_register(device_t dev, struct smmu_unit *iommu, intptr_t xref)
+{
+
+ iommu->dev = dev;
+ iommu->xref = xref;
+
+ LIST_INIT(&iommu->domain_list);
+ mtx_init(&iommu->unit.lock, "IOMMU", NULL, MTX_DEF);
+
+ IOMMU_LIST_LOCK();
+ LIST_INSERT_HEAD(&iommu_list, iommu, next);
+ IOMMU_LIST_UNLOCK();
+
+ iommu_init_busdma((struct iommu_unit *)iommu);
+
+ return (0);
+}
+
+int
+iommu_unregister(device_t dev)
+{
+ struct smmu_unit *iommu;
+ bool found;
+
+ found = false;
+
+ IOMMU_LIST_LOCK();
+ LIST_FOREACH(iommu, &iommu_list, next) {
+ if (iommu->dev == dev) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ IOMMU_LIST_UNLOCK();
+ return (ENOENT);
+ }
+
+ if (!LIST_EMPTY(&iommu->domain_list)) {
+ IOMMU_LIST_UNLOCK();
+ return (EBUSY);
+ }
+
+ LIST_REMOVE(iommu, next);
+ IOMMU_LIST_UNLOCK();
+
+ free(iommu, M_IOMMU);
+
+ return (0);
+}
+
+static struct smmu_unit *
+smmu_lookup(intptr_t xref)
+{
+ struct smmu_unit *iommu;
+
+ LIST_FOREACH(iommu, &iommu_list, next) {
+ if (iommu->xref == xref)
+ return (iommu);
+ }
+
+ return (NULL);
+}
+
+struct iommu_unit *
+iommu_find(device_t dev, bool verbose)
+{
+ struct smmu_unit *iommu;
+ u_int xref, sid;
+ uint16_t rid;
+ int error;
+ int seg;
+
+ rid = pci_get_rid(dev);
+ seg = pci_get_domain(dev);
+
+ /*
+ * Find an xref of an IOMMU controller that serves traffic for dev.
+ */
+#ifdef DEV_ACPI
+ error = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
+ if (error) {
+ /* Could not find reference to an SMMU device. */
+ return (NULL);
+ }
+#else
+ /* TODO: add FDT support. */
+ return (NULL);
+#endif
+
+ /*
+ * Find a registered IOMMU controller by xref.
+ */
+ iommu = smmu_lookup(xref);
+ if (iommu == NULL) {
+ /* SMMU device is not registered in the IOMMU framework. */
+ return (NULL);
+ }
+
+ return ((struct iommu_unit *)iommu);
+}
+
+void
+iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free)
+{
+
+ dprintf("%s\n", __func__);
+
+ smmu_domain_free_entry(entry, free);
+}
+
+static void
+iommu_init(void)
+{
+
+ mtx_init(&iommu_mtx, "IOMMU", NULL, MTX_DEF);
+}
+
+SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);
Index: sys/arm64/iommu/iommu_if.m
===================================================================
--- /dev/null
+++ sys/arm64/iommu/iommu_if.m
@@ -0,0 +1,105 @@
+#-
+# SPDX-License-Identifier: BSD-2-Clause
+#:
+# Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+#
+# This software was developed by SRI International and the University of
+# Cambridge Computer Laboratory (Department of Computer Science and
+# Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+# DARPA SSITH research programme.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+#include <vm/vm.h>
+
+#include <sys/types.h>
+#include <sys/taskqueue.h>
+#include <sys/bus.h>
+#include <sys/sysctl.h>
+#include <sys/tree.h>
+#include <sys/mutex.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include <arm64/iommu/iommu.h>
+
+INTERFACE iommu;
+
+#
+# Map a virtual address VA to a physical address PA.
+#
+METHOD int map {
+ device_t dev;
+ struct smmu_domain *domain;
+ vm_offset_t va;
+ vm_paddr_t pa;
+ bus_size_t size;
+ vm_prot_t prot;
+};
+
+#
+# Unmap a virtual address VA.
+#
+METHOD int unmap {
+ device_t dev;
+ struct smmu_domain *domain;
+ vm_offset_t va;
+ bus_size_t size;
+};
+
+#
+# Allocate an IOMMU domain.
+#
+METHOD struct smmu_domain * domain_alloc {
+ device_t dev;
+};
+
+#
+# Release all the resources held by IOMMU domain.
+#
+METHOD int domain_free {
+ device_t dev;
+ struct smmu_domain *domain;
+};
+
+#
+# Attach a consumer device to a IOMMU domain.
+#
+METHOD int ctx_attach {
+ device_t dev;
+ struct smmu_domain *domain;
+ struct smmu_ctx *ctx;
+};
+
+#
+# Detach a consumer device from IOMMU domain.
+#
+METHOD int ctx_detach {
+ device_t dev;
+ struct smmu_ctx *ctx;
+};
Index: sys/arm64/iommu/smmu.c
===================================================================
--- /dev/null
+++ sys/arm64/iommu/smmu.c
@@ -0,0 +1,1784 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Hardware overview.
+ *
+ * An incoming transaction from a peripheral device has an address, size,
+ * attributes and StreamID.
+ *
+ * In case of PCI-based devices, StreamID is a PCI rid.
+ *
+ * The StreamID is used to select a Stream Table Entry (STE) in a Stream table,
+ * which contains per-device configuration.
+ *
+ * Stream table is a linear or 2-level walk table (this driver supports both).
+ * Note that a linear table could occupy 1GB or more of memory depending on
+ * sid_bits value.
+ *
+ * STE is used to locate a Context Descriptor, which is a struct in memory
+ * that describes stages of translation, translation table type, pointer to
+ * level 0 of page tables, ASID, etc.
+ *
+ * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2):
+ * o S1 is used for the host machine traffic translation
+ * o S2 is for a hypervisor
+ *
+ * This driver enables S1 stage with standard AArch64 page tables.
+ *
+ * Note that SMMU does not share TLB with a main CPU.
+ * Command queue is used by this driver to Invalidate SMMU TLB, STE cache.
+ *
+ * An arm64 SoC could have more than one SMMU instance.
+ * ACPI IORT table describes which SMMU unit is assigned for a particular
+ * peripheral device.
+ *
+ * Queues.
+ *
+ * Register interface and Memory-based circular buffer queues are used
+ * to inferface SMMU.
+ *
+ * These are a Command queue for commands to send to the SMMU and an Event
+ * queue for event/fault reports from the SMMU. Optionally PRI queue is for
+ * receipt of PCIe page requests.
+ *
+ * Note that not every hardware supports PRI services. For instance they were
+ * not found in Neoverse N1 SDP machine.
+ * (This drivers does not implement PRI queue.)
+ *
+ * All SMMU queues are arranged as circular buffers in memory. They are used
+ * in a producer-consumer fashion so that an output queue contains data
+ * produced by the SMMU and consumed by software.
+ * An input queue contains data produced by software, consumed by the SMMU.
+ *
+ * Interrupts.
+ *
+ * Interrupts are not required by this driver for normal operation.
+ * The standard wired interrupt is only triggered when an event comes from
+ * the SMMU, which is only in a case of a translation fault.
+ */
+
+#include "opt_platform.h"
+#include "opt_acpi.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bitstring.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/rman.h>
+#include <sys/mutex.h>
+#include <sys/lock.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#if DEV_ACPI
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <dev/acpica/acpivar.h>
+#include <dev/acpica/acpi_pcibvar.h>
+#endif
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/intr.h>
+
+#include "iommu.h"
+#include "iommu_if.h"
+
+#include "smmu_reg.h"
+#include "smmu_var.h"
+
+#define STRTAB_L1_SZ_SHIFT 20
+#define STRTAB_SPLIT 8
+
+#define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6)
+#define STRTAB_L1_DESC_DWORDS 1
+
+#define STRTAB_STE_DWORDS 8
+
+#define CMDQ_ENTRY_DWORDS 2
+#define EVTQ_ENTRY_DWORDS 4
+#define PRIQ_ENTRY_DWORDS 2
+
+#define CD_DWORDS 8
+
+#define Q_WRP(q, p) ((p) & (1 << (q)->size_log2))
+#define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1))
+#define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */
+
+#define SMMU_Q_ALIGN (64 * 1024)
+
+static struct resource_spec smmu_spec[] = {
+ { SYS_RES_MEMORY, 0, RF_ACTIVE },
+ { SYS_RES_IRQ, 0, RF_ACTIVE },
+ { SYS_RES_IRQ, 1, RF_ACTIVE },
+ { SYS_RES_IRQ, 2, RF_ACTIVE },
+ RESOURCE_SPEC_END
+};
+
+MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR);
+
+struct smmu_event {
+ int ident;
+ char *str;
+ char *msg;
+};
+
+static struct smmu_event events[] = {
+ { 0x01, "F_UUT",
+ "Unsupported Upstream Transaction."},
+ { 0x02, "C_BAD_STREAMID",
+ "Transaction StreamID out of range."},
+ { 0x03, "F_STE_FETCH",
+ "Fetch of STE caused external abort."},
+ { 0x04, "C_BAD_STE",
+ "Used STE invalid."},
+ { 0x05, "F_BAD_ATS_TREQ",
+ "Address Translation Request disallowed for a StreamID "
+ "and a PCIe ATS Translation Request received."},
+ { 0x06, "F_STREAM_DISABLED",
+ "The STE of a transaction marks non-substream transactions "
+ "disabled."},
+ { 0x07, "F_TRANSL_FORBIDDEN",
+ "An incoming PCIe transaction is marked Translated but "
+ "SMMU bypass is disallowed for this StreamID."},
+ { 0x08, "C_BAD_SUBSTREAMID",
+ "Incoming SubstreamID present, but configuration is invalid."},
+ { 0x09, "F_CD_FETCH",
+ "Fetch of CD caused external abort."},
+ { 0x0a, "C_BAD_CD",
+ "Fetched CD invalid."},
+ { 0x0b, "F_WALK_EABT",
+ "An external abort occurred fetching (or updating) "
+ "a translation table descriptor."},
+ { 0x10, "F_TRANSLATION",
+ "Translation fault."},
+ { 0x11, "F_ADDR_SIZE",
+ "Address Size fault."},
+ { 0x12, "F_ACCESS",
+ "Access flag fault due to AF == 0 in a page or block TTD."},
+ { 0x13, "F_PERMISSION",
+ "Permission fault occurred on page access."},
+ { 0x20, "F_TLB_CONFLICT",
+ "A TLB conflict occurred because of the transaction."},
+ { 0x21, "F_CFG_CONFLICT",
+ "A configuration cache conflict occurred due to "
+ "the transaction."},
+ { 0x24, "E_PAGE_REQUEST",
+ "Speculative page request hint."},
+ { 0x25, "F_VMS_FETCH",
+ "Fetch of VMS caused external abort."},
+ { 0, NULL, NULL },
+};
+
+static int
+smmu_q_has_space(struct smmu_queue *q)
+{
+
+ /*
+ * See 6.3.27 SMMU_CMDQ_PROD
+ *
+ * There is space in the queue for additional commands if:
+ * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR ||
+ * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP
+ */
+
+ if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||
+ Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
+ return (1);
+
+ return (0);
+}
+
+static int
+smmu_q_empty(struct smmu_queue *q)
+{
+
+ if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&
+ Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
+ return (1);
+
+ return (0);
+}
+
+static int __unused
+smmu_q_consumed(struct smmu_queue *q, uint32_t prod)
+{
+
+ if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&
+ (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))
+ return (1);
+
+ if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&
+ (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))
+ return (1);
+
+ return (0);
+}
+
+static uint32_t
+smmu_q_inc_cons(struct smmu_queue *q)
+{
+ uint32_t cons;
+ uint32_t val;
+
+ cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;
+ val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));
+
+ return (val);
+}
+
+static uint32_t
+smmu_q_inc_prod(struct smmu_queue *q)
+{
+ uint32_t prod;
+ uint32_t val;
+
+ prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;
+ val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));
+
+ return (val);
+}
+
+static int
+smmu_write_ack(struct smmu_softc *sc, uint32_t reg,
+ uint32_t reg_ack, uint32_t val)
+{
+ uint32_t v;
+ int timeout;
+
+ timeout = 100000;
+
+ bus_write_4(sc->res[0], reg, val);
+
+ do {
+ v = bus_read_4(sc->res[0], reg_ack);
+ if (v == val)
+ break;
+ } while (timeout--);
+
+ if (timeout <= 0) {
+ device_printf(sc->dev, "Failed to write reg.\n");
+ return (-1);
+ }
+
+ return (0);
+}
+
+static inline int
+ilog2(long x)
+{
+
+ KASSERT(x > 0 && powerof2(x),
+ ("%s: invalid arg %ld", __func__, x));
+
+ return (flsl(x) - 1);
+}
+
+static int
+smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q,
+ uint32_t prod_off, uint32_t cons_off, uint32_t dwords)
+{
+ int sz;
+
+ sz = (1 << q->size_log2) * dwords * 8;
+
+ /* Set up the command circular buffer */
+ q->vaddr = contigmalloc(sz, M_SMMU,
+ M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);
+ if (q->vaddr == NULL) {
+ device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
+ return (-1);
+ }
+
+ q->prod_off = prod_off;
+ q->cons_off = cons_off;
+ q->paddr = vtophys(q->vaddr);
+
+ q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;
+ q->base |= q->paddr & Q_BASE_ADDR_M;
+ q->base |= q->size_log2 << Q_LOG2SIZE_S;
+
+ return (0);
+}
+
+static int
+smmu_init_queues(struct smmu_softc *sc)
+{
+ int err;
+
+ /* Command queue. */
+ err = smmu_init_queue(sc, &sc->cmdq,
+ SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS);
+ if (err)
+ return (ENXIO);
+
+ /* Event queue. */
+ err = smmu_init_queue(sc, &sc->evtq,
+ SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS);
+ if (err)
+ return (ENXIO);
+
+ if (!(sc->features & SMMU_FEATURE_PRI))
+ return (0);
+
+ /* PRI queue. */
+ err = smmu_init_queue(sc, &sc->priq,
+ SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS);
+ if (err)
+ return (ENXIO);
+
+ return (0);
+}
+
+/*
+ * Dump 2LVL or linear STE.
+ */
+static void
+smmu_dump_ste(struct smmu_softc *sc, int sid)
+{
+ struct smmu_strtab *strtab;
+ struct l1_desc *l1_desc;
+ uint64_t *ste, *l1;
+ int i;
+
+ strtab = &sc->strtab;
+
+ if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+ i = sid >> STRTAB_SPLIT;
+ l1 = (void *)((uint64_t)strtab->vaddr +
+ STRTAB_L1_DESC_DWORDS * 8 * i);
+ device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);
+
+ l1_desc = &strtab->l1[i];
+ ste = l1_desc->va;
+ if (ste == NULL) /* L2 is not initialized */
+ return;
+ } else {
+ ste = (void *)((uint64_t)strtab->vaddr +
+ sid * (STRTAB_STE_DWORDS << 3));
+ }
+
+ /* Dump L2 or linear STE. */
+ for (i = 0; i < STRTAB_STE_DWORDS; i++)
+ device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);
+}
+
+static void __unused
+smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd)
+{
+ uint64_t *vaddr;
+ int i;
+
+ device_printf(sc->dev, "%s\n", __func__);
+
+ vaddr = cd->vaddr;
+ for (i = 0; i < CD_DWORDS; i++)
+ device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);
+}
+
+static void
+smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt)
+{
+ struct smmu_queue *evtq;
+ void *entry_addr;
+
+ evtq = &sc->evtq;
+
+ evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);
+ entry_addr = (void *)((uint64_t)evtq->vaddr +
+ evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);
+ memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8);
+ evtq->lc.cons = smmu_q_inc_cons(evtq);
+ bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);
+}
+
+static void
+smmu_print_event(struct smmu_softc *sc, uint32_t *evt)
+{
+ struct smmu_event *ev;
+ uint64_t input_addr;
+ uint8_t event_id;
+ int sid;
+ int i;
+
+ ev = NULL;
+ event_id = evt[0] & 0xff;
+ for (i = 0; events[i].ident != 0; i++) {
+ if (events[i].ident == event_id) {
+ ev = &events[i];
+ break;
+ }
+ }
+
+ if (ev) {
+ device_printf(sc->dev,
+ "Event %s (%s) received.\n", ev->str, ev->msg);
+ } else
+ device_printf(sc->dev, "Event 0x%x received\n", event_id);
+
+ sid = evt[1];
+ input_addr = evt[5];
+ input_addr <<= 32;
+ input_addr |= evt[4];
+
+ device_printf(sc->dev, "SID %x, Input Address: %jx\n",
+ sid, input_addr);
+
+ for (i = 0; i < 8; i++)
+ device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);
+
+ smmu_dump_ste(sc, sid);
+}
+
+static void
+make_cmd(struct smmu_softc *sc, uint64_t *cmd,
+ struct smmu_cmdq_entry *entry)
+{
+
+ memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8);
+ cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;
+
+ switch (entry->opcode) {
+ case CMD_TLBI_NH_VA:
+ cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
+ cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;
+ if (entry->tlbi.leaf) {
+ /*
+ * Leaf flag means that only cached entries
+ * for the last level of translation table walk
+ * are required to be invalidated.
+ */
+ cmd[1] |= TLBI_1_LEAF;
+ }
+ break;
+ case CMD_TLBI_NH_ASID:
+ cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
+ break;
+ case CMD_TLBI_NSNH_ALL:
+ case CMD_TLBI_NH_ALL:
+ case CMD_TLBI_EL2_ALL:
+ break;
+ case CMD_CFGI_CD:
+ cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);
+ /* FALLTROUGH */
+ case CMD_CFGI_STE:
+ cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);
+ cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);
+ break;
+ case CMD_CFGI_STE_RANGE:
+ cmd[1] = (31 << CFGI_1_STE_RANGE_S);
+ break;
+ case CMD_SYNC:
+ cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB;
+ if (entry->sync.msiaddr) {
+ cmd[0] |= SYNC_0_CS_SIG_IRQ;
+ cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);
+ } else
+ cmd[0] |= SYNC_0_CS_SIG_SEV;
+ break;
+ case CMD_PREFETCH_CONFIG:
+ cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);
+ break;
+ };
+}
+
+static void
+smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry)
+{
+ uint64_t cmd[CMDQ_ENTRY_DWORDS];
+ struct smmu_queue *cmdq;
+ void *entry_addr;
+
+ cmdq = &sc->cmdq;
+
+ make_cmd(sc, cmd, entry);
+
+ SMMU_LOCK(sc);
+
+ /* Ensure that a space is available. */
+ do {
+ cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);
+ } while (smmu_q_has_space(cmdq) == 0);
+
+ /* Write the command to the current prod entry. */
+ entry_addr = (void *)((uint64_t)cmdq->vaddr +
+ Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);
+ memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8);
+
+ /* Increment prod index. */
+ cmdq->lc.prod = smmu_q_inc_prod(cmdq);
+ bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);
+
+ SMMU_UNLOCK(sc);
+}
+
+static void __unused
+smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q)
+{
+
+ while (1) {
+ q->lc.val = bus_read_8(sc->res[0], q->prod_off);
+ if (smmu_q_empty(q))
+ break;
+ cpu_spinwait();
+ }
+}
+
+static int
+smmu_sync(struct smmu_softc *sc)
+{
+ struct smmu_cmdq_entry cmd;
+ struct smmu_queue *q;
+ uint32_t *base;
+ int timeout;
+ int prod;
+
+ q = &sc->cmdq;
+ prod = q->lc.prod;
+
+ /* Enqueue sync command. */
+ cmd.opcode = CMD_SYNC;
+ cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+
+ /* Wait for the sync completion. */
+ base = (void *)((uint64_t)q->vaddr +
+ Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8);
+
+ /*
+ * It takes around 200 loops (6 instructions each)
+ * on Neoverse N1 to complete the sync.
+ */
+ timeout = 10000;
+
+ do {
+ if (*base == 0) {
+ /* MSI write completed. */
+ break;
+ }
+ cpu_spinwait();
+ } while (timeout--);
+
+ if (timeout < 0)
+ device_printf(sc->dev, "Failed to sync\n");
+
+ return (0);
+}
+
+static int
+smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf)
+{
+ struct smmu_cmdq_entry cmd;
+
+ cmd.opcode = CMD_CFGI_CD;
+ cmd.cfgi.sid = sid;
+ cmd.cfgi.ssid = ssid;
+ cmd.cfgi.leaf = leaf;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+
+ return (0);
+}
+
+static void
+smmu_invalidate_all_sid(struct smmu_softc *sc)
+{
+ struct smmu_cmdq_entry cmd;
+
+ /* Invalidate cached config */
+ cmd.opcode = CMD_CFGI_STE_RANGE;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_all(struct smmu_softc *sc)
+{
+ struct smmu_cmdq_entry cmd;
+
+ /* Invalidate entire TLB */
+ cmd.opcode = CMD_TLBI_NSNH_ALL;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid)
+{
+ struct smmu_cmdq_entry cmd;
+
+ /* Invalidate TLB for an ASID. */
+ cmd.opcode = CMD_TLBI_NH_ASID;
+ cmd.tlbi.asid = asid;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ smmu_sync(sc);
+}
+
+static void
+smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid)
+{
+ struct smmu_cmdq_entry cmd;
+
+ /* Invalidate specific range */
+ cmd.opcode = CMD_TLBI_NH_VA;
+ cmd.tlbi.asid = asid;
+ cmd.tlbi.vmid = 0;
+ cmd.tlbi.leaf = true; /* We change only L3. */
+ cmd.tlbi.addr = va;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+}
+
+static void
+smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid)
+{
+ struct smmu_cmdq_entry cmd;
+
+ /* Invalidate cached config */
+ cmd.opcode = CMD_CFGI_STE;
+ cmd.cfgi.sid = sid;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ smmu_sync(sc);
+}
+
+static void
+smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid)
+{
+ struct smmu_cmdq_entry cmd;
+
+ cmd.opcode = CMD_PREFETCH_CONFIG;
+ cmd.prefetch.sid = sid;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ smmu_sync(sc);
+}
+
+/*
+ * Init STE in bypass mode. Traffic is not translated for the sid.
+ */
+static void
+smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste)
+{
+ uint64_t val;
+
+ val = STE0_VALID | STE0_CONFIG_BYPASS;
+
+ ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS;
+ ste[2] = 0;
+ ste[3] = 0;
+ ste[4] = 0;
+ ste[5] = 0;
+ ste[6] = 0;
+ ste[7] = 0;
+
+ smmu_invalidate_sid(sc, sid);
+ ste[0] = val;
+ dsb(sy);
+ smmu_invalidate_sid(sc, sid);
+
+ smmu_prefetch_sid(sc, sid);
+}
+
+/*
+ * Enable Stage1 (S1) translation for the sid.
+ */
+static int
+smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd,
+ uint32_t sid, uint64_t *ste)
+{
+ uint64_t val;
+
+ val = STE0_VALID;
+
+ ste[1] = 0;
+ ste[2] = 0;
+ ste[3] = 0;
+ ste[4] = 0;
+ ste[5] = 0;
+ ste[6] = 0;
+ ste[7] = 0;
+
+ /* S1 */
+ ste[1] |= STE1_EATS_FULLATS
+ | STE1_S1CSH_IS
+ | STE1_S1CIR_WBRA
+ | STE1_S1COR_WBRA
+ | STE1_STRW_NS_EL1;
+
+ if (sc->features & SMMU_FEATURE_STALL &&
+ ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))
+ ste[1] |= STE1_S1STALLD;
+
+ /* Configure STE */
+ val |= (cd->paddr & STE0_S1CONTEXTPTR_M);
+ val |= STE0_CONFIG_S1_TRANS;
+
+ /* One Context descriptor (S1Fmt is IGNORED). */
+
+ /*
+ * Set for a linear table of CDs.
+ *
+ * val |= STE0_S1FMT_LINEAR;
+ * val |= 1 << STE0_S1CDMAX_S;
+ */
+
+ smmu_invalidate_sid(sc, sid);
+
+ /* The STE[0] has to be written in a single blast, last of all. */
+ ste[0] = val;
+ dsb(sy);
+
+ smmu_invalidate_sid(sc, sid);
+ smmu_sync_cd(sc, sid, 0, true);
+ smmu_invalidate_sid(sc, sid);
+
+ /* The sid will be used soon most likely. */
+ smmu_prefetch_sid(sc, sid);
+
+ return (0);
+}
+
+static int
+smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass)
+{
+ struct smmu_strtab *strtab;
+ struct l1_desc *l1_desc;
+ uint64_t *addr;
+
+ strtab = &sc->strtab;
+
+ if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+ l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+ addr = l1_desc->va;
+ } else {
+ addr = (void *)((uint64_t)strtab->vaddr +
+ STRTAB_STE_DWORDS * 8 * sid);
+ };
+
+ if (bypass)
+ smmu_init_ste_bypass(sc, sid, addr);
+ else
+ smmu_init_ste_s1(sc, cd, sid, addr);
+
+ smmu_sync(sc);
+
+ return (0);
+}
+
+static int
+smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain)
+{
+ vm_paddr_t paddr;
+ uint64_t *ptr;
+ uint64_t val;
+ vm_size_t size;
+ struct smmu_cd *cd;
+ pmap_t p;
+
+ size = 1 * (CD_DWORDS << 3);
+
+ p = &domain->p;
+ cd = domain->cd = malloc(sizeof(struct smmu_cd),
+ M_SMMU, M_WAITOK | M_ZERO);
+
+ cd->vaddr = contigmalloc(size, M_SMMU,
+ M_WAITOK | M_ZERO, /* flags */
+ 0, /* low */
+ (1ul << 40) - 1, /* high */
+ size, /* alignment */
+ 0); /* boundary */
+ if (cd->vaddr == NULL) {
+ device_printf(sc->dev, "Failed to allocate CD\n");
+ return (ENXIO);
+ }
+
+ cd->size = size;
+ cd->paddr = vtophys(cd->vaddr);
+
+ ptr = cd->vaddr;
+
+ val = CD0_VALID;
+ val |= CD0_AA64;
+ val |= CD0_R;
+ val |= CD0_A;
+ val |= CD0_ASET;
+ val |= (uint64_t)domain->asid << CD0_ASID_S;
+ val |= CD0_TG0_4KB;
+ val |= CD0_EPD1; /* Disable TT1 */
+ val |= ((64 - sc->ias) << CD0_T0SZ_S);
+ val |= CD0_IPS_48BITS;
+
+ paddr = p->pm_l0_paddr & CD1_TTB0_M;
+ KASSERT(paddr == p->pm_l0_paddr, ("bad allocation 1"));
+
+ ptr[1] = paddr;
+ ptr[2] = 0;
+ ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) |\
+ MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) |\
+ MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) |\
+ MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH);
+
+ /* Install the CD. */
+ ptr[0] = val;
+
+ return (0);
+}
+
+static int
+smmu_init_strtab_linear(struct smmu_softc *sc)
+{
+ struct smmu_strtab *strtab;
+ vm_paddr_t base;
+ uint32_t size;
+ uint64_t reg;
+
+ strtab = &sc->strtab;
+ strtab->num_l1_entries = (1 << sc->sid_bits);
+
+ size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);
+
+ if (bootverbose)
+ device_printf(sc->dev,
+ "%s: linear strtab size %d, num_l1_entries %d\n",
+ __func__, size, strtab->num_l1_entries);
+
+ strtab->vaddr = contigmalloc(size, M_SMMU,
+ M_WAITOK | M_ZERO, /* flags */
+ 0, /* low */
+ (1ul << 48) - 1, /* high */
+ size, /* alignment */
+ 0); /* boundary */
+ if (strtab->vaddr == NULL) {
+ device_printf(sc->dev, "failed to allocate strtab\n");
+ return (ENXIO);
+ }
+
+ reg = STRTAB_BASE_CFG_FMT_LINEAR;
+ reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;
+ strtab->base_cfg = (uint32_t)reg;
+
+ base = vtophys(strtab->vaddr);
+
+ reg = base & STRTAB_BASE_ADDR_M;
+ KASSERT(reg == base, ("bad allocation 2"));
+ reg |= STRTAB_BASE_RA;
+ strtab->base = reg;
+
+ return (0);
+}
+
+static int
+smmu_init_strtab_2lvl(struct smmu_softc *sc)
+{
+ struct smmu_strtab *strtab;
+ vm_paddr_t base;
+ uint64_t reg_base;
+ uint32_t l1size;
+ uint32_t size;
+ uint32_t reg;
+ int sz;
+
+ strtab = &sc->strtab;
+
+ size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
+ size = min(size, sc->sid_bits - STRTAB_SPLIT);
+ strtab->num_l1_entries = (1 << size);
+ size += STRTAB_SPLIT;
+
+ l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);
+
+ if (bootverbose)
+ device_printf(sc->dev,
+ "%s: size %d, l1 entries %d, l1size %d\n",
+ __func__, size, strtab->num_l1_entries, l1size);
+
+ strtab->vaddr = contigmalloc(l1size, M_SMMU,
+ M_WAITOK | M_ZERO, /* flags */
+ 0, /* low */
+ (1ul << 48) - 1, /* high */
+ l1size, /* alignment */
+ 0); /* boundary */
+ if (strtab->vaddr == NULL) {
+ device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");
+ return (ENOMEM);
+ }
+
+ sz = strtab->num_l1_entries * sizeof(struct l1_desc);
+
+ strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);
+ if (strtab->l1 == NULL) {
+ contigfree(strtab->vaddr, l1size, M_SMMU);
+ return (ENOMEM);
+ }
+
+ reg = STRTAB_BASE_CFG_FMT_2LVL;
+ reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S;
+ reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S;
+ strtab->base_cfg = (uint32_t)reg;
+
+ base = vtophys(strtab->vaddr);
+
+ reg_base = base & STRTAB_BASE_ADDR_M;
+ KASSERT(reg_base == base, ("bad allocation 3"));
+ reg_base |= STRTAB_BASE_RA;
+ strtab->base = reg_base;
+
+ return (0);
+}
+
+static int
+smmu_init_strtab(struct smmu_softc *sc)
+{
+ int error;
+
+ if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)
+ error = smmu_init_strtab_2lvl(sc);
+ else
+ error = smmu_init_strtab_linear(sc);
+
+ return (error);
+}
+
+static int
+smmu_init_l1_entry(struct smmu_softc *sc, int sid)
+{
+ struct smmu_strtab *strtab;
+ struct l1_desc *l1_desc;
+ uint64_t *addr;
+ uint64_t val;
+ size_t size;
+ int i;
+
+ strtab = &sc->strtab;
+ l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+
+ size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
+
+ l1_desc->span = STRTAB_SPLIT + 1;
+ l1_desc->size = size;
+ l1_desc->va = contigmalloc(size, M_SMMU,
+ M_WAITOK | M_ZERO, /* flags */
+ 0, /* low */
+ (1ul << 48) - 1, /* high */
+ size, /* alignment */
+ 0); /* boundary */
+ if (l1_desc->va == NULL) {
+ device_printf(sc->dev, "failed to allocate l2 entry\n");
+ return (ENXIO);
+ }
+
+ l1_desc->pa = vtophys(l1_desc->va);
+
+ i = sid >> STRTAB_SPLIT;
+ addr = (void *)((uint64_t)strtab->vaddr +
+ STRTAB_L1_DESC_DWORDS * 8 * i);
+
+ /* Install the L1 entry. */
+ val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;
+ KASSERT(val == l1_desc->pa, ("bad allocation 4"));
+ val |= l1_desc->span;
+ *addr = val;
+
+ return (0);
+}
+
+static void
+smmu_deinit_l1_entry(struct smmu_softc *sc, int sid)
+{
+ struct smmu_strtab *strtab;
+ struct l1_desc *l1_desc;
+ uint64_t *addr;
+ int i;
+
+ strtab = &sc->strtab;
+
+ i = sid >> STRTAB_SPLIT;
+ addr = (void *)((uint64_t)strtab->vaddr +
+ STRTAB_L1_DESC_DWORDS * 8 * i);
+ *addr = 0;
+
+ if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+ l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
+ contigfree(l1_desc->va, l1_desc->size, M_SMMU);
+ }
+}
+
+static int
+smmu_disable(struct smmu_softc *sc)
+{
+ uint32_t reg;
+ int error;
+
+ /* Disable SMMU */
+ reg = bus_read_4(sc->res[0], SMMU_CR0);
+ reg &= ~CR0_SMMUEN;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error)
+ device_printf(sc->dev, "Could not disable SMMU.\n");
+
+ return (0);
+}
+
+static int
+smmu_event_intr(void *arg)
+{
+ uint32_t evt[EVTQ_ENTRY_DWORDS * 2];
+ struct smmu_softc *sc;
+
+ sc = arg;
+
+ do {
+ smmu_evtq_dequeue(sc, evt);
+ smmu_print_event(sc, evt);
+ } while (!smmu_q_empty(&sc->evtq));
+
+ return (FILTER_HANDLED);
+}
+
+static int __unused
+smmu_sync_intr(void *arg)
+{
+ struct smmu_softc *sc;
+
+ sc = arg;
+
+ device_printf(sc->dev, "%s\n", __func__);
+
+ return (FILTER_HANDLED);
+}
+
+static int
+smmu_gerr_intr(void *arg)
+{
+ struct smmu_softc *sc;
+
+ sc = arg;
+
+ device_printf(sc->dev, "SMMU Global Error\n");
+
+ return (FILTER_HANDLED);
+}
+
+static int
+smmu_enable_interrupts(struct smmu_softc *sc)
+{
+ uint32_t reg;
+ int error;
+
+ /* Disable MSI. */
+ bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);
+ bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);
+ bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);
+
+ bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);
+ bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);
+ bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);
+
+ if (sc->features & CR0_PRIQEN) {
+ bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);
+ bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);
+ bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);
+ }
+
+ /* Disable any interrupts. */
+ error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0);
+ if (error) {
+ device_printf(sc->dev, "Could not disable interrupts.\n");
+ return (ENXIO);
+ }
+
+ /* Enable interrupts. */
+ reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
+ if (sc->features & SMMU_FEATURE_PRI)
+ reg |= IRQ_CTRL_PRIQ_IRQEN;
+
+ error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable interrupts.\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+#if DEV_ACPI
+static void
+smmu_configure_intr(struct smmu_softc *sc, struct resource *res)
+{
+ struct intr_map_data_acpi *ad;
+ struct intr_map_data *data;
+
+ data = rman_get_virtual(res);
+ KASSERT(data != NULL, ("data is NULL"));
+
+ if (data->type == INTR_MAP_DATA_ACPI) {
+ ad = (struct intr_map_data_acpi *)data;
+ ad->trig = INTR_TRIGGER_EDGE;
+ ad->pol = INTR_POLARITY_HIGH;
+ }
+}
+#endif
+
+static int
+smmu_setup_interrupts(struct smmu_softc *sc)
+{
+ device_t dev;
+ int error;
+
+ dev = sc->dev;
+
+#if DEV_ACPI
+ /*
+ * Configure SMMU interrupts as EDGE triggered manually
+ * as ACPI tables carries no information for that.
+ */
+ smmu_configure_intr(sc, sc->res[1]);
+ smmu_configure_intr(sc, sc->res[2]);
+ smmu_configure_intr(sc, sc->res[3]);
+#endif
+
+ error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
+ smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);
+ if (error) {
+ device_printf(dev, "Couldn't setup Event interrupt handler\n");
+ return (ENXIO);
+ }
+
+ error = bus_setup_intr(dev, sc->res[3], INTR_TYPE_MISC,
+ smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);
+ if (error) {
+ device_printf(dev, "Couldn't setup Gerr interrupt handler\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+static int
+smmu_reset(struct smmu_softc *sc)
+{
+ struct smmu_cmdq_entry cmd;
+ struct smmu_strtab *strtab;
+ int error;
+ int reg;
+
+ reg = bus_read_4(sc->res[0], SMMU_CR0);
+
+ if (reg & CR0_SMMUEN)
+ device_printf(sc->dev,
+ "%s: Warning: SMMU is enabled\n", __func__);
+
+ error = smmu_disable(sc);
+ if (error)
+ device_printf(sc->dev,
+ "%s: Could not disable SMMU.\n", __func__);
+
+ if (smmu_enable_interrupts(sc) != 0) {
+ device_printf(sc->dev, "Could not enable interrupts.\n");
+ return (ENXIO);
+ }
+
+ reg = CR1_TABLE_SH_IS
+ | CR1_TABLE_OC_WBC
+ | CR1_TABLE_IC_WBC
+ | CR1_QUEUE_SH_IS
+ | CR1_QUEUE_OC_WBC
+ | CR1_QUEUE_IC_WBC;
+ bus_write_4(sc->res[0], SMMU_CR1, reg);
+
+ reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
+ bus_write_4(sc->res[0], SMMU_CR2, reg);
+
+ /* Stream table. */
+ strtab = &sc->strtab;
+ bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);
+ bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);
+
+ /* Command queue. */
+ bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);
+ bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);
+ bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);
+
+ reg = CR0_CMDQEN;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable command queue\n");
+ return (ENXIO);
+ }
+
+ /* Invalidate cached configuration. */
+ smmu_invalidate_all_sid(sc);
+
+ if (sc->features & SMMU_FEATURE_HYP) {
+ cmd.opcode = CMD_TLBI_EL2_ALL;
+ smmu_cmdq_enqueue_cmd(sc, &cmd);
+ };
+
+ /* Invalidate TLB. */
+ smmu_tlbi_all(sc);
+
+ /* Event queue */
+ bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);
+ bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);
+ bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);
+
+ reg |= CR0_EVENTQEN;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable event queue\n");
+ return (ENXIO);
+ }
+
+ if (sc->features & SMMU_FEATURE_PRI) {
+ /* PRI queue */
+ bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);
+ bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);
+ bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);
+
+ reg |= CR0_PRIQEN;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable PRI queue\n");
+ return (ENXIO);
+ }
+ }
+
+ if (sc->features & SMMU_FEATURE_ATS) {
+ reg |= CR0_ATSCHK;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable ATS check.\n");
+ return (ENXIO);
+ }
+ }
+
+ reg |= CR0_SMMUEN;
+ error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
+ if (error) {
+ device_printf(sc->dev, "Could not enable SMMU.\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+static int
+smmu_check_features(struct smmu_softc *sc)
+{
+ uint32_t reg;
+ uint32_t val;
+
+ sc->features = 0;
+
+ reg = bus_read_4(sc->res[0], SMMU_IDR0);
+
+ if (reg & IDR0_ST_LVL_2) {
+ device_printf(sc->dev, "2-level stream table supported.\n");
+ sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;
+ }
+
+ if (reg & IDR0_CD2L) {
+ device_printf(sc->dev, "2-level CD table supported.\n");
+ sc->features |= SMMU_FEATURE_2_LVL_CD;
+ }
+
+ switch (reg & IDR0_TTENDIAN_M) {
+ case IDR0_TTENDIAN_MIXED:
+ device_printf(sc->dev, "Mixed endianess supported.\n");
+ sc->features |= SMMU_FEATURE_TT_LE;
+ sc->features |= SMMU_FEATURE_TT_BE;
+ break;
+ case IDR0_TTENDIAN_LITTLE:
+ device_printf(sc->dev, "Little endian supported only.\n");
+ sc->features |= SMMU_FEATURE_TT_LE;
+ break;
+ case IDR0_TTENDIAN_BIG:
+ device_printf(sc->dev, "Big endian supported only.\n");
+ sc->features |= SMMU_FEATURE_TT_BE;
+ break;
+ default:
+ device_printf(sc->dev, "Unsupported endianness.\n");
+ return (ENXIO);
+ }
+
+ if (reg & IDR0_SEV)
+ sc->features |= SMMU_FEATURE_SEV;
+
+ if (reg & IDR0_MSI) {
+ device_printf(sc->dev, "MSI feature present.\n");
+ sc->features |= SMMU_FEATURE_MSI;
+ } else
+ device_printf(sc->dev, "MSI feature not present.\n");
+
+ if (reg & IDR0_HYP) {
+ device_printf(sc->dev, "HYP feature present.\n");
+ sc->features |= SMMU_FEATURE_HYP;
+ }
+
+ if (reg & IDR0_ATS)
+ sc->features |= SMMU_FEATURE_ATS;
+
+ if (reg & IDR0_PRI)
+ sc->features |= SMMU_FEATURE_PRI;
+
+ switch (reg & IDR0_STALL_MODEL_M) {
+ case IDR0_STALL_MODEL_FORCE:
+ /* Stall is forced. */
+ sc->features |= SMMU_FEATURE_STALL_FORCE;
+ /* FALLTHROUGH */
+ case IDR0_STALL_MODEL_STALL:
+ sc->features |= SMMU_FEATURE_STALL;
+ break;
+ }
+
+ /* Grab translation stages supported. */
+ if (reg & IDR0_S1P) {
+ device_printf(sc->dev, "Stage 1 translation supported.\n");
+ sc->features |= SMMU_FEATURE_S1P;
+ }
+ if (reg & IDR0_S2P) {
+ device_printf(sc->dev, "Stage 2 translation supported.\n");
+ sc->features |= SMMU_FEATURE_S2P;
+ }
+
+ switch (reg & IDR0_TTF_M) {
+ case IDR0_TTF_ALL:
+ case IDR0_TTF_AA64:
+ sc->ias = 40;
+ break;
+ default:
+ device_printf(sc->dev, "No AArch64 table format support.\n");
+ return (ENXIO);
+ }
+
+ if (reg & IDR0_ASID16)
+ sc->asid_bits = 16;
+ else
+ sc->asid_bits = 8;
+
+ device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);
+
+ if (reg & IDR0_VMID16)
+ sc->vmid_bits = 16;
+ else
+ sc->vmid_bits = 8;
+
+ reg = bus_read_4(sc->res[0], SMMU_IDR1);
+
+ if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
+ device_printf(sc->dev,
+ "Embedded implementations not supported by this driver.\n");
+ return (ENXIO);
+ }
+
+ val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S;
+ sc->cmdq.size_log2 = val;
+ device_printf(sc->dev, "CMD queue bits %d\n", val);
+
+ val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S;
+ sc->evtq.size_log2 = val;
+ device_printf(sc->dev, "EVENT queue bits %d\n", val);
+
+ if (sc->features & SMMU_FEATURE_PRI) {
+ val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S;
+ sc->priq.size_log2 = val;
+ device_printf(sc->dev, "PRI queue bits %d\n", val);
+ }
+
+ sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;
+ sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;
+
+ if (sc->sid_bits <= STRTAB_SPLIT)
+ sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;
+
+ device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);
+ device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);
+
+ /* IDR3 */
+ reg = bus_read_4(sc->res[0], SMMU_IDR3);
+ if (reg & IDR3_RIL)
+ sc->features |= SMMU_FEATURE_RANGE_INV;
+
+ /* IDR5 */
+ reg = bus_read_4(sc->res[0], SMMU_IDR5);
+
+ switch (reg & IDR5_OAS_M) {
+ case IDR5_OAS_32:
+ sc->oas = 32;
+ break;
+ case IDR5_OAS_36:
+ sc->oas = 36;
+ break;
+ case IDR5_OAS_40:
+ sc->oas = 40;
+ break;
+ case IDR5_OAS_42:
+ sc->oas = 42;
+ break;
+ case IDR5_OAS_44:
+ sc->oas = 44;
+ break;
+ case IDR5_OAS_48:
+ sc->oas = 48;
+ break;
+ case IDR5_OAS_52:
+ sc->oas = 52;
+ break;
+ }
+
+ sc->pgsizes = 0;
+ if (reg & IDR5_GRAN64K)
+ sc->pgsizes |= 64 * 1024;
+ if (reg & IDR5_GRAN16K)
+ sc->pgsizes |= 16 * 1024;
+ if (reg & IDR5_GRAN4K)
+ sc->pgsizes |= 4 * 1024;
+
+ if ((reg & IDR5_VAX_M) == IDR5_VAX_52)
+ sc->features |= SMMU_FEATURE_VAX;
+
+ return (0);
+}
+
+static void
+smmu_init_asids(struct smmu_softc *sc)
+{
+
+ sc->asid_set_size = (1 << sc->asid_bits);
+ sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);
+ mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);
+}
+
+static int
+smmu_asid_alloc(struct smmu_softc *sc, int *new_asid)
+{
+
+ mtx_lock_spin(&sc->asid_set_mutex);
+ bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);
+ if (*new_asid == -1) {
+ mtx_unlock_spin(&sc->asid_set_mutex);
+ return (ENOMEM);
+ }
+ bit_set(sc->asid_set, *new_asid);
+ mtx_unlock_spin(&sc->asid_set_mutex);
+
+ return (0);
+}
+
+static void
+smmu_asid_free(struct smmu_softc *sc, int asid)
+{
+
+ mtx_lock_spin(&sc->asid_set_mutex);
+ bit_clear(sc->asid_set, asid);
+ mtx_unlock_spin(&sc->asid_set_mutex);
+}
+
+/*
+ * Device interface.
+ */
+int
+smmu_attach(device_t dev)
+{
+ struct smmu_softc *sc;
+ int error;
+
+ sc = device_get_softc(dev);
+ sc->dev = dev;
+
+ mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);
+
+ error = bus_alloc_resources(dev, smmu_spec, sc->res);
+ if (error) {
+ device_printf(dev, "Couldn't allocate resources.\n");
+ return (ENXIO);
+ }
+
+ error = smmu_setup_interrupts(sc);
+ if (error) {
+ bus_release_resources(dev, smmu_spec, sc->res);
+ return (ENXIO);
+ }
+
+ error = smmu_check_features(sc);
+ if (error) {
+ device_printf(dev, "Some features are required "
+ "but not supported by hardware.\n");
+ return (ENXIO);
+ }
+
+ smmu_init_asids(sc);
+
+ error = smmu_init_queues(sc);
+ if (error) {
+ device_printf(dev, "Couldn't allocate queues.\n");
+ return (ENXIO);
+ }
+
+ error = smmu_init_strtab(sc);
+ if (error) {
+ device_printf(dev, "Couldn't allocate strtab.\n");
+ return (ENXIO);
+ }
+
+ error = smmu_reset(sc);
+ if (error) {
+ device_printf(dev, "Couldn't reset SMMU.\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+int
+smmu_detach(device_t dev)
+{
+ struct smmu_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ bus_release_resources(dev, smmu_spec, sc->res);
+
+ return (0);
+}
+
+static int
+smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+ struct smmu_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ device_printf(sc->dev, "%s\n", __func__);
+
+ return (ENOENT);
+}
+
+static int
+smmu_unmap(device_t dev, struct smmu_domain *domain,
+ vm_offset_t va, bus_size_t size)
+{
+ struct smmu_domain *smmu_domain;
+ struct smmu_softc *sc;
+ int err;
+ int i;
+
+ sc = device_get_softc(dev);
+ smmu_domain = (struct smmu_domain *)domain;
+
+ err = 0;
+
+ for (i = 0; i < size; i += PAGE_SIZE) {
+ if (pmap_sremove(&smmu_domain->p, va)) {
+ /* pmap entry removed, invalidate TLB. */
+ smmu_tlbi_va(sc, va, smmu_domain->asid);
+ } else {
+ err = ENOENT;
+ break;
+ }
+ va += PAGE_SIZE;
+ }
+
+ smmu_sync(sc);
+
+ return (err);
+}
+
+static int
+smmu_map(device_t dev, struct smmu_domain *domain,
+ vm_offset_t va, vm_paddr_t pa, vm_size_t size,
+ vm_prot_t prot)
+{
+ struct smmu_domain *smmu_domain;
+ struct smmu_softc *sc;
+ int error;
+
+ sc = device_get_softc(dev);
+ smmu_domain = (struct smmu_domain *)domain;
+
+ for (; size > 0; size -= PAGE_SIZE) {
+ error = pmap_senter(&smmu_domain->p, va, pa, prot, 0);
+ if (error)
+ return (error);
+ smmu_tlbi_va(sc, va, smmu_domain->asid);
+ pa += PAGE_SIZE;
+ va += PAGE_SIZE;
+ }
+
+ smmu_sync(sc);
+
+ return (0);
+}
+
+static struct smmu_domain *
+smmu_domain_alloc(device_t dev)
+{
+ struct smmu_domain *domain;
+ struct smmu_softc *sc;
+ int error;
+ int new_asid;
+
+ sc = device_get_softc(dev);
+
+ domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO);
+
+ error = smmu_asid_alloc(sc, &new_asid);
+ if (error) {
+ free(domain, M_SMMU);
+ device_printf(sc->dev,
+ "Could not allocate ASID for a new domain.\n");
+ return (NULL);
+ }
+
+ domain->asid = (uint16_t)new_asid;
+
+ pmap_pinit(&domain->p);
+ PMAP_LOCK_INIT(&domain->p);
+
+ error = smmu_init_cd(sc, domain);
+ if (error) {
+ free(domain, M_SMMU);
+ device_printf(sc->dev, "Could not initialize CD\n");
+ return (NULL);
+ }
+
+ smmu_tlbi_asid(sc, domain->asid);
+
+ return (domain);
+}
+
+static int
+smmu_domain_free(device_t dev, struct smmu_domain *domain)
+{
+ struct smmu_softc *sc;
+ struct smmu_cd *cd;
+ int error;
+
+ sc = device_get_softc(dev);
+
+ cd = domain->cd;
+
+ error = pmap_sremove_all(&domain->p);
+ if (error != 0)
+ return (error);
+
+ pmap_release(&domain->p);
+
+ smmu_tlbi_asid(sc, domain->asid);
+ smmu_asid_free(sc, domain->asid);
+
+ contigfree(cd->vaddr, cd->size, M_SMMU);
+ free(cd, M_SMMU);
+
+ free(domain, M_SMMU);
+
+ return (0);
+}
+
+static int
+smmu_device_attach(device_t dev, struct smmu_domain *domain,
+ struct smmu_ctx *ctx)
+{
+ struct smmu_softc *sc;
+ uint16_t rid;
+ u_int xref, sid;
+ int seg;
+ int err;
+
+ sc = device_get_softc(dev);
+
+ seg = pci_get_domain(ctx->dev);
+ rid = pci_get_rid(ctx->dev);
+ err = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
+ if (err)
+ return (ENOENT);
+
+ ctx->sid = sid;
+
+ if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
+ err = smmu_init_l1_entry(sc, sid);
+ if (err)
+ return (ENXIO);
+ }
+
+ /*
+ * Neoverse N1 SDP:
+ * 0x800 xhci
+ * 0x700 re
+ * 0x600 sata
+ */
+
+ smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);
+
+ return (0);
+}
+
+static int
+smmu_device_detach(device_t dev, struct smmu_ctx *ctx)
+{
+ struct smmu_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ smmu_deinit_l1_entry(sc, ctx->sid);
+
+ return (0);
+}
+
+static device_method_t smmu_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_detach, smmu_detach),
+
+ /* IOMMU interface */
+ DEVMETHOD(iommu_map, smmu_map),
+ DEVMETHOD(iommu_unmap, smmu_unmap),
+ DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc),
+ DEVMETHOD(iommu_domain_free, smmu_domain_free),
+ DEVMETHOD(iommu_ctx_attach, smmu_device_attach),
+ DEVMETHOD(iommu_ctx_detach, smmu_device_detach),
+
+ /* Bus interface */
+ DEVMETHOD(bus_read_ivar, smmu_read_ivar),
+
+ /* End */
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods,
+ sizeof(struct smmu_softc));
Index: sys/arm64/iommu/smmu_acpi.c
===================================================================
--- /dev/null
+++ sys/arm64/iommu/smmu_acpi.c
@@ -0,0 +1,252 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_acpi.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/bitstring.h>
+#include <sys/kernel.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/iommu/iommu.h>
+
+#include <machine/intr.h>
+#include <machine/resource.h>
+#include <arm64/iommu/iommu.h>
+
+#include "smmu_reg.h"
+#include "smmu_var.h"
+#include "pic_if.h"
+
+/* TODO: check this */
+#define MEMORY_RESOURCE_SIZE 0x40000
+#define MAX_SMMU 8
+
+struct smmu_acpi_devinfo {
+ struct resource_list di_rl;
+};
+
+struct iort_table_data {
+ device_t parent;
+ device_t dev;
+ ACPI_IORT_SMMU_V3 *smmu[MAX_SMMU];
+ int count;
+};
+
+static void
+iort_handler(ACPI_SUBTABLE_HEADER *entry, void *arg)
+{
+ struct iort_table_data *iort_data;
+ ACPI_IORT_NODE *node;
+ int i;
+
+ iort_data = (struct iort_table_data *)arg;
+ i = iort_data->count;
+
+ switch(entry->Type) {
+ case ACPI_IORT_NODE_SMMU_V3:
+ if (i == MAX_SMMU) {
+ printf("SMMUv3 found, but no space available.\n");
+ break;
+ }
+
+ if (iort_data->smmu[i] != NULL) {
+ if (bootverbose)
+ device_printf(iort_data->parent,
+ "smmu: Already have an SMMU table");
+ break;
+ }
+ node = (ACPI_IORT_NODE *)entry;
+ iort_data->smmu[i] = (ACPI_IORT_SMMU_V3 *)node->NodeData;
+ iort_data->count++;
+ break;
+ default:
+ break;
+ }
+}
+
+static void
+smmu_acpi_identify(driver_t *driver, device_t parent)
+{
+ struct iort_table_data iort_data;
+ ACPI_TABLE_IORT *iort;
+ vm_paddr_t iort_pa;
+ uintptr_t priv;
+ device_t dev;
+ int i;
+
+ iort_pa = acpi_find_table(ACPI_SIG_IORT);
+ if (iort_pa == 0)
+ return;
+
+ iort = acpi_map_table(iort_pa, ACPI_SIG_IORT);
+ if (iort == NULL) {
+ device_printf(parent, "smmu: Unable to map the IORT\n");
+ return;
+ }
+
+ iort_data.parent = parent;
+ for (i = 0; i < MAX_SMMU; i++)
+ iort_data.smmu[i] = NULL;
+ iort_data.count = 0;
+
+ acpi_walk_subtables(iort + 1, (char *)iort + iort->Header.Length,
+ iort_handler, &iort_data);
+ if (iort_data.count == 0) {
+ device_printf(parent, "No SMMU found.\n");
+ goto out;
+ }
+
+ for (i = 0; i < iort_data.count; i++) {
+ dev = BUS_ADD_CHILD(parent,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, "smmu", -1);
+ if (dev == NULL) {
+ device_printf(parent, "add smmu child failed\n");
+ goto out;
+ }
+
+ /* Add the IORT data */
+ BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 0,
+ iort_data.smmu[i]->EventGsiv, 1);
+ BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 1,
+ iort_data.smmu[i]->SyncGsiv, 1);
+ BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 2,
+ iort_data.smmu[i]->GerrGsiv, 1);
+ BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0,
+ iort_data.smmu[i]->BaseAddress, MEMORY_RESOURCE_SIZE);
+
+ priv = iort_data.smmu[i]->Flags;
+ priv <<= 32;
+ priv |= iort_data.smmu[i]->Model;
+
+ acpi_set_private(dev, (void *)priv);
+ }
+
+ iort_data.dev = dev;
+
+out:
+ acpi_unmap_table(iort);
+}
+
+static int
+smmu_acpi_probe(device_t dev)
+{
+
+ switch((uintptr_t)acpi_get_private(dev) & 0xffffffff) {
+ case ACPI_IORT_SMMU_V3_GENERIC:
+ /* Generic SMMUv3 */
+ break;
+ default:
+ return (ENXIO);
+ }
+
+ device_set_desc(dev, SMMU_DEVSTR);
+
+ return (BUS_PROBE_NOWILDCARD);
+}
+
+static int
+smmu_acpi_attach(device_t dev)
+{
+ struct smmu_softc *sc;
+ uintptr_t start;
+ uintptr_t priv;
+ int err;
+
+ sc = device_get_softc(dev);
+ sc->dev = dev;
+
+ priv = (uintptr_t)acpi_get_private(dev);
+ if ((priv >> 32) & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
+ sc->features |= SMMU_FEATURE_COHERENCY;
+
+ if (bootverbose)
+ device_printf(sc->dev, "%s: features %x\n",
+ __func__, sc->features);
+
+ err = smmu_attach(dev);
+ if (err != 0)
+ goto error;
+
+ /* Use memory start address as an xref. */
+ start = bus_get_resource_start(dev, SYS_RES_MEMORY, 0);
+ err = iommu_register(dev, &sc->unit, start);
+ if (err) {
+ device_printf(dev, "Failed to register SMMU.\n");
+ return (ENXIO);
+ }
+
+ return (0);
+
+error:
+ if (bootverbose) {
+ device_printf(dev,
+ "Failed to attach. Error %d\n", err);
+ }
+ /* Failure so free resources. */
+ smmu_detach(dev);
+
+ return (err);
+}
+
+static device_method_t smmu_acpi_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_identify, smmu_acpi_identify),
+ DEVMETHOD(device_probe, smmu_acpi_probe),
+ DEVMETHOD(device_attach, smmu_acpi_attach),
+
+ /* End */
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(smmu, smmu_acpi_driver, smmu_acpi_methods,
+ sizeof(struct smmu_softc), smmu_driver);
+
+static devclass_t smmu_acpi_devclass;
+
+EARLY_DRIVER_MODULE(smmu, acpi, smmu_acpi_driver, smmu_acpi_devclass,
+ 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
Index: sys/arm64/iommu/smmu_reg.h
===================================================================
--- /dev/null
+++ sys/arm64/iommu/smmu_reg.h
@@ -0,0 +1,478 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _DEV_IOMMU_SMMU_REG_H_
+#define _DEV_IOMMU_SMMU_REG_H_
+
+#define SMMU_IDR0 0x000
+#define IDR0_ST_LVL_S 27
+#define IDR0_ST_LVL_M (0x3 << IDR0_ST_LVL_S)
+#define IDR0_ST_LVL_LINEAR (0x0 << IDR0_ST_LVL_S) /* Linear Stream table*/
+#define IDR0_ST_LVL_2 (0x1 << IDR0_ST_LVL_S) /* 2-level Stream Table*/
+#define IDR0_ST_TERM_MODEL (1 << 26) /* Terminate model behavior */
+#define IDR0_STALL_MODEL_S 24 /* Stall model support */
+#define IDR0_STALL_MODEL_M (0x3 << IDR0_STALL_MODEL_S)
+#define IDR0_STALL_MODEL_STALL (0x0 << IDR0_STALL_MODEL_S) /* Stall and Term*/
+#define IDR0_STALL_MODEL_FORCE (0x2 << IDR0_STALL_MODEL_S) /* Stall is forced*/
+#define IDR0_TTENDIAN_S 21 /* Endianness for translation table walks.*/
+#define IDR0_TTENDIAN_M (0x3 << IDR0_TTENDIAN_S)
+#define IDR0_TTENDIAN_MIXED (0x0 << IDR0_TTENDIAN_S)
+#define IDR0_TTENDIAN_LITTLE (0x2 << IDR0_TTENDIAN_S)
+#define IDR0_TTENDIAN_BIG (0x3 << IDR0_TTENDIAN_S)
+#define IDR0_VATOS (1 << 20) / * Virtual ATOS page interface */
+#define IDR0_CD2L (1 << 19) /* 2-level Context descriptor table*/
+#define IDR0_VMID16 (1 << 18) /* 16-bit VMID supported */
+#define IDR0_VMW (1 << 17) /* VMID wildcard-matching */
+#define IDR0_PRI (1 << 16) /* Page Request Interface supported*/
+#define IDR0_ATOS (1 << 15) /* Address Translation Operations */
+#define IDR0_SEV (1 << 14) /* WFE wake-up events */
+#define IDR0_MSI (1 << 13) /* Message Signalled Interrupts */
+#define IDR0_ASID16 (1 << 12) /* 16-bit ASID supported */
+#define IDR0_NS1ATS (1 << 11) /* Split-stage ATS not supported */
+#define IDR0_ATS (1 << 10) /* PCIe ATS supported by SMMU */
+#define IDR0_HYP (1 << 9) /* Hypervisor stage 1 contexts */
+#define IDR0_DORMHINT (1 << 8) /* Dormant hint supported */
+#define IDR0_HTTU_S 6 /* H/W transl. table A-flag and Dirty state */
+#define IDR0_HTTU_M (0x3 << IDR0_HTTU_S)
+#define IDR0_HTTU_A (0x1 << IDR0_HTTU_S) /* Access flag (A-flag) */
+#define IDR0_HTTU_AD (0x2 << IDR0_HTTU_S) /* A-flag and Dirty State*/
+#define IDR0_BTM (1 << 5) /* Broadcast TLB Maintenance */
+#define IDR0_COHACC (1 << 4) /* Coherent access to translations*/
+#define IDR0_TTF_S 2 /* Translation Table Formats supported */
+#define IDR0_TTF_M (0x3 << IDR0_TTF_S)
+#define IDR0_TTF_AA32 (0x1 << IDR0_TTF_S) /* AArch32 (LPAE) */
+#define IDR0_TTF_AA64 (0x2 << IDR0_TTF_S) /* AArch64 */
+#define IDR0_TTF_ALL (0x3 << IDR0_TTF_S) /* AArch32 and AArch64 */
+#define IDR0_S1P (1 << 1) /* Stage1 translation supported. */
+#define IDR0_S2P (1 << 0) /* Stage2 translation supported. */
+#define SMMU_IDR1 0x004
+#define IDR1_TABLES_PRESET (1 << 30) /* Table base addresses fixed. */
+#define IDR1_QUEUES_PRESET (1 << 29) /* Queue base addresses fixed. */
+#define IDR1_REL (1 << 28) /* Relative base pointers */
+#define IDR1_ATTR_TYPES_OVR (1 << 27) /* Incoming attrs can be overridden*/
+#define IDR1_ATTR_PERMS_OVR (1 << 26) /* Incoming attrs can be overridden*/
+#define IDR1_CMDQS_S 21 /* Maximum number of Command queue entries*/
+#define IDR1_CMDQS_M (0x1f << IDR1_CMDQS_S)
+#define IDR1_EVENTQS_S 16 /* Maximum number of Event queue entries */
+#define IDR1_EVENTQS_M (0x1f << IDR1_EVENTQS_S)
+#define IDR1_PRIQS_S 11 /* Maximum number of PRI queue entries */
+#define IDR1_PRIQS_M (0x1f << IDR1_PRIQS_S)
+#define IDR1_SSIDSIZE_S 6 /* Max bits of SubstreamID */
+#define IDR1_SSIDSIZE_M (0x1f << IDR1_SSIDSIZE_S)
+#define IDR1_SIDSIZE_S 0 /* Max bits of StreamID */
+#define IDR1_SIDSIZE_M (0x3f << IDR1_SIDSIZE_S)
+#define SMMU_IDR2 0x008
+#define SMMU_IDR3 0x00C
+#define IDR3_RIL (1 << 10) /* Range-based Invalidations. */
+#define SMMU_IDR4 0x010
+#define SMMU_IDR5 0x014
+#define IDR5_STALL_MAX_S 16 /* Max outstanding stalled transactions */
+#define IDR5_STALL_MAX_M (0xffff << IDR5_STALL_MAX_S)
+#define IDR5_VAX_S 10 /* Virtual Address eXtend */
+#define IDR5_VAX_M (0x3 << IDR5_VAX_S)
+#define IDR5_VAX_48 (0 << IDR5_VAX_S)
+#define IDR5_VAX_52 (1 << IDR5_VAX_S)
+#define IDR5_GRAN64K (1 << 6) /* 64KB translation granule */
+#define IDR5_GRAN16K (1 << 5) /* 16KB translation granule */
+#define IDR5_GRAN4K (1 << 4) /* 4KB translation granule */
+#define IDR5_OAS_S 0 /* Output Address Size */
+#define IDR5_OAS_M (0x7 << IDR5_OAS_S)
+#define IDR5_OAS_32 (0x0 << IDR5_OAS_S)
+#define IDR5_OAS_36 (0x1 << IDR5_OAS_S)
+#define IDR5_OAS_40 (0x2 << IDR5_OAS_S)
+#define IDR5_OAS_42 (0x3 << IDR5_OAS_S)
+#define IDR5_OAS_44 (0x4 << IDR5_OAS_S)
+#define IDR5_OAS_48 (0x5 << IDR5_OAS_S)
+#define IDR5_OAS_52 (0x6 << IDR5_OAS_S) /* Reserved in SMMU v3.0 */
+#define SMMU_IIDR 0x018
+#define SMMU_AIDR 0x01C
+#define SMMU_CR0 0x020
+#define CR0_VMW_S 6 /* VMID Wildcard */
+#define CR0_VMW_M (0x7 << CR0_VMW_S)
+#define CR0_ATSCHK (1 << 4) /* ATS behavior: Safe mode */
+#define CR0_CMDQEN (1 << 3) /* Enable Command queue processing */
+#define CR0_EVENTQEN (1 << 2) /* Enable Event queue writes */
+#define CR0_PRIQEN (1 << 1) /* Enable PRI queue writes */
+#define CR0_SMMUEN (1 << 0) /* Non-secure SMMU enable */
+#define SMMU_CR0ACK 0x024
+#define SMMU_CR1 0x028
+#define CR1_TABLE_SH_S 10 /* Table access Shareability. */
+#define CR1_TABLE_SH_M (0x3 << CR1_TABLE_SH_S)
+#define CR1_TABLE_SH_NS (0x0 << CR1_TABLE_SH_S)
+#define CR1_TABLE_SH_OS (0x2 << CR1_TABLE_SH_S)
+#define CR1_TABLE_SH_IS (0x3 << CR1_TABLE_SH_S)
+#define CR1_TABLE_OC_S 8 /* Table access Outer Cacheability. */
+#define CR1_TABLE_OC_M (0x3 << CR1_TABLE_OC_S)
+#define CR1_TABLE_OC_NC (0x0 << CR1_TABLE_OC_S)
+#define CR1_TABLE_OC_WBC (0x1 << CR1_TABLE_OC_S)
+#define CR1_TABLE_OC_WTC (0x2 << CR1_TABLE_OC_S)
+#define CR1_TABLE_IC_S 6 /* Table access Inner Cacheability. */
+#define CR1_TABLE_IC_M (0x3 << CR1_TABLE_IC_S)
+#define CR1_TABLE_IC_NC (0x0 << CR1_TABLE_IC_S)
+#define CR1_TABLE_IC_WBC (0x1 << CR1_TABLE_IC_S)
+#define CR1_TABLE_IC_WTC (0x2 << CR1_TABLE_IC_S)
+#define CR1_QUEUE_SH_S 4 /* Queue access Shareability. */
+#define CR1_QUEUE_SH_M (0x3 << CR1_QUEUE_SH_S)
+#define CR1_QUEUE_SH_NS (0x0 << CR1_QUEUE_SH_S)
+#define CR1_QUEUE_SH_OS (0x2 << CR1_QUEUE_SH_S)
+#define CR1_QUEUE_SH_IS (0x3 << CR1_QUEUE_SH_S)
+#define CR1_QUEUE_OC_S 2 /* Queue access Outer Cacheability. */
+#define CR1_QUEUE_OC_M (0x3 << CR1_QUEUE_OC_S)
+#define CR1_QUEUE_OC_NC (0x0 << CR1_QUEUE_OC_S)
+#define CR1_QUEUE_OC_WBC (0x1 << CR1_QUEUE_OC_S)
+#define CR1_QUEUE_OC_WTC (0x2 << CR1_QUEUE_OC_S)
+#define CR1_QUEUE_IC_S 0 /* Queue access Inner Cacheability. */
+#define CR1_QUEUE_IC_M (0x3 << CR1_QUEUE_IC_S)
+#define CR1_QUEUE_IC_NC (0x0 << CR1_QUEUE_IC_S)
+#define CR1_QUEUE_IC_WBC (0x1 << CR1_QUEUE_IC_S)
+#define CR1_QUEUE_IC_WTC (0x2 << CR1_QUEUE_IC_S)
+#define SMMU_CR2 0x02C
+#define CR2_PTM (1 << 2) /* Private TLB Maintenance. */
+#define CR2_RECINVSID (1 << 1) /* Record invalid SID. */
+#define CR2_E2H (1 << 0) /* Enable EL2-E2H translation regime */
+#define SMMU_STATUSR 0x040
+#define SMMU_GBPA 0x044
+#define SMMU_AGBPA 0x048
+#define SMMU_IRQ_CTRL 0x050
+#define IRQ_CTRL_EVENTQ_IRQEN (1 << 2) /* NS Event queue interrupts enabled.*/
+#define IRQ_CTRL_PRIQ_IRQEN (1 << 1) /* PRI queue interrupts are enabled.*/
+#define IRQ_CTRL_GERROR_IRQEN (1 << 0) /* Global errors int are enabled. */
+#define SMMU_IRQ_CTRLACK 0x054
+#define SMMU_GERROR 0x060
+#define SMMU_GERRORN 0x064
+#define SMMU_GERROR_IRQ_CFG0 0x068
+#define SMMU_GERROR_IRQ_CFG1 0x070
+#define SMMU_GERROR_IRQ_CFG2 0x074
+#define SMMU_STRTAB_BASE 0x080
+#define STRTAB_BASE_RA (1UL << 62) /* Read-Allocate. */
+#define STRTAB_BASE_ADDR_S 6 /* Physical address of Stream table base */
+#define STRTAB_BASE_ADDR_M (0x3fffffffffffUL << STRTAB_BASE_ADDR_S)
+#define SMMU_STRTAB_BASE_CFG 0x088
+#define STRTAB_BASE_CFG_FMT_S 16 /* Format of Stream table. */
+#define STRTAB_BASE_CFG_FMT_M (0x3 << STRTAB_BASE_CFG_FMT_S)
+#define STRTAB_BASE_CFG_FMT_LINEAR (0x0 << STRTAB_BASE_CFG_FMT_S)
+#define STRTAB_BASE_CFG_FMT_2LVL (0x1 << STRTAB_BASE_CFG_FMT_S)
+#define STRTAB_BASE_CFG_SPLIT_S 6 /* SID split point for 2lvl table. */
+#define STRTAB_BASE_CFG_SPLIT_M (0x1f << STRTAB_BASE_CFG_SPLIT_S)
+#define STRTAB_BASE_CFG_SPLIT_4KB (6 << STRTAB_BASE_CFG_SPLIT_S)
+#define STRTAB_BASE_CFG_SPLIT_16KB (8 << STRTAB_BASE_CFG_SPLIT_S)
+#define STRTAB_BASE_CFG_SPLIT_64KB (10 << STRTAB_BASE_CFG_SPLIT_S)
+#define STRTAB_BASE_CFG_LOG2SIZE_S 0 /* Table size as log2(entries) */
+#define STRTAB_BASE_CFG_LOG2SIZE_M (0x3f << STRTAB_BASE_CFG_LOG2SIZE_S)
+#define SMMU_CMDQ_BASE 0x090
+#define CMDQ_BASE_RA (1UL << 62) /* Read-Allocate. */
+#define Q_BASE_ADDR_S 5 /* PA of queue base */
+#define Q_BASE_ADDR_M (0x7fffffffffff << Q_BASE_ADDR_S)
+#define Q_LOG2SIZE_S 0 /* Queue size as log2(entries) */
+#define Q_LOG2SIZE_M (0x1f << Q_LOG2SIZE_S)
+#define SMMU_CMDQ_PROD 0x098
+#define SMMU_CMDQ_CONS 0x09C
+#define CMDQ_CONS_ERR_S 24
+#define CMDQ_CONS_ERR_M (0x7f << CMDQ_CONS_ERR_S)
+#define SMMU_EVENTQ_BASE 0x0A0
+#define EVENTQ_BASE_WA (1UL << 62) /* Write-Allocate. */
+#define SMMU_EVENTQ_PROD 0x100A8
+#define SMMU_EVENTQ_CONS 0x100AC
+#define SMMU_EVENTQ_IRQ_CFG0 0x0B0
+#define SMMU_EVENTQ_IRQ_CFG1 0x0B8
+#define SMMU_EVENTQ_IRQ_CFG2 0x0BC
+#define SMMU_PRIQ_BASE 0x0C0
+#define PRIQ_BASE_WA (1UL < 62) /* Write-Allocate. */
+#define SMMU_PRIQ_PROD 0x100C8
+#define SMMU_PRIQ_CONS 0x100CC
+#define SMMU_PRIQ_IRQ_CFG0 0x0D0
+#define SMMU_PRIQ_IRQ_CFG1 0x0D8
+#define SMMU_PRIQ_IRQ_CFG2 0x0DC
+#define SMMU_GATOS_CTRL 0x100
+#define SMMU_GATOS_SID 0x108
+#define SMMU_GATOS_ADDR 0x110
+#define SMMU_GATOS_PAR 0x118
+#define SMMU_VATOS_SEL 0x180
+#define SMMU_S_IDR0 0x8000
+#define SMMU_S_IDR1 0x8004
+#define SMMU_S_IDR2 0x8008
+#define SMMU_S_IDR3 0x800C
+#define SMMU_S_IDR4 0x8010
+#define SMMU_S_CR0 0x8020
+#define SMMU_S_CR0ACK 0x8024
+#define SMMU_S_CR1 0x8028
+#define SMMU_S_CR2 0x802C
+#define SMMU_S_INIT 0x803C
+#define SMMU_S_GBPA 0x8044
+#define SMMU_S_AGBPA 0x8048
+#define SMMU_S_IRQ_CTRL 0x8050
+#define SMMU_S_IRQ_CTRLACK 0x8054
+#define SMMU_S_GERROR 0x8060
+#define SMMU_S_GERRORN 0x8064
+#define SMMU_S_GERROR_IRQ_CFG0 0x8068
+#define SMMU_S_GERROR_IRQ_CFG1 0x8070
+#define SMMU_S_GERROR_IRQ_CFG2 0x8074
+#define SMMU_S_STRTAB_BASE 0x8080
+#define SMMU_S_STRTAB_BASE_CFG 0x8088
+#define SMMU_S_CMDQ_BASE 0x8090
+#define SMMU_S_CMDQ_PROD 0x8098
+#define SMMU_S_CMDQ_CONS 0x809C
+#define SMMU_S_EVENTQ_BASE 0x80A0
+#define SMMU_S_EVENTQ_PROD 0x80A8
+#define SMMU_S_EVENTQ_CONS 0x80AC
+#define SMMU_S_EVENTQ_IRQ_CFG0 0x80B0
+#define SMMU_S_EVENTQ_IRQ_CFG1 0x80B8
+#define SMMU_S_EVENTQ_IRQ_CFG2 0x80BC
+#define SMMU_S_GATOS_CTRL 0x8100
+#define SMMU_S_GATOS_SID 0x8108
+#define SMMU_S_GATOS_ADDR 0x8110
+#define SMMU_S_GATOS_PAR 0x8118
+
+#define CMD_QUEUE_OPCODE_S 0
+#define CMD_QUEUE_OPCODE_M (0xff << CMD_QUEUE_OPCODE_S)
+
+#define CMD_PREFETCH_CONFIG 0x01
+#define PREFETCH_0_SID_S 32
+#define CMD_PREFETCH_ADDR 0x02
+#define CMD_CFGI_STE 0x03
+#define CFGI_0_STE_SID_S 32
+#define CMD_CFGI_STE_RANGE 0x04
+#define CFGI_1_STE_RANGE_S 0
+#define CMD_CFGI_CD 0x05
+#define CFGI_0_SSID_S 12
+#define CFGI_1_LEAF_S 0
+#define CMD_CFGI_CD_ALL 0x06
+#define CMD_TLBI_NH_ALL 0x10
+#define CMD_TLBI_NH_ASID 0x11
+#define CMD_TLBI_NH_VA 0x12
+#define TLBI_0_ASID_S 48
+#define TLBI_1_LEAF (1 << 0)
+#define TLBI_1_ADDR_S 12
+#define TLBI_1_ADDR_M (0xfffffffffffff << TLBI_1_ADDR_S)
+#define CMD_TLBI_NH_VAA 0x13
+#define CMD_TLBI_EL3_ALL 0x18
+#define CMD_TLBI_EL3_VA 0x1A
+#define CMD_TLBI_EL2_ALL 0x20
+#define CMD_TLBI_EL2_ASID 0x21
+#define CMD_TLBI_EL2_VA 0x22
+#define CMD_TLBI_EL2_VAA 0x23
+#define CMD_TLBI_S12_VMALL 0x28
+#define CMD_TLBI_S2_IPA 0x2A
+#define CMD_TLBI_NSNH_ALL 0x30
+#define CMD_ATC_INV 0x40
+#define CMD_PRI_RESP 0x41
+#define CMD_RESUME 0x44
+#define CMD_STALL_TERM 0x45
+#define CMD_SYNC 0x46
+#define SYNC_0_CS_S 12 /* The ComplSignal */
+#define SYNC_0_CS_M (0x3 << SYNC_0_CS_S)
+#define SYNC_0_CS_SIG_NONE (0x0 << SYNC_0_CS_S)
+#define SYNC_0_CS_SIG_IRQ (0x1 << SYNC_0_CS_S)
+#define SYNC_0_CS_SIG_SEV (0x2 << SYNC_0_CS_S)
+#define SYNC_0_MSH_S 22 /* Shareability attribute for MSI write */
+#define SYNC_0_MSH_M (0x3 << SYNC_0_MSH_S)
+#define SYNC_0_MSH_NS (0x0 << SYNC_0_MSH_S) /* Non-shareable */
+#define SYNC_0_MSH_OS (0x2 << SYNC_0_MSH_S) /* Outer Shareable */
+#define SYNC_0_MSH_IS (0x3 << SYNC_0_MSH_S) /* Inner Shareable */
+#define SYNC_0_MSIATTR_S 24 /* Write attribute for MSI */
+#define SYNC_0_MSIATTR_M (0xf << SYNC_0_MSIATTR_S)
+#define SYNC_0_MSIATTR_OIWB (0xf << SYNC_0_MSIATTR_S)
+#define SYNC_0_MSIDATA_S 32
+#define SYNC_1_MSIADDRESS_S 2
+#define SYNC_1_MSIADDRESS_M (0x3ffffffffffff << SYNC_1_MSIADDRESS_S)
+#define STE0_VALID (1 << 0) /* Structure contents are valid. */
+#define STE0_CONFIG_S 1
+#define STE0_CONFIG_M (0x7 << STE0_CONFIG_S)
+#define STE0_CONFIG_ABORT (0x0 << STE0_CONFIG_S)
+#define STE0_CONFIG_BYPASS (0x4 << STE0_CONFIG_S)
+#define STE0_CONFIG_S1_TRANS (0x5 << STE0_CONFIG_S)
+#define STE0_CONFIG_S2_TRANS (0x6 << STE0_CONFIG_S)
+#define STE0_CONFIG_ALL_TRANS (0x7 << STE0_CONFIG_S)
+#define STE0_S1FMT_S 4
+#define STE0_S1FMT_M (0x3 << STE0_S1FMT_S)
+#define STE0_S1FMT_LINEAR (0x0 << STE0_S1FMT_S)
+#define STE0_S1FMT_4KB_L2 (0x1 << STE0_S1FMT_S)
+#define STE0_S1FMT_64KB_L2 (0x2 << STE0_S1FMT_S)
+#define STE0_S1CONTEXTPTR_S 6
+#define STE0_S1CONTEXTPTR_M (0x3fffffffffff << STE0_S1CONTEXTPTR_S)
+#define STE0_S1CDMAX_S 59
+#define STE0_S1CDMAX_M (0x1f << STE0_S1CDMAX_S)
+
+#define STE1_S1DSS_S 0
+#define STE1_S1DSS_M (0x3 << STE1_S1DSS_S)
+#define STE1_S1DSS_TERMINATE (0x0 << STE1_S1DSS_S)
+#define STE1_S1DSS_BYPASS (0x1 << STE1_S1DSS_S)
+#define STE1_S1DSS_SUBSTREAM0 (0x2 << STE1_S1DSS_S)
+#define STE1_S1CIR_S 2
+#define STE1_S1CIR_M (0x3 << STE1_S1CIR_S)
+#define STE1_S1CIR_NC (0x0 << STE1_S1CIR_S)
+#define STE1_S1CIR_WBRA (0x1 << STE1_S1CIR_S)
+#define STE1_S1CIR_WT (0x2 << STE1_S1CIR_S)
+#define STE1_S1CIR_WB (0x3 << STE1_S1CIR_S)
+#define STE1_S1COR_S 4
+#define STE1_S1COR_M (0x3 << STE1_S1COR_S)
+#define STE1_S1COR_NC (0x0 << STE1_S1COR_S)
+#define STE1_S1COR_WBRA (0x1 << STE1_S1COR_S)
+#define STE1_S1COR_WT (0x2 << STE1_S1COR_S)
+#define STE1_S1COR_WB (0x3 << STE1_S1COR_S)
+#define STE1_S1CSH_S 6
+#define STE1_S1CSH_NS (0x0 << STE1_S1CSH_S)
+#define STE1_S1CSH_OS (0x2 << STE1_S1CSH_S)
+#define STE1_S1CSH_IS (0x3 << STE1_S1CSH_S)
+#define STE1_S2HWU59 (1 << 8)
+#define STE1_S2HWU60 (1 << 9)
+#define STE1_S2HWU61 (1 << 10)
+#define STE1_S2HWU62 (1 << 11)
+#define STE1_DRE (1 << 12) /* Destructive Read Enable. */
+#define STE1_CONT_S 13 /* Contiguous Hint */
+#define STE1_CONT_M (0xf << STE1_CONT_S)
+#define STE1_DCP (1 << 17) /* Directed Cache Prefetch. */
+#define STE1_PPAR (1 << 18) /* PRI Page request Auto Responses */
+#define STE1_MEV (1 << 19) /* Merge Events */
+#define STE1_S1STALLD (1 << 27) /* Stage 1 Stall Disable */
+#define STE1_EATS_S 28 /* Enable PCIe ATS translation and traffic */
+#define STE1_EATS_M (0x3 << STE1_EATS_S)
+#define STE1_EATS_ABORT (0x0 << STE1_EATS_S)
+#define STE1_EATS_FULLATS (0x1 << STE1_EATS_S) /* Full ATS */
+#define STE1_EATS_S1 (0x2 << STE1_EATS_S) /* Split-stage ATS */
+#define STE1_STRW_S 30 /* StreamWorld control */
+#define STE1_STRW_M (0x3 << STE1_STRW_S)
+#define STE1_STRW_NS_EL1 (0x0 << STE1_STRW_S)
+#define STE1_STRW_NS_EL2 (0x2 << STE1_STRW_S)
+#define STE1_MEMATTR_S 32
+#define STE1_MTCFG (1 << 36)
+#define STE1_ALLOCCFG_S 37
+#define STE1_SHCFG_S 44
+#define STE1_SHCFG_M (0x3UL << STE1_SHCFG_S)
+#define STE1_SHCFG_NS (0x0UL << STE1_SHCFG_S)
+#define STE1_SHCFG_INCOMING (0x1UL << STE1_SHCFG_S)
+#define STE1_SHCFG_OS (0x2UL << STE1_SHCFG_S)
+#define STE1_SHCFG_IS (0x3UL << STE1_SHCFG_S)
+#define STE1_NSCFG_S 46
+#define STE1_NSCFG_M (0x3UL << STE1_NSCFG_S)
+#define STE1_NSCFG_SECURE (0x2UL << STE1_NSCFG_S)
+#define STE1_NSCFG_NONSECURE (0x3UL << STE1_NSCFG_S)
+#define STE1_PRIVCFG_S 48
+#define STE1_INSTCFG_S 50
+
+#define STE2_S2VMID_S 0
+#define STE2_S2VMID_M (0xffff << STE2_S2VMID_S)
+#define STE2_S2T0SZ_S 32 /* Size of IPA input region */
+#define STE2_S2T0SZ_M (0x3f << STE2_S2T0SZ_S)
+#define STE2_S2SL0_S 38 /* Starting level of stage 2 tt walk */
+#define STE2_S2SL0_M (0x3 << STE2_S2SL0_S)
+#define STE2_S2IR0_S 40
+#define STE2_S2IR0_M (0x3 << STE2_S2IR0_S)
+#define STE2_S2OR0_S 42
+#define STE2_S2OR0_M (0x3 << STE2_S2OR0_S)
+#define STE2_S2SH0_S 44
+#define STE2_S2SH0_M (0x3 << STE2_S2SH0_S)
+#define STE2_S2TG_S 46
+#define STE2_S2TG_M (0x3 << STE2_S2TG_S)
+#define STE2_S2PS_S 48 /* Physical address Size */
+#define STE2_S2PS_M (0x7 << STE2_S2PS_S)
+#define STE2_S2AA64 (1 << 51) /* Stage 2 tt is AArch64 */
+#define STE2_S2ENDI (1 << 52) /* Stage 2 tt endianness */
+#define STE2_S2AFFD (1 << 53) /* Stage 2 Access Flag Fault Disable*/
+#define STE2_S2PTW (1 << 54) /* Protected Table Walk */
+#define STE2_S2S (1 << 57)
+#define STE2_S2R (1 << 58)
+
+#define STE3_S2TTB_S 4 /* Address of Translation Table base */
+#define STE3_S2TTB_M (0xffffffffffff << STE3_S2TTB_S)
+
+#define CD0_T0SZ_S 0 /* VA region size covered by TT0. */
+#define CD0_T0SZ_M (0x3f << CD0_T0SZ_S)
+#define CD0_TG0_S 6 /* TT0 Translation Granule size */
+#define CD0_TG0_M (0x3 << CD0_TG0_S)
+#define CD0_TG0_4KB (0x0 << CD0_TG0_S)
+#define CD0_TG0_64KB (0x1 << CD0_TG0_S)
+#define CD0_TG0_16KB (0x2 << CD0_TG0_S)
+#define CD0_IR0_S 8 /* Inner region Cacheability for TT0 access*/
+#define CD0_IR0_M (0x3 << CD0_IR0_S)
+#define CD0_IR0_NC (0x0 << CD0_IR0_S)
+#define CD0_IR0_WBC_RWA (0x1 << CD0_IR0_S)
+#define CD0_IR0_WTC_RA (0x2 << CD0_IR0_S)
+#define CD0_IR0_WBC_RA (0x3 << CD0_IR0_S)
+#define CD0_OR0_S 10 /* Outer region Cacheability for TT0 access*/
+#define CD0_OR0_M (0x3 << CD0_OR0_S)
+#define CD0_OR0_NC (0x0 << CD0_OR0_S)
+#define CD0_OR0_WBC_RWA (0x1 << CD0_OR0_S)
+#define CD0_OR0_WTC_RA (0x2 << CD0_OR0_S)
+#define CD0_OR0_WBC_RA (0x3 << CD0_OR0_S)
+#define CD0_SH0_S 12 /* Shareability for TT0 access */
+#define CD0_SH0_M (0x3 << CD0_SH0_S)
+#define CD0_SH0_NS (0x0 << CD0_SH0_S)
+#define CD0_SH0_OS (0x2 << CD0_SH0_S) /* Outer Shareable */
+#define CD0_SH0_IS (0x3 << CD0_SH0_S) /* Inner Shareable */
+#define CD0_EPD0 (1 << 14) /* TT0 walk disable */
+#define CD0_ENDI (1 << 15) /* Big Endian */
+#define CD0_T1SZ_S 16 /* VA region size covered by TT1 */
+#define CD0_T1SZ_M (0x3f << CD0_T1SZ_S)
+#define CD0_TG1_S 22 /* TT1 Translation Granule size */
+#define CD0_TG1_M (0x3 << CD0_TG1_S)
+#define CD0_TG1_4KB (0x2 << CD0_TG1_S)
+#define CD0_TG1_64KB (0x3 << CD0_TG1_S)
+#define CD0_TG1_16KB (0x1 << CD0_TG1_S)
+#define CD0_IR1_S 24 /* Inner region Cacheability for TT1 access*/
+#define CD0_IR1_M (0x3 << CD0_IR1_S)
+#define CD0_OR1_S 26
+#define CD0_OR1_M (0x3 << CD0_OR1_S)
+#define CD0_SH1_S 28
+#define CD0_SH1_M (0x3 << CD0_SH1_S)
+#define CD0_EPD1 (1UL << 30) /* TT1 tt walk disable*/
+#define CD0_VALID (1UL << 31) /* CD Valid. */
+#define CD0_IPS_S 32 /* Intermediate Physical Size */
+#define CD0_IPS_M (0x7UL << CD0_IPS_S)
+#define CD0_IPS_32BITS (0x0UL << CD0_IPS_S)
+#define CD0_IPS_36BITS (0x1UL << CD0_IPS_S)
+#define CD0_IPS_40BITS (0x2UL << CD0_IPS_S)
+#define CD0_IPS_42BITS (0x3UL << CD0_IPS_S)
+#define CD0_IPS_44BITS (0x4UL << CD0_IPS_S)
+#define CD0_IPS_48BITS (0x5UL << CD0_IPS_S)
+#define CD0_IPS_52BITS (0x6UL << CD0_IPS_S) /* SMMUv3.1 only */
+#define CD0_AFFD (1UL << 35) /* Access Flag Fault Disable */
+#define CD0_WXN (1UL << 36) /* Write eXecute Never */
+#define CD0_UWXN (1UL << 37) /* Unprivileged Write eXecut Never*/
+#define CD0_TBI0 (1UL << 38) /* Top Byte Ignore for TTB0 */
+#define CD0_TBI1 (1UL << 39) /* Top Byte Ignore for TTB1 */
+#define CD0_PAN (1UL << 40) /* Privileged Access Never */
+#define CD0_AA64 (1UL << 41) /* TTB{0,1} is AArch64-format TT */
+#define CD0_HD (1UL << 42)
+#define CD0_HA (1UL << 43)
+#define CD0_S (1UL << 44)
+#define CD0_R (1UL << 45)
+#define CD0_A (1UL << 46)
+#define CD0_ASET (1UL << 47) /* ASID Set. */
+#define CD0_ASID_S 48 /* Address Space Identifier */
+#define CD0_ASID_M (0xffff << CD0_ASID_S)
+#define CD1_TTB0_S 4 /* Address of TT0 base. */
+#define CD1_TTB0_M (0xffffffffffff << CD1_TTB0_S)
+
+#endif /* _DEV_IOMMU_SMMU_REG_H_ */
Index: sys/arm64/iommu/smmu_var.h
===================================================================
--- /dev/null
+++ sys/arm64/iommu/smmu_var.h
@@ -0,0 +1,155 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _DEV_IOMMU_SMMU_VAR_H_
+#define _DEV_IOMMU_SMMU_VAR_H_
+
+#define SMMU_DEVSTR "ARM System Memory Management Unit"
+#define SMMU_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
+#define SMMU_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
+
+DECLARE_CLASS(smmu_driver);
+
+struct smmu_queue_local_copy {
+ union {
+ uint64_t val;
+ struct {
+ uint32_t prod;
+ uint32_t cons;
+ };
+ };
+};
+
+struct smmu_cd {
+ vm_paddr_t paddr;
+ vm_size_t size;
+ void *vaddr;
+};
+
+struct smmu_queue {
+ struct smmu_queue_local_copy lc;
+ vm_paddr_t paddr;
+ void *vaddr;
+ uint32_t prod_off;
+ uint32_t cons_off;
+ int size_log2;
+ uint64_t base;
+};
+
+struct smmu_cmdq_entry {
+ uint8_t opcode;
+ union {
+ struct {
+ uint16_t asid;
+ uint16_t vmid;
+ vm_offset_t addr;
+ bool leaf;
+ } tlbi;
+ struct {
+ uint32_t sid;
+ uint32_t ssid;
+ bool leaf;
+ } cfgi;
+ struct {
+ uint32_t sid;
+ } prefetch;
+ struct {
+ uint64_t msiaddr;
+ } sync;
+ };
+};
+
+struct l1_desc {
+ uint8_t span;
+ size_t size;
+ void *va;
+ vm_paddr_t pa;
+};
+
+struct smmu_strtab {
+ void *vaddr;
+ uint64_t base;
+ uint32_t base_cfg;
+ uint32_t num_l1_entries;
+ struct l1_desc *l1;
+};
+
+struct smmu_softc {
+ device_t dev;
+ struct resource *res[4];
+ void *intr_cookie[3];
+ uint32_t ias; /* Intermediate Physical Address */
+ uint32_t oas; /* Physical Address */
+ uint32_t asid_bits;
+ uint32_t vmid_bits;
+ uint32_t sid_bits;
+ uint32_t ssid_bits;
+ uint32_t pgsizes;
+ uint32_t features;
+#define SMMU_FEATURE_2_LVL_STREAM_TABLE (1 << 0)
+#define SMMU_FEATURE_2_LVL_CD (1 << 1)
+#define SMMU_FEATURE_TT_LE (1 << 2)
+#define SMMU_FEATURE_TT_BE (1 << 3)
+#define SMMU_FEATURE_SEV (1 << 4)
+#define SMMU_FEATURE_MSI (1 << 5)
+#define SMMU_FEATURE_HYP (1 << 6)
+#define SMMU_FEATURE_ATS (1 << 7)
+#define SMMU_FEATURE_PRI (1 << 8)
+#define SMMU_FEATURE_STALL_FORCE (1 << 9)
+#define SMMU_FEATURE_STALL (1 << 10)
+#define SMMU_FEATURE_S1P (1 << 11)
+#define SMMU_FEATURE_S2P (1 << 12)
+#define SMMU_FEATURE_VAX (1 << 13)
+#define SMMU_FEATURE_COHERENCY (1 << 14)
+#define SMMU_FEATURE_RANGE_INV (1 << 15)
+ struct smmu_queue cmdq;
+ struct smmu_queue evtq;
+ struct smmu_queue priq;
+ struct smmu_strtab strtab;
+ int sync;
+ struct mtx sc_mtx;
+
+ bitstr_t *asid_set;
+ int asid_set_size;
+ struct mtx asid_set_mutex;
+ struct smmu_unit unit;
+};
+
+MALLOC_DECLARE(M_SMMU);
+
+/* Device methods */
+int smmu_attach(device_t dev);
+int smmu_detach(device_t dev);
+
+#endif /* _DEV_IOMMU_SMMU_VAR_H_ */
Index: sys/conf/files.arm64
===================================================================
--- sys/conf/files.arm64
+++ sys/conf/files.arm64
@@ -210,6 +210,10 @@
arm64/intel/firmware.c optional soc_intel_stratix10
arm64/intel/stratix10-soc-fpga-mgr.c optional soc_intel_stratix10
arm64/intel/stratix10-svc.c optional soc_intel_stratix10
+arm64/iommu/iommu.c optional acpi acpi_smmu pci
+arm64/iommu/iommu_if.m optional acpi acpi_smmu pci
+arm64/iommu/smmu.c optional acpi acpi_smmu pci
+arm64/iommu/smmu_acpi.c optional acpi acpi_smmu pci
arm64/qoriq/ls1046_gpio.c optional ls1046_gpio gpio fdt SOC_NXP_LS
arm64/qoriq/clk/ls1046a_clkgen.c optional clk SOC_NXP_LS
arm64/qoriq/clk/qoriq_clk_pll.c optional clk SOC_NXP_LS
@@ -304,6 +308,8 @@
dev/iicbus/twsi/mv_twsi.c optional twsi fdt
dev/iicbus/twsi/a10_twsi.c optional twsi fdt
dev/iicbus/twsi/twsi.c optional twsi fdt
+dev/iommu/busdma_iommu.c optional acpi acpi_smmu pci
+dev/iommu/iommu_gas.c optional acpi acpi_smmu pci
dev/hwpmc/hwpmc_arm64.c optional hwpmc
dev/hwpmc/hwpmc_arm64_md.c optional hwpmc
dev/mbox/mbox_if.m optional soc_brcm_bcm2837
Index: sys/conf/options
===================================================================
--- sys/conf/options
+++ sys/conf/options
@@ -705,6 +705,7 @@
ACPI_MAX_TASKS opt_acpi.h
ACPI_MAX_THREADS opt_acpi.h
ACPI_DMAR opt_acpi.h
+ACPI_SMMU opt_acpi.h
DEV_ACPI opt_acpi.h
# ISA support
Index: sys/dev/acpica/acpi_pci.c
===================================================================
--- sys/dev/acpica/acpi_pci.c
+++ sys/dev/acpica/acpi_pci.c
@@ -455,7 +455,7 @@
return (pci_detach(dev));
}
-#ifdef ACPI_DMAR
+#if defined(ACPI_DMAR) || defined(ACPI_SMMU)
bus_dma_tag_t acpi_iommu_get_dma_tag(device_t dev, device_t child);
static bus_dma_tag_t
acpi_pci_get_dma_tag(device_t bus, device_t child)
Index: sys/dev/iommu/busdma_iommu.c
===================================================================
--- sys/dev/iommu/busdma_iommu.c
+++ sys/dev/iommu/busdma_iommu.c
@@ -69,6 +69,9 @@
#include <dev/iommu/busdma_iommu.h>
#include <dev/iommu/iommu.h>
#include <x86/iommu/intel_dmar.h>
+#elif defined(__aarch64__)
+#include <dev/iommu/busdma_iommu.h>
+#include <arm64/iommu/iommu.h>
#endif
/*
@@ -883,7 +886,7 @@
struct bus_dmamap_iommu *map;
struct iommu_ctx *ctx;
struct iommu_domain *domain;
-#if defined(__amd64__)
+#if defined(__amd64__) || defined(__aarch64__)
struct iommu_map_entries_tailq entries;
#endif
Index: sys/dev/iommu/iommu_gas.c
===================================================================
--- sys/dev/iommu/iommu_gas.c
+++ sys/dev/iommu/iommu_gas.c
@@ -67,6 +67,9 @@
#include <machine/md_var.h>
#if defined(__amd64__) || defined(__i386__)
#include <x86/iommu/intel_reg.h>
+#elif defined(__aarch64__)
+#include <dev/iommu/busdma_iommu.h>
+#include <arm64/iommu/iommu.h>
#endif
#include <dev/iommu/busdma_iommu.h>
Index: sys/dev/pci/pci.c
===================================================================
--- sys/dev/pci/pci.c
+++ sys/dev/pci/pci.c
@@ -5685,7 +5685,7 @@
return (&dinfo->resources);
}
-#ifdef ACPI_DMAR
+#if defined(ACPI_DMAR) || defined(ACPI_SMMU)
bus_dma_tag_t acpi_iommu_get_dma_tag(device_t dev, device_t child);
bus_dma_tag_t
pci_get_dma_tag(device_t bus, device_t dev)

File Metadata

Mime Type
text/plain
Expires
Sat, Jan 11, 7:08 PM (19 h, 18 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
15756946
Default Alt Text
D24618.id75304.diff (103 KB)

Event Timeline