diff --git a/sys/arm64/conf/NOTES b/sys/arm64/conf/NOTES index 3e8499c872b9..cc713db86538 100644 --- a/sys/arm64/conf/NOTES +++ b/sys/arm64/conf/NOTES @@ -1,253 +1,255 @@ # # NOTES -- Lines that can be cut/pasted into kernel and hints configs. # # This file contains machine dependent kernel configuration notes. For # machine independent notes, look in /sys/conf/NOTES. # # # # We want LINT to cover profiling as well. # Except it's broken. #profile 2 # # Enable the kernel DTrace hooks which are required to load the DTrace # kernel modules. # options KDTRACE_HOOKS # # Most of the following is copied from ARM64 GENERIC. cpu ARM64 makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. options KDTRACE_FRAME # Ensure frames are compiled in options VFP # Floating-point support options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default # SoC support options SOC_ALLWINNER_A64 options SOC_ALLWINNER_H5 options SOC_ALLWINNER_H6 options SOC_BRCM_BCM2837 options SOC_BRCM_BCM2838 options SOC_BRCM_NS2 options SOC_CAVM_THUNDERX options SOC_FREESCALE_IMX8 options SOC_HISI_HI6220 options SOC_INTEL_STRATIX10 options SOC_MARVELL_8K options SOC_NVIDIA_TEGRA210 options SOC_NXP_LS options SOC_ROCKCHIP_RK3328 options SOC_ROCKCHIP_RK3399 options SOC_XILINX_ZYNQ # Timer drivers device a10_timer # Annapurna Alpine drivers device al_ccu # Alpine Cache Coherency Unit device al_nb_service # Alpine North Bridge Service device al_iofic # I/O Fabric Interrupt Controller device al_serdes # Serializer/Deserializer device al_udma # Universal DMA # Qualcomm Snapdragon drivers device qcom_gcc # Global Clock Controller # Google Virtual NIC device gve # Google Virtual NIC (gVNIC) driver # Microsoft Hyper-V device hyperv # CPU frequency control device cpufreq # Accelerated software crypto device armv8crypto # ARMv8 OpenCrypto module device ossl # OpenSSL OpenCrypto module # Bus drivers device al_pci # Annapurna Alpine PCI-E options PCI_HP # PCI-Express native HotPlug options PCI_IOV # PCI SR-IOV support # Ethernet NICs device mdio device awg # Allwinner EMAC Gigabit Ethernet device axa # AMD Opteron A1100 integrated NIC device neta # Marvell Armada 370/38x/XP/3700 NIC device smc # SMSC LAN91C111 device vnic # Cavium ThunderX NIC device al_eth # Annapurna Alpine Ethernet NIC device dwc # Synopsys Designware GMAC Ethernet device dwc_rk # Rockchip Designware device dwc_socfpga # Altera SOCFPGA Ethernet MAC device ice # Intel 800 Series Physical Function device ice_ddp # Intel 800 Series DDP Package # Etherswitch devices device e6000sw # Marvell mv88e6085 based switches # MMC/SD/SDIO Card slot support device sdhci_xenon # Marvell Xenon SD/MMC controller device aw_mmc # Allwinner SD/MMC controller device dwmmc device dwmmc_altera device rk_emmcphy # Serial (COM) ports device uart_msm # Qualcomm MSM UART driver device uart_mu # RPI3 aux port device uart_mvebu # Armada 3700 UART driver device uart_ns8250 # ns8250-type UART driver device uart_snps device pl011 # Early printf using the pl011 uart under the Arm FVP options SOCDEV_PA=0x1c090000 options EARLY_PRINTF=pl011 # USB support device aw_usbphy # Allwinner USB PHY device dwcotg # DWC OTG controller device ehci_mv # Marvell EHCI USB interface # USB ethernet support device muge device smsc # Sound support device a10_codec # DMA controller device a31_dmac # GPIO / PINCTRL device a37x0_gpio # Marvell Armada 37x0 GPIO controller device aw_gpio # Allwinner GPIO controller device fdt_pinctrl device mv_gpio # Marvell GPIO controller device mvebu_pinctrl # Marvell Pinmux Controller device rk_gpio # RockChip GPIO Controller device rk_pinctrl # RockChip Pinmux Controller # I2C device aw_rsb # Allwinner Reduced Serial Bus device bcm2835_bsc # Broadcom BCM283x I2C bus device twsi # Allwinner I2C controller device rk_i2c # RockChip I2C controller # Clock and reset controllers device aw_ccu # Allwinner clock controller # Interrupt controllers device aw_nmi # Allwinner NMI support device mv_cp110_icu # Marvell CP110 ICU device mv_ap806_gicp # Marvell AP806 GICP # Real-time clock support device aw_rtc # Allwinner Real-time Clock device mv_rtc # Marvell Real-time Clock # Watchdog controllers device aw_wdog # Allwinner Watchdog # Power management controllers device axp81x # X-Powers AXP81x PMIC device rk805 # RockChip RK805 PMIC # EFUSE device aw_sid # Allwinner Secure ID EFUSE # Thermal sensors device aw_thermal # Allwinner Thermal Sensor Controller device mv_thermal # Marvell Thermal Sensor Controller # SPI device bcm2835_spi # Broadcom BCM283x SPI bus # PWM device pwm device aw_pwm device vt_efifb device vt_simplefb # EVDEV support options EVDEV_SUPPORT # evdev support in legacy drivers device aw_cir # Pseudo devices. device clk device efidev # EFI pseudo-device device efirtc # EFI RTC device phy device hwreset device nvmem device regulator device syscon device aw_syscon # Backlight subsystem device backlight # Misc devices. device pl330 # ARM PL330 dma controller device xdma # xDMA framework for SoC on-chip dma controllers # Chip-specific errata options THUNDERX_PASS_1_1_ERRATA options EFIRT # EFI Runtime Services options FDT device acpi # DTBs makeoptions MODULES_EXTRA="dtb/allwinner dtb/rockchip dtb/rpi" # Add CAMDEBUG stuff options CAMDEBUG options CAM_DEBUG_FLAGS=(CAM_DEBUG_INFO|CAM_DEBUG_PROBE|CAM_DEBUG_PERIPH) # bring in camified MMC too options MMCCAM # arm64 doesn't support inb/outb, so disable chipset probing which needs it nooptions PPC_PROBE_CHIPSET # These cause weird issues, not sure why nooptions DEBUG # Makes assumptions about bus tags that aren't true on arm64 nodevice snd_cmi # arm64 didn't exist for these releases, so doesn't have the required compat # support. Just disable them because they are meaningless. nooptions COMPAT_FREEBSD4 nooptions COMPAT_FREEBSD5 nooptions COMPAT_FREEBSD6 nooptions COMPAT_FREEBSD7 nooptions COMPAT_FREEBSD9 nooptions COMPAT_FREEBSD10 # arm64 supports 32-bit FreeBSD/arm binaries (armv[67] ABIs) options COMPAT_FREEBSD32 # Compatible with FreeBSD/arm +options IOMMU # ARM64 SMMU/IOMMU + ##################################################################### # ZFS support options ZFS # # HID-over-I2C support # device iichid # HID-over-I2C support options IICHID_DEBUG # Enable HID-over-I2C debug messages -options IICHID_SAMPLING # Workaround missing GPIO INTR support +options IICHID_SAMPLING # Workaround missing GPIO INTR support \ No newline at end of file diff --git a/sys/arm64/iommu/iommu.c b/sys/arm64/iommu/iommu.c index cb2b86c9dc41..b765763e3a60 100644 --- a/sys/arm64/iommu/iommu.c +++ b/sys/arm64/iommu/iommu.c @@ -1,522 +1,521 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Ruslan Bukin * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Portions of this work was supported by Innovate UK project 105694, * "Digital Security by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include "iommu.h" #include "iommu_if.h" static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework"); #define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx) #define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx) #define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED) #define dprintf(fmt, ...) static struct sx iommu_sx; struct iommu_entry { struct iommu_unit *iommu; LIST_ENTRY(iommu_entry) next; }; static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list); static int iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, int flags) { struct iommu_unit *iommu; int error; iommu = iodom->iommu; error = IOMMU_UNMAP(iommu->dev, iodom, base, size); return (error); } static int iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags) { struct iommu_unit *iommu; vm_prot_t prot; vm_offset_t va; int error; dprintf("%s: base %lx, size %lx\n", __func__, base, size); prot = 0; if (eflags & IOMMU_MAP_ENTRY_READ) prot |= VM_PROT_READ; if (eflags & IOMMU_MAP_ENTRY_WRITE) prot |= VM_PROT_WRITE; va = base; iommu = iodom->iommu; error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot); return (error); } static const struct iommu_domain_map_ops domain_map_ops = { .map = iommu_domain_map_buf, .unmap = iommu_domain_unmap_buf, }; static struct iommu_domain * iommu_domain_alloc(struct iommu_unit *iommu) { struct iommu_domain *iodom; iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu); if (iodom == NULL) return (NULL); KASSERT(iodom->end != 0, ("domain end is not set")); iommu_domain_init(iommu, iodom, &domain_map_ops); iodom->iommu = iommu; iommu_gas_init_domain(iodom); return (iodom); } static int iommu_domain_free(struct iommu_domain *iodom) { struct iommu_unit *iommu; iommu = iodom->iommu; IOMMU_LOCK(iommu); if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) { IOMMU_DOMAIN_LOCK(iodom); iommu_gas_fini_domain(iodom); IOMMU_DOMAIN_UNLOCK(iodom); } iommu_domain_fini(iodom); IOMMU_DOMAIN_FREE(iommu->dev, iodom); IOMMU_UNLOCK(iommu); return (0); } static void iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t) { bus_addr_t maxaddr; maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR); - t->common.ref_count = 0; t->common.impl = &bus_dma_iommu_impl; t->common.alignment = 1; t->common.boundary = 0; t->common.lowaddr = maxaddr; t->common.highaddr = maxaddr; t->common.maxsize = maxaddr; t->common.nsegments = BUS_SPACE_UNRESTRICTED; t->common.maxsegsz = maxaddr; } static struct iommu_ctx * iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled) { struct iommu_unit *iommu; struct iommu_ctx *ioctx; iommu = iodom->iommu; ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled); if (ioctx == NULL) return (NULL); ioctx->domain = iodom; return (ioctx); } static int iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx) { struct bus_dma_tag_iommu *tag; struct iommu_domain *iodom; struct iommu_unit *iommu; int error; iodom = ioctx->domain; iommu = iodom->iommu; error = IOMMU_CTX_INIT(iommu->dev, ioctx); if (error) return (error); tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu), M_IOMMU, M_WAITOK | M_ZERO); tag->owner = requester; tag->ctx = ioctx; tag->ctx->domain = iodom; iommu_tag_init(iodom, tag); return (error); } static struct iommu_unit * iommu_lookup(device_t dev) { struct iommu_entry *entry; struct iommu_unit *iommu; IOMMU_LIST_LOCK(); LIST_FOREACH(entry, &iommu_list, next) { iommu = entry->iommu; if (iommu->dev == dev) { IOMMU_LIST_UNLOCK(); return (iommu); } } IOMMU_LIST_UNLOCK(); return (NULL); } #ifdef FDT struct iommu_ctx * iommu_get_ctx_ofw(device_t dev, int channel) { struct iommu_domain *iodom; struct iommu_unit *iommu; struct iommu_ctx *ioctx; phandle_t node, parent; device_t iommu_dev; pcell_t *cells; int niommus; int ncells; int error; node = ofw_bus_get_node(dev); if (node <= 0) { device_printf(dev, "%s called on not ofw based device.\n", __func__); return (NULL); } error = ofw_bus_parse_xref_list_get_length(node, "iommus", "#iommu-cells", &niommus); if (error) { device_printf(dev, "%s can't get iommu list.\n", __func__); return (NULL); } if (niommus == 0) { device_printf(dev, "%s iommu list is empty.\n", __func__); return (NULL); } error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells", channel, &parent, &ncells, &cells); if (error != 0) { device_printf(dev, "%s can't get iommu device xref.\n", __func__); return (NULL); } iommu_dev = OF_device_from_xref(parent); if (iommu_dev == NULL) { device_printf(dev, "%s can't get iommu device.\n", __func__); return (NULL); } iommu = iommu_lookup(iommu_dev); if (iommu == NULL) { device_printf(dev, "%s can't lookup iommu.\n", __func__); return (NULL); } /* * In our current configuration we have a domain per each ctx, * so allocate a domain first. */ iodom = iommu_domain_alloc(iommu); if (iodom == NULL) { device_printf(dev, "%s can't allocate domain.\n", __func__); return (NULL); } ioctx = iommu_ctx_alloc(dev, iodom, false); if (ioctx == NULL) { iommu_domain_free(iodom); return (NULL); } ioctx->domain = iodom; error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells); if (error) { device_printf(dev, "%s can't set MD data\n", __func__); return (NULL); } error = iommu_ctx_init(dev, ioctx); if (error) { IOMMU_CTX_FREE(iommu->dev, ioctx); iommu_domain_free(iodom); return (NULL); } return (ioctx); } #endif struct iommu_ctx * iommu_get_ctx(struct iommu_unit *iommu, device_t requester, uint16_t rid, bool disabled, bool rmrr) { struct iommu_domain *iodom; struct iommu_ctx *ioctx; int error; IOMMU_LOCK(iommu); ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester); if (ioctx) { IOMMU_UNLOCK(iommu); return (ioctx); } IOMMU_UNLOCK(iommu); /* * In our current configuration we have a domain per each ctx. * So allocate a domain first. */ iodom = iommu_domain_alloc(iommu); if (iodom == NULL) return (NULL); ioctx = iommu_ctx_alloc(requester, iodom, disabled); if (ioctx == NULL) { iommu_domain_free(iodom); return (NULL); } error = iommu_ctx_init(requester, ioctx); if (error) { IOMMU_CTX_FREE(iommu->dev, ioctx); iommu_domain_free(iodom); return (NULL); } return (ioctx); } void iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx) { struct bus_dma_tag_iommu *tag; IOMMU_ASSERT_LOCKED(iommu); tag = ioctx->tag; IOMMU_CTX_FREE(iommu->dev, ioctx); free(tag, M_IOMMU); } void iommu_free_ctx(struct iommu_ctx *ioctx) { struct iommu_unit *iommu; struct iommu_domain *iodom; int error; iodom = ioctx->domain; iommu = iodom->iommu; IOMMU_LOCK(iommu); iommu_free_ctx_locked(iommu, ioctx); IOMMU_UNLOCK(iommu); /* Since we have a domain per each ctx, remove the domain too. */ error = iommu_domain_free(iodom); if (error) device_printf(iommu->dev, "Could not free a domain\n"); } static void iommu_domain_free_entry(struct iommu_map_entry *entry, bool free) { iommu_gas_free_space(entry); if (free) iommu_gas_free_entry(entry); else entry->flags = 0; } void iommu_domain_unload(struct iommu_domain *iodom, struct iommu_map_entries_tailq *entries, bool cansleep) { struct iommu_map_entry *entry, *entry1; int error __diagused; TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, ("not mapped entry %p %p", iodom, entry)); error = iodom->ops->unmap(iodom, entry->start, entry->end - entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); KASSERT(error == 0, ("unmap %p error %d", iodom, error)); TAILQ_REMOVE(entries, entry, dmamap_link); iommu_domain_free_entry(entry, true); } if (TAILQ_EMPTY(entries)) return; panic("entries map is not empty"); } int iommu_register(struct iommu_unit *iommu) { struct iommu_entry *entry; mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF); entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO); entry->iommu = iommu; IOMMU_LIST_LOCK(); LIST_INSERT_HEAD(&iommu_list, entry, next); IOMMU_LIST_UNLOCK(); iommu_init_busdma(iommu); return (0); } int iommu_unregister(struct iommu_unit *iommu) { struct iommu_entry *entry, *tmp; IOMMU_LIST_LOCK(); LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) { if (entry->iommu == iommu) { LIST_REMOVE(entry, next); free(entry, M_IOMMU); } } IOMMU_LIST_UNLOCK(); iommu_fini_busdma(iommu); mtx_destroy(&iommu->lock); return (0); } struct iommu_unit * iommu_find(device_t dev, bool verbose) { struct iommu_entry *entry; struct iommu_unit *iommu; int error; IOMMU_LIST_LOCK(); LIST_FOREACH(entry, &iommu_list, next) { iommu = entry->iommu; error = IOMMU_FIND(iommu->dev, dev); if (error == 0) { IOMMU_LIST_UNLOCK(); return (entry->iommu); } } IOMMU_LIST_UNLOCK(); return (NULL); } void iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free, bool cansleep __unused) { dprintf("%s\n", __func__); iommu_domain_free_entry(entry, free); } static void iommu_init(void) { sx_init(&iommu_sx, "IOMMU list"); } SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL); diff --git a/sys/arm64/iommu/iommu_pmap.c b/sys/arm64/iommu/iommu_pmap.c index d356a92c4d66..8a0ea641e231 100644 --- a/sys/arm64/iommu/iommu_pmap.c +++ b/sys/arm64/iommu/iommu_pmap.c @@ -1,863 +1,863 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020-2021 Ruslan Bukin * Copyright (c) 2014-2021 Andrew Turner * Copyright (c) 2014-2016 The FreeBSD Foundation * All rights reserved. * * This work was supported by Innovate UK project 105694, "Digital Security * by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * Manages physical address maps for ARM SMMUv3 and ARM Mali GPU. */ #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define IOMMU_PAGE_SIZE 4096 #define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx) #define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx) #define SMMU_PMAP_LOCK_ASSERT(pmap, type) \ mtx_assert(&(pmap)->sp_mtx, (type)) #define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t))) #define NUL0E IOMMU_L0_ENTRIES #define NUL1E (NUL0E * NL1PG) #define NUL2E (NUL1E * NL2PG) #define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) #define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) #define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) #define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK) #define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK) #define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK) #define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK) static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex); static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, struct spglist *free); /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at * the same time as the CPU. */ #define smmu_pmap_load(table) (*table) #define smmu_pmap_clear(table) atomic_store_64(table, 0) #define smmu_pmap_store(table, entry) atomic_store_64(table, entry) /********************/ /* Inline functions */ /********************/ static __inline pd_entry_t * smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va) { return (&pmap->sp_l0[smmu_l0_index(va)]); } static __inline pd_entry_t * smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) { pd_entry_t *l1; l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK); return (&l1[smmu_l1_index(va)]); } static __inline pd_entry_t * smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va) { pd_entry_t *l0; l0 = smmu_pmap_l0(pmap, va); if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) return (NULL); return (smmu_pmap_l0_to_l1(l0, va)); } static __inline pd_entry_t * smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) { pd_entry_t l1, *l2p; l1 = smmu_pmap_load(l1p); /* * The valid bit may be clear if pmap_update_entry() is concurrently * modifying the entry, so for KVA only the entry type may be checked. */ KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); return (&l2p[smmu_l2_index(va)]); } static __inline pd_entry_t * smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va) { pd_entry_t *l1; l1 = smmu_pmap_l1(pmap, va); if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) return (NULL); return (smmu_pmap_l1_to_l2(l1, va)); } static __inline pt_entry_t * smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) { pd_entry_t l2; pt_entry_t *l3p; l2 = smmu_pmap_load(l2p); /* * The valid bit may be clear if pmap_update_entry() is concurrently * modifying the entry, so for KVA only the entry type may be checked. */ KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); return (&l3p[smmu_l3_index(va)]); } /* * Returns the lowest valid pde for a given virtual address. * The next level may or may not point to a valid page or block. */ static __inline pd_entry_t * smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level) { pd_entry_t *l0, *l1, *l2, desc; l0 = smmu_pmap_l0(pmap, va); desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK; if (desc != IOMMU_L0_TABLE) { *level = -1; return (NULL); } l1 = smmu_pmap_l0_to_l1(l0, va); desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; if (desc != IOMMU_L1_TABLE) { *level = 0; return (l0); } l2 = smmu_pmap_l1_to_l2(l1, va); desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; if (desc != IOMMU_L2_TABLE) { *level = 1; return (l1); } *level = 2; return (l2); } /* * Returns the lowest valid pte block or table entry for a given virtual * address. If there are no valid entries return NULL and set the level to * the first invalid level. */ static __inline pt_entry_t * smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level) { pd_entry_t *l1, *l2, desc; pt_entry_t *l3; l1 = smmu_pmap_l1(pmap, va); if (l1 == NULL) { *level = 0; return (NULL); } desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; if (desc == IOMMU_L1_BLOCK) { *level = 1; return (l1); } if (desc != IOMMU_L1_TABLE) { *level = 1; return (NULL); } l2 = smmu_pmap_l1_to_l2(l1, va); desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; if (desc == IOMMU_L2_BLOCK) { *level = 2; return (l2); } if (desc != IOMMU_L2_TABLE) { *level = 2; return (NULL); } *level = 3; l3 = smmu_pmap_l2_to_l3(l2, va); if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) return (NULL); return (l3); } static __inline int smmu_pmap_l3_valid(pt_entry_t l3) { return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE); } CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK); #ifdef INVARIANTS static __inline void smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) { SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); pmap->sp_resident_count += count; } static __inline void smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) { SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(pmap->sp_resident_count >= count, ("pmap %p resident count underflow %ld %d", pmap, pmap->sp_resident_count, count)); pmap->sp_resident_count -= count; } #else static __inline void smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) { } static __inline void smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) { } #endif /*************************************************** * Page table page management routines..... ***************************************************/ /* * Schedule the specified unused page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, boolean_t set_PG_ZERO) { if (set_PG_ZERO) m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /*************************************************** * Low level mapping routines..... ***************************************************/ /* * Decrements a page table page's reference count, which is used to record the * number of valid page table entries within the page. If the reference count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static inline boolean_t smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { --m->ref_count; if (m->ref_count == 0) { _smmu_pmap_unwire_l3(pmap, va, m, free); return (TRUE); } else return (FALSE); } static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ if (m->pindex >= (NUL2E + NUL1E)) { /* l1 page */ pd_entry_t *l0; l0 = smmu_pmap_l0(pmap, va); smmu_pmap_clear(l0); } else if (m->pindex >= NUL2E) { /* l2 page */ pd_entry_t *l1; l1 = smmu_pmap_l1(pmap, va); smmu_pmap_clear(l1); } else { /* l3 page */ pd_entry_t *l2; l2 = smmu_pmap_l2(pmap, va); smmu_pmap_clear(l2); } smmu_pmap_resident_count_dec(pmap, 1); if (m->pindex < NUL2E) { /* We just released an l3, unhold the matching l2 */ pd_entry_t *l1, tl1; vm_page_t l2pg; l1 = smmu_pmap_l1(pmap, va); tl1 = smmu_pmap_load(l1); l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); smmu_pmap_unwire_l3(pmap, va, l2pg, free); } else if (m->pindex < (NUL2E + NUL1E)) { /* We just released an l2, unhold the matching l1 */ pd_entry_t *l0, tl0; vm_page_t l1pg; l0 = smmu_pmap_l0(pmap, va); tl0 = smmu_pmap_load(l0); l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); smmu_pmap_unwire_l3(pmap, va, l1pg, free); } /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ smmu_pmap_add_delayed_free_list(m, free, TRUE); } int smmu_pmap_pinit(struct smmu_pmap *pmap) { vm_page_t m; /* * allocate the l0 page */ m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED | VM_ALLOC_ZERO); pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m); pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr); #ifdef INVARIANTS pmap->sp_resident_count = 0; #endif mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF); return (1); } /* * This routine is called if the desired page table page does not exist. * * If page table page allocation fails, this routine may sleep before * returning NULL. It sleeps only if a lock pointer was given. * * Note: If a page allocation fails at page table level two or three, * one or two pages may be held during the wait, only to be released * afterwards. This conservative approach is easily argued to avoid * race conditions. */ static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex) { vm_page_t m, l1pg, l2pg; SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Allocate a page table page. */ if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { /* * Indicate the need to retry. While waiting, the page table * page may have been allocated. */ return (NULL); } m->pindex = ptepindex; /* * Because of AArch64's weak memory consistency model, we must have a * barrier here to ensure that the stores for zeroing "m", whether by * pmap_zero_page() or an earlier function, are visible before adding * "m" to the page table. Otherwise, a page table walk by another * processor's MMU could see the mapping to "m" and a stale, non-zero * PTE within "m". */ dmb(ishst); /* * Map the pagetable page into the process address space, if * it isn't already there. */ if (ptepindex >= (NUL2E + NUL1E)) { pd_entry_t *l0; vm_pindex_t l0index; l0index = ptepindex - (NUL2E + NUL1E); l0 = &pmap->sp_l0[l0index]; smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); } else if (ptepindex >= NUL2E) { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1; pd_entry_t tl0; l1index = ptepindex - NUL2E; l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; l0 = &pmap->sp_l0[l0index]; tl0 = smmu_pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } } else { l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); l1pg->ref_count++; } l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK); l1 = &l1[ptepindex & Ln_ADDR_MASK]; smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); } else { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1, *l2; pd_entry_t tl0, tl1; l1index = ptepindex >> IOMMU_Ln_ENTRIES_SHIFT; l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; l0 = &pmap->sp_l0[l0index]; tl0 = smmu_pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } tl0 = smmu_pmap_load(l0); l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; } else { l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; tl1 = smmu_pmap_load(l1); if (tl1 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } } else { l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); l2pg->ref_count++; } } l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK); l2 = &l2[ptepindex & Ln_ADDR_MASK]; smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); } smmu_pmap_resident_count_inc(pmap, 1); return (m); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void smmu_pmap_release(struct smmu_pmap *pmap) { vm_page_t m; KASSERT(pmap->sp_resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->sp_resident_count)); m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr); vm_page_unwire_noq(m); vm_page_free_zero(m); mtx_destroy(&pmap->sp_mtx); } /*************************************************** * page management routines. ***************************************************/ /* * Add a single Mali GPU entry. This function does not sleep. */ int pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags) { pd_entry_t *pde; pt_entry_t new_l3; pt_entry_t orig_l3 __diagused; pt_entry_t *l3; vm_page_t mpte; pd_entry_t *l1p; pd_entry_t *l2p; int lvl; int rv; KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK); if ((prot & VM_PROT_WRITE) != 0) new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); if ((prot & VM_PROT_READ) != 0) new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); if ((prot & VM_PROT_EXECUTE) == 0) new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL); CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa); SMMU_PMAP_LOCK(pmap); /* * In the case that a page table page is not * resident, we are creating it here. */ retry: pde = smmu_pmap_pde(pmap, va, &lvl); if (pde != NULL && lvl == 2) { l3 = smmu_pmap_l2_to_l3(pde, va); } else { mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); if (mpte == NULL) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); rv = KERN_RESOURCE_SHORTAGE; goto out; } /* * Ensure newly created l1, l2 are visible to GPU. * l0 is already visible by similar call in panfrost driver. * The cache entry for l3 handled below. */ l1p = smmu_pmap_l1(pmap, va); l2p = smmu_pmap_l2(pmap, va); - cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t)); - cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t)); + cpu_dcache_wb_range(l1p, sizeof(pd_entry_t)); + cpu_dcache_wb_range(l2p, sizeof(pd_entry_t)); goto retry; } orig_l3 = smmu_pmap_load(l3); KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); /* New mapping */ smmu_pmap_store(l3, new_l3); - cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t)); + cpu_dcache_wb_range(l3, sizeof(pt_entry_t)); smmu_pmap_resident_count_inc(pmap, 1); dsb(ishst); rv = KERN_SUCCESS; out: SMMU_PMAP_UNLOCK(pmap); return (rv); } /* * Remove a single Mali GPU entry. */ int pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va) { pd_entry_t *pde; pt_entry_t *pte; int lvl; int rc; KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); SMMU_PMAP_LOCK(pmap); pde = smmu_pmap_pde(pmap, va, &lvl); if (pde == NULL || lvl != 2) { rc = KERN_FAILURE; goto out; } pte = smmu_pmap_l2_to_l3(pde, va); smmu_pmap_resident_count_dec(pmap, 1); smmu_pmap_clear(pte); - cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t)); + cpu_dcache_wb_range(pte, sizeof(pt_entry_t)); rc = KERN_SUCCESS; out: SMMU_PMAP_UNLOCK(pmap); return (rc); } /* * Add a single SMMU entry. This function does not sleep. */ int smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags) { pd_entry_t *pde; pt_entry_t new_l3; pt_entry_t orig_l3 __diagused; pt_entry_t *l3; vm_page_t mpte; int lvl; int rv; KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); va = trunc_page(va); new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE); if ((prot & VM_PROT_WRITE) == 0) new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); new_l3 |= ATTR_S1_XN; /* Execute never. */ new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER); new_l3 |= ATTR_S1_nG; /* Non global. */ CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa); SMMU_PMAP_LOCK(pmap); /* * In the case that a page table page is not * resident, we are creating it here. */ retry: pde = smmu_pmap_pde(pmap, va, &lvl); if (pde != NULL && lvl == 2) { l3 = smmu_pmap_l2_to_l3(pde, va); } else { mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); if (mpte == NULL) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); rv = KERN_RESOURCE_SHORTAGE; goto out; } goto retry; } orig_l3 = smmu_pmap_load(l3); KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); /* New mapping */ smmu_pmap_store(l3, new_l3); smmu_pmap_resident_count_inc(pmap, 1); dsb(ishst); rv = KERN_SUCCESS; out: SMMU_PMAP_UNLOCK(pmap); return (rv); } /* * Remove a single SMMU entry. */ int smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va) { pt_entry_t *pte; int lvl; int rc; SMMU_PMAP_LOCK(pmap); pte = smmu_pmap_pte(pmap, va, &lvl); KASSERT(lvl == 3, ("Invalid SMMU pagetable level: %d != 3", lvl)); if (pte != NULL) { smmu_pmap_resident_count_dec(pmap, 1); smmu_pmap_clear(pte); rc = KERN_SUCCESS; } else rc = KERN_FAILURE; SMMU_PMAP_UNLOCK(pmap); return (rc); } /* * Remove all the allocated L1, L2 pages from SMMU pmap. * All the L3 entires must be cleared in advance, otherwise * this function panics. */ void smmu_pmap_remove_pages(struct smmu_pmap *pmap) { pd_entry_t l0e, *l1, l1e, *l2, l2e; pt_entry_t *l3, l3e; vm_page_t m, m0, m1; vm_paddr_t pa; vm_paddr_t pa0; vm_paddr_t pa1; int i, j, k, l; SMMU_PMAP_LOCK(pmap); for (i = 0; i < IOMMU_L0_ENTRIES; i++) { l0e = pmap->sp_l0[i]; if ((l0e & ATTR_DESCR_VALID) == 0) { continue; } pa0 = l0e & ~ATTR_MASK; m0 = PHYS_TO_VM_PAGE(pa0); l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0); for (j = 0; j < IOMMU_Ln_ENTRIES; j++) { l1e = l1[j]; if ((l1e & ATTR_DESCR_VALID) == 0) { continue; } if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) { continue; } pa1 = l1e & ~ATTR_MASK; m1 = PHYS_TO_VM_PAGE(pa1); l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1); for (k = 0; k < IOMMU_Ln_ENTRIES; k++) { l2e = l2[k]; if ((l2e & ATTR_DESCR_VALID) == 0) { continue; } pa = l2e & ~ATTR_MASK; m = PHYS_TO_VM_PAGE(pa); l3 = (pt_entry_t *)PHYS_TO_DMAP(pa); for (l = 0; l < IOMMU_Ln_ENTRIES; l++) { l3e = l3[l]; if ((l3e & ATTR_DESCR_VALID) == 0) continue; panic( "%s: l3e found (indexes %d %d %d %d)", __func__, i, j, k, l); } vm_page_unwire_noq(m1); vm_page_unwire_noq(m); smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m); smmu_pmap_clear(&l2[k]); } vm_page_unwire_noq(m0); smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m1); smmu_pmap_clear(&l1[j]); } smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m0); smmu_pmap_clear(&pmap->sp_l0[i]); } KASSERT(pmap->sp_resident_count == 0, ("Invalid resident count %jd", pmap->sp_resident_count)); SMMU_PMAP_UNLOCK(pmap); }