diff --git a/sys/arm64/conf/NOTES b/sys/arm64/conf/NOTES
index 3e8499c872b9..cc713db86538 100644
--- a/sys/arm64/conf/NOTES
+++ b/sys/arm64/conf/NOTES
@@ -1,253 +1,255 @@
#
# NOTES -- Lines that can be cut/pasted into kernel and hints configs.
#
# This file contains machine dependent kernel configuration notes. For
# machine independent notes, look in /sys/conf/NOTES.
#
#
#
# We want LINT to cover profiling as well.
# Except it's broken.
#profile 2
#
# Enable the kernel DTrace hooks which are required to load the DTrace
# kernel modules.
#
options KDTRACE_HOOKS
#
# Most of the following is copied from ARM64 GENERIC.
cpu ARM64
makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support
options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed.
options KDTRACE_FRAME # Ensure frames are compiled in
options VFP # Floating-point support
options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
# SoC support
options SOC_ALLWINNER_A64
options SOC_ALLWINNER_H5
options SOC_ALLWINNER_H6
options SOC_BRCM_BCM2837
options SOC_BRCM_BCM2838
options SOC_BRCM_NS2
options SOC_CAVM_THUNDERX
options SOC_FREESCALE_IMX8
options SOC_HISI_HI6220
options SOC_INTEL_STRATIX10
options SOC_MARVELL_8K
options SOC_NVIDIA_TEGRA210
options SOC_NXP_LS
options SOC_ROCKCHIP_RK3328
options SOC_ROCKCHIP_RK3399
options SOC_XILINX_ZYNQ
# Timer drivers
device a10_timer
# Annapurna Alpine drivers
device al_ccu # Alpine Cache Coherency Unit
device al_nb_service # Alpine North Bridge Service
device al_iofic # I/O Fabric Interrupt Controller
device al_serdes # Serializer/Deserializer
device al_udma # Universal DMA
# Qualcomm Snapdragon drivers
device qcom_gcc # Global Clock Controller
# Google Virtual NIC
device gve # Google Virtual NIC (gVNIC) driver
# Microsoft Hyper-V
device hyperv
# CPU frequency control
device cpufreq
# Accelerated software crypto
device armv8crypto # ARMv8 OpenCrypto module
device ossl # OpenSSL OpenCrypto module
# Bus drivers
device al_pci # Annapurna Alpine PCI-E
options PCI_HP # PCI-Express native HotPlug
options PCI_IOV # PCI SR-IOV support
# Ethernet NICs
device mdio
device awg # Allwinner EMAC Gigabit Ethernet
device axa # AMD Opteron A1100 integrated NIC
device neta # Marvell Armada 370/38x/XP/3700 NIC
device smc # SMSC LAN91C111
device vnic # Cavium ThunderX NIC
device al_eth # Annapurna Alpine Ethernet NIC
device dwc # Synopsys Designware GMAC Ethernet
device dwc_rk # Rockchip Designware
device dwc_socfpga # Altera SOCFPGA Ethernet MAC
device ice # Intel 800 Series Physical Function
device ice_ddp # Intel 800 Series DDP Package
# Etherswitch devices
device e6000sw # Marvell mv88e6085 based switches
# MMC/SD/SDIO Card slot support
device sdhci_xenon # Marvell Xenon SD/MMC controller
device aw_mmc # Allwinner SD/MMC controller
device dwmmc
device dwmmc_altera
device rk_emmcphy
# Serial (COM) ports
device uart_msm # Qualcomm MSM UART driver
device uart_mu # RPI3 aux port
device uart_mvebu # Armada 3700 UART driver
device uart_ns8250 # ns8250-type UART driver
device uart_snps
device pl011
# Early printf using the pl011 uart under the Arm FVP
options SOCDEV_PA=0x1c090000
options EARLY_PRINTF=pl011
# USB support
device aw_usbphy # Allwinner USB PHY
device dwcotg # DWC OTG controller
device ehci_mv # Marvell EHCI USB interface
# USB ethernet support
device muge
device smsc
# Sound support
device a10_codec
# DMA controller
device a31_dmac
# GPIO / PINCTRL
device a37x0_gpio # Marvell Armada 37x0 GPIO controller
device aw_gpio # Allwinner GPIO controller
device fdt_pinctrl
device mv_gpio # Marvell GPIO controller
device mvebu_pinctrl # Marvell Pinmux Controller
device rk_gpio # RockChip GPIO Controller
device rk_pinctrl # RockChip Pinmux Controller
# I2C
device aw_rsb # Allwinner Reduced Serial Bus
device bcm2835_bsc # Broadcom BCM283x I2C bus
device twsi # Allwinner I2C controller
device rk_i2c # RockChip I2C controller
# Clock and reset controllers
device aw_ccu # Allwinner clock controller
# Interrupt controllers
device aw_nmi # Allwinner NMI support
device mv_cp110_icu # Marvell CP110 ICU
device mv_ap806_gicp # Marvell AP806 GICP
# Real-time clock support
device aw_rtc # Allwinner Real-time Clock
device mv_rtc # Marvell Real-time Clock
# Watchdog controllers
device aw_wdog # Allwinner Watchdog
# Power management controllers
device axp81x # X-Powers AXP81x PMIC
device rk805 # RockChip RK805 PMIC
# EFUSE
device aw_sid # Allwinner Secure ID EFUSE
# Thermal sensors
device aw_thermal # Allwinner Thermal Sensor Controller
device mv_thermal # Marvell Thermal Sensor Controller
# SPI
device bcm2835_spi # Broadcom BCM283x SPI bus
# PWM
device pwm
device aw_pwm
device vt_efifb
device vt_simplefb
# EVDEV support
options EVDEV_SUPPORT # evdev support in legacy drivers
device aw_cir
# Pseudo devices.
device clk
device efidev # EFI pseudo-device
device efirtc # EFI RTC
device phy
device hwreset
device nvmem
device regulator
device syscon
device aw_syscon
# Backlight subsystem
device backlight
# Misc devices.
device pl330 # ARM PL330 dma controller
device xdma # xDMA framework for SoC on-chip dma controllers
# Chip-specific errata
options THUNDERX_PASS_1_1_ERRATA
options EFIRT # EFI Runtime Services
options FDT
device acpi
# DTBs
makeoptions MODULES_EXTRA="dtb/allwinner dtb/rockchip dtb/rpi"
# Add CAMDEBUG stuff
options CAMDEBUG
options CAM_DEBUG_FLAGS=(CAM_DEBUG_INFO|CAM_DEBUG_PROBE|CAM_DEBUG_PERIPH)
# bring in camified MMC too
options MMCCAM
# arm64 doesn't support inb/outb, so disable chipset probing which needs it
nooptions PPC_PROBE_CHIPSET
# These cause weird issues, not sure why
nooptions DEBUG
# Makes assumptions about bus tags that aren't true on arm64
nodevice snd_cmi
# arm64 didn't exist for these releases, so doesn't have the required compat
# support. Just disable them because they are meaningless.
nooptions COMPAT_FREEBSD4
nooptions COMPAT_FREEBSD5
nooptions COMPAT_FREEBSD6
nooptions COMPAT_FREEBSD7
nooptions COMPAT_FREEBSD9
nooptions COMPAT_FREEBSD10
# arm64 supports 32-bit FreeBSD/arm binaries (armv[67] ABIs)
options COMPAT_FREEBSD32 # Compatible with FreeBSD/arm
+options IOMMU # ARM64 SMMU/IOMMU
+
#####################################################################
# ZFS support
options ZFS
#
# HID-over-I2C support
#
device iichid # HID-over-I2C support
options IICHID_DEBUG # Enable HID-over-I2C debug messages
-options IICHID_SAMPLING # Workaround missing GPIO INTR support
+options IICHID_SAMPLING # Workaround missing GPIO INTR support
\ No newline at end of file
diff --git a/sys/arm64/iommu/iommu.c b/sys/arm64/iommu/iommu.c
index cb2b86c9dc41..b765763e3a60 100644
--- a/sys/arm64/iommu/iommu.c
+++ b/sys/arm64/iommu/iommu.c
@@ -1,522 +1,521 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
*
* This software was developed by SRI International and the University of
* Cambridge Computer Laboratory (Department of Computer Science and
* Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
* DARPA SSITH research programme.
*
* Portions of this work was supported by Innovate UK project 105694,
* "Digital Security by Design (DSbD) Technology Platform Prototype".
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "opt_platform.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/memdesc.h>
#include <sys/tree.h>
#include <sys/taskqueue.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <machine/bus.h>
#include <dev/iommu/busdma_iommu.h>
#include <machine/vmparam.h>
#ifdef FDT
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#endif
#include "iommu.h"
#include "iommu_if.h"
static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
#define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx)
#define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx)
#define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED)
#define dprintf(fmt, ...)
static struct sx iommu_sx;
struct iommu_entry {
struct iommu_unit *iommu;
LIST_ENTRY(iommu_entry) next;
};
static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
static int
iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
iommu_gaddr_t size, int flags)
{
struct iommu_unit *iommu;
int error;
iommu = iodom->iommu;
error = IOMMU_UNMAP(iommu->dev, iodom, base, size);
return (error);
}
static int
iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
{
struct iommu_unit *iommu;
vm_prot_t prot;
vm_offset_t va;
int error;
dprintf("%s: base %lx, size %lx\n", __func__, base, size);
prot = 0;
if (eflags & IOMMU_MAP_ENTRY_READ)
prot |= VM_PROT_READ;
if (eflags & IOMMU_MAP_ENTRY_WRITE)
prot |= VM_PROT_WRITE;
va = base;
iommu = iodom->iommu;
error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot);
return (error);
}
static const struct iommu_domain_map_ops domain_map_ops = {
.map = iommu_domain_map_buf,
.unmap = iommu_domain_unmap_buf,
};
static struct iommu_domain *
iommu_domain_alloc(struct iommu_unit *iommu)
{
struct iommu_domain *iodom;
iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu);
if (iodom == NULL)
return (NULL);
KASSERT(iodom->end != 0, ("domain end is not set"));
iommu_domain_init(iommu, iodom, &domain_map_ops);
iodom->iommu = iommu;
iommu_gas_init_domain(iodom);
return (iodom);
}
static int
iommu_domain_free(struct iommu_domain *iodom)
{
struct iommu_unit *iommu;
iommu = iodom->iommu;
IOMMU_LOCK(iommu);
if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
IOMMU_DOMAIN_LOCK(iodom);
iommu_gas_fini_domain(iodom);
IOMMU_DOMAIN_UNLOCK(iodom);
}
iommu_domain_fini(iodom);
IOMMU_DOMAIN_FREE(iommu->dev, iodom);
IOMMU_UNLOCK(iommu);
return (0);
}
static void
iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t)
{
bus_addr_t maxaddr;
maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR);
- t->common.ref_count = 0;
t->common.impl = &bus_dma_iommu_impl;
t->common.alignment = 1;
t->common.boundary = 0;
t->common.lowaddr = maxaddr;
t->common.highaddr = maxaddr;
t->common.maxsize = maxaddr;
t->common.nsegments = BUS_SPACE_UNRESTRICTED;
t->common.maxsegsz = maxaddr;
}
static struct iommu_ctx *
iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled)
{
struct iommu_unit *iommu;
struct iommu_ctx *ioctx;
iommu = iodom->iommu;
ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled);
if (ioctx == NULL)
return (NULL);
ioctx->domain = iodom;
return (ioctx);
}
static int
iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx)
{
struct bus_dma_tag_iommu *tag;
struct iommu_domain *iodom;
struct iommu_unit *iommu;
int error;
iodom = ioctx->domain;
iommu = iodom->iommu;
error = IOMMU_CTX_INIT(iommu->dev, ioctx);
if (error)
return (error);
tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu),
M_IOMMU, M_WAITOK | M_ZERO);
tag->owner = requester;
tag->ctx = ioctx;
tag->ctx->domain = iodom;
iommu_tag_init(iodom, tag);
return (error);
}
static struct iommu_unit *
iommu_lookup(device_t dev)
{
struct iommu_entry *entry;
struct iommu_unit *iommu;
IOMMU_LIST_LOCK();
LIST_FOREACH(entry, &iommu_list, next) {
iommu = entry->iommu;
if (iommu->dev == dev) {
IOMMU_LIST_UNLOCK();
return (iommu);
}
}
IOMMU_LIST_UNLOCK();
return (NULL);
}
#ifdef FDT
struct iommu_ctx *
iommu_get_ctx_ofw(device_t dev, int channel)
{
struct iommu_domain *iodom;
struct iommu_unit *iommu;
struct iommu_ctx *ioctx;
phandle_t node, parent;
device_t iommu_dev;
pcell_t *cells;
int niommus;
int ncells;
int error;
node = ofw_bus_get_node(dev);
if (node <= 0) {
device_printf(dev,
"%s called on not ofw based device.\n", __func__);
return (NULL);
}
error = ofw_bus_parse_xref_list_get_length(node,
"iommus", "#iommu-cells", &niommus);
if (error) {
device_printf(dev, "%s can't get iommu list.\n", __func__);
return (NULL);
}
if (niommus == 0) {
device_printf(dev, "%s iommu list is empty.\n", __func__);
return (NULL);
}
error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells",
channel, &parent, &ncells, &cells);
if (error != 0) {
device_printf(dev, "%s can't get iommu device xref.\n",
__func__);
return (NULL);
}
iommu_dev = OF_device_from_xref(parent);
if (iommu_dev == NULL) {
device_printf(dev, "%s can't get iommu device.\n", __func__);
return (NULL);
}
iommu = iommu_lookup(iommu_dev);
if (iommu == NULL) {
device_printf(dev, "%s can't lookup iommu.\n", __func__);
return (NULL);
}
/*
* In our current configuration we have a domain per each ctx,
* so allocate a domain first.
*/
iodom = iommu_domain_alloc(iommu);
if (iodom == NULL) {
device_printf(dev, "%s can't allocate domain.\n", __func__);
return (NULL);
}
ioctx = iommu_ctx_alloc(dev, iodom, false);
if (ioctx == NULL) {
iommu_domain_free(iodom);
return (NULL);
}
ioctx->domain = iodom;
error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells);
if (error) {
device_printf(dev, "%s can't set MD data\n", __func__);
return (NULL);
}
error = iommu_ctx_init(dev, ioctx);
if (error) {
IOMMU_CTX_FREE(iommu->dev, ioctx);
iommu_domain_free(iodom);
return (NULL);
}
return (ioctx);
}
#endif
struct iommu_ctx *
iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
uint16_t rid, bool disabled, bool rmrr)
{
struct iommu_domain *iodom;
struct iommu_ctx *ioctx;
int error;
IOMMU_LOCK(iommu);
ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester);
if (ioctx) {
IOMMU_UNLOCK(iommu);
return (ioctx);
}
IOMMU_UNLOCK(iommu);
/*
* In our current configuration we have a domain per each ctx.
* So allocate a domain first.
*/
iodom = iommu_domain_alloc(iommu);
if (iodom == NULL)
return (NULL);
ioctx = iommu_ctx_alloc(requester, iodom, disabled);
if (ioctx == NULL) {
iommu_domain_free(iodom);
return (NULL);
}
error = iommu_ctx_init(requester, ioctx);
if (error) {
IOMMU_CTX_FREE(iommu->dev, ioctx);
iommu_domain_free(iodom);
return (NULL);
}
return (ioctx);
}
void
iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx)
{
struct bus_dma_tag_iommu *tag;
IOMMU_ASSERT_LOCKED(iommu);
tag = ioctx->tag;
IOMMU_CTX_FREE(iommu->dev, ioctx);
free(tag, M_IOMMU);
}
void
iommu_free_ctx(struct iommu_ctx *ioctx)
{
struct iommu_unit *iommu;
struct iommu_domain *iodom;
int error;
iodom = ioctx->domain;
iommu = iodom->iommu;
IOMMU_LOCK(iommu);
iommu_free_ctx_locked(iommu, ioctx);
IOMMU_UNLOCK(iommu);
/* Since we have a domain per each ctx, remove the domain too. */
error = iommu_domain_free(iodom);
if (error)
device_printf(iommu->dev, "Could not free a domain\n");
}
static void
iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
{
iommu_gas_free_space(entry);
if (free)
iommu_gas_free_entry(entry);
else
entry->flags = 0;
}
void
iommu_domain_unload(struct iommu_domain *iodom,
struct iommu_map_entries_tailq *entries, bool cansleep)
{
struct iommu_map_entry *entry, *entry1;
int error __diagused;
TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
("not mapped entry %p %p", iodom, entry));
error = iodom->ops->unmap(iodom, entry->start, entry->end -
entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
KASSERT(error == 0, ("unmap %p error %d", iodom, error));
TAILQ_REMOVE(entries, entry, dmamap_link);
iommu_domain_free_entry(entry, true);
}
if (TAILQ_EMPTY(entries))
return;
panic("entries map is not empty");
}
int
iommu_register(struct iommu_unit *iommu)
{
struct iommu_entry *entry;
mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF);
entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO);
entry->iommu = iommu;
IOMMU_LIST_LOCK();
LIST_INSERT_HEAD(&iommu_list, entry, next);
IOMMU_LIST_UNLOCK();
iommu_init_busdma(iommu);
return (0);
}
int
iommu_unregister(struct iommu_unit *iommu)
{
struct iommu_entry *entry, *tmp;
IOMMU_LIST_LOCK();
LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) {
if (entry->iommu == iommu) {
LIST_REMOVE(entry, next);
free(entry, M_IOMMU);
}
}
IOMMU_LIST_UNLOCK();
iommu_fini_busdma(iommu);
mtx_destroy(&iommu->lock);
return (0);
}
struct iommu_unit *
iommu_find(device_t dev, bool verbose)
{
struct iommu_entry *entry;
struct iommu_unit *iommu;
int error;
IOMMU_LIST_LOCK();
LIST_FOREACH(entry, &iommu_list, next) {
iommu = entry->iommu;
error = IOMMU_FIND(iommu->dev, dev);
if (error == 0) {
IOMMU_LIST_UNLOCK();
return (entry->iommu);
}
}
IOMMU_LIST_UNLOCK();
return (NULL);
}
void
iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
bool cansleep __unused)
{
dprintf("%s\n", __func__);
iommu_domain_free_entry(entry, free);
}
static void
iommu_init(void)
{
sx_init(&iommu_sx, "IOMMU list");
}
SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);
diff --git a/sys/arm64/iommu/iommu_pmap.c b/sys/arm64/iommu/iommu_pmap.c
index d356a92c4d66..8a0ea641e231 100644
--- a/sys/arm64/iommu/iommu_pmap.c
+++ b/sys/arm64/iommu/iommu_pmap.c
@@ -1,863 +1,863 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2020-2021 Ruslan Bukin <br@bsdpad.com>
* Copyright (c) 2014-2021 Andrew Turner
* Copyright (c) 2014-2016 The FreeBSD Foundation
* All rights reserved.
*
* This work was supported by Innovate UK project 105694, "Digital Security
* by Design (DSbD) Technology Platform Prototype".
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
/*
* Manages physical address maps for ARM SMMUv3 and ARM Mali GPU.
*/
#include "opt_vm.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_pageout.h>
#include <vm/vm_radix.h>
#include <machine/machdep.h>
#include <arm64/iommu/iommu_pmap.h>
#include <arm64/iommu/iommu_pte.h>
#define IOMMU_PAGE_SIZE 4096
#define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx)
#define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx)
#define SMMU_PMAP_LOCK_ASSERT(pmap, type) \
mtx_assert(&(pmap)->sp_mtx, (type))
#define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
#define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
#define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t)))
#define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t)))
#define NUL0E IOMMU_L0_ENTRIES
#define NUL1E (NUL0E * NL1PG)
#define NUL2E (NUL1E * NL2PG)
#define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT))
#define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT))
#define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT)
#define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK)
#define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK)
#define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK)
#define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK)
static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex);
static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va,
vm_page_t m, struct spglist *free);
/*
* These load the old table data and store the new value.
* They need to be atomic as the System MMU may write to the table at
* the same time as the CPU.
*/
#define smmu_pmap_load(table) (*table)
#define smmu_pmap_clear(table) atomic_store_64(table, 0)
#define smmu_pmap_store(table, entry) atomic_store_64(table, entry)
/********************/
/* Inline functions */
/********************/
static __inline pd_entry_t *
smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va)
{
return (&pmap->sp_l0[smmu_l0_index(va)]);
}
static __inline pd_entry_t *
smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
{
pd_entry_t *l1;
l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK);
return (&l1[smmu_l1_index(va)]);
}
static __inline pd_entry_t *
smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *l0;
l0 = smmu_pmap_l0(pmap, va);
if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE)
return (NULL);
return (smmu_pmap_l0_to_l1(l0, va));
}
static __inline pd_entry_t *
smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va)
{
pd_entry_t l1, *l2p;
l1 = smmu_pmap_load(l1p);
/*
* The valid bit may be clear if pmap_update_entry() is concurrently
* modifying the entry, so for KVA only the entry type may be checked.
*/
KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0,
("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va));
KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va));
l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK);
return (&l2p[smmu_l2_index(va)]);
}
static __inline pd_entry_t *
smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *l1;
l1 = smmu_pmap_l1(pmap, va);
if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE)
return (NULL);
return (smmu_pmap_l1_to_l2(l1, va));
}
static __inline pt_entry_t *
smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va)
{
pd_entry_t l2;
pt_entry_t *l3p;
l2 = smmu_pmap_load(l2p);
/*
* The valid bit may be clear if pmap_update_entry() is concurrently
* modifying the entry, so for KVA only the entry type may be checked.
*/
KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0,
("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va));
KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE,
("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va));
l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK);
return (&l3p[smmu_l3_index(va)]);
}
/*
* Returns the lowest valid pde for a given virtual address.
* The next level may or may not point to a valid page or block.
*/
static __inline pd_entry_t *
smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level)
{
pd_entry_t *l0, *l1, *l2, desc;
l0 = smmu_pmap_l0(pmap, va);
desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK;
if (desc != IOMMU_L0_TABLE) {
*level = -1;
return (NULL);
}
l1 = smmu_pmap_l0_to_l1(l0, va);
desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK;
if (desc != IOMMU_L1_TABLE) {
*level = 0;
return (l0);
}
l2 = smmu_pmap_l1_to_l2(l1, va);
desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK;
if (desc != IOMMU_L2_TABLE) {
*level = 1;
return (l1);
}
*level = 2;
return (l2);
}
/*
* Returns the lowest valid pte block or table entry for a given virtual
* address. If there are no valid entries return NULL and set the level to
* the first invalid level.
*/
static __inline pt_entry_t *
smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level)
{
pd_entry_t *l1, *l2, desc;
pt_entry_t *l3;
l1 = smmu_pmap_l1(pmap, va);
if (l1 == NULL) {
*level = 0;
return (NULL);
}
desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK;
if (desc == IOMMU_L1_BLOCK) {
*level = 1;
return (l1);
}
if (desc != IOMMU_L1_TABLE) {
*level = 1;
return (NULL);
}
l2 = smmu_pmap_l1_to_l2(l1, va);
desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK;
if (desc == IOMMU_L2_BLOCK) {
*level = 2;
return (l2);
}
if (desc != IOMMU_L2_TABLE) {
*level = 2;
return (NULL);
}
*level = 3;
l3 = smmu_pmap_l2_to_l3(l2, va);
if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE)
return (NULL);
return (l3);
}
static __inline int
smmu_pmap_l3_valid(pt_entry_t l3)
{
return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE);
}
CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK);
#ifdef INVARIANTS
static __inline void
smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)
{
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
pmap->sp_resident_count += count;
}
static __inline void
smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)
{
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT(pmap->sp_resident_count >= count,
("pmap %p resident count underflow %ld %d", pmap,
pmap->sp_resident_count, count));
pmap->sp_resident_count -= count;
}
#else
static __inline void
smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count)
{
}
static __inline void
smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count)
{
}
#endif
/***************************************************
* Page table page management routines.....
***************************************************/
/*
* Schedule the specified unused page table page to be freed. Specifically,
* add the page to the specified list of pages that will be released to the
* physical memory manager after the TLB has been updated.
*/
static __inline void
smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
boolean_t set_PG_ZERO)
{
if (set_PG_ZERO)
m->flags |= PG_ZERO;
else
m->flags &= ~PG_ZERO;
SLIST_INSERT_HEAD(free, m, plinks.s.ss);
}
/***************************************************
* Low level mapping routines.....
***************************************************/
/*
* Decrements a page table page's reference count, which is used to record the
* number of valid page table entries within the page. If the reference count
* drops to zero, then the page table page is unmapped. Returns TRUE if the
* page table page was unmapped and FALSE otherwise.
*/
static inline boolean_t
smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,
struct spglist *free)
{
--m->ref_count;
if (m->ref_count == 0) {
_smmu_pmap_unwire_l3(pmap, va, m, free);
return (TRUE);
} else
return (FALSE);
}
static void
_smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m,
struct spglist *free)
{
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*
* unmap the page table page
*/
if (m->pindex >= (NUL2E + NUL1E)) {
/* l1 page */
pd_entry_t *l0;
l0 = smmu_pmap_l0(pmap, va);
smmu_pmap_clear(l0);
} else if (m->pindex >= NUL2E) {
/* l2 page */
pd_entry_t *l1;
l1 = smmu_pmap_l1(pmap, va);
smmu_pmap_clear(l1);
} else {
/* l3 page */
pd_entry_t *l2;
l2 = smmu_pmap_l2(pmap, va);
smmu_pmap_clear(l2);
}
smmu_pmap_resident_count_dec(pmap, 1);
if (m->pindex < NUL2E) {
/* We just released an l3, unhold the matching l2 */
pd_entry_t *l1, tl1;
vm_page_t l2pg;
l1 = smmu_pmap_l1(pmap, va);
tl1 = smmu_pmap_load(l1);
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
smmu_pmap_unwire_l3(pmap, va, l2pg, free);
} else if (m->pindex < (NUL2E + NUL1E)) {
/* We just released an l2, unhold the matching l1 */
pd_entry_t *l0, tl0;
vm_page_t l1pg;
l0 = smmu_pmap_l0(pmap, va);
tl0 = smmu_pmap_load(l0);
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
smmu_pmap_unwire_l3(pmap, va, l1pg, free);
}
/*
* Put page on a list so that it is released after
* *ALL* TLB shootdown is done
*/
smmu_pmap_add_delayed_free_list(m, free, TRUE);
}
int
smmu_pmap_pinit(struct smmu_pmap *pmap)
{
vm_page_t m;
/*
* allocate the l0 page
*/
m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED |
VM_ALLOC_ZERO);
pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m);
pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr);
#ifdef INVARIANTS
pmap->sp_resident_count = 0;
#endif
mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF);
return (1);
}
/*
* This routine is called if the desired page table page does not exist.
*
* If page table page allocation fails, this routine may sleep before
* returning NULL. It sleeps only if a lock pointer was given.
*
* Note: If a page allocation fails at page table level two or three,
* one or two pages may be held during the wait, only to be released
* afterwards. This conservative approach is easily argued to avoid
* race conditions.
*/
static vm_page_t
_pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex)
{
vm_page_t m, l1pg, l2pg;
SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*
* Allocate a page table page.
*/
if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
/*
* Indicate the need to retry. While waiting, the page table
* page may have been allocated.
*/
return (NULL);
}
m->pindex = ptepindex;
/*
* Because of AArch64's weak memory consistency model, we must have a
* barrier here to ensure that the stores for zeroing "m", whether by
* pmap_zero_page() or an earlier function, are visible before adding
* "m" to the page table. Otherwise, a page table walk by another
* processor's MMU could see the mapping to "m" and a stale, non-zero
* PTE within "m".
*/
dmb(ishst);
/*
* Map the pagetable page into the process address space, if
* it isn't already there.
*/
if (ptepindex >= (NUL2E + NUL1E)) {
pd_entry_t *l0;
vm_pindex_t l0index;
l0index = ptepindex - (NUL2E + NUL1E);
l0 = &pmap->sp_l0[l0index];
smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE);
} else if (ptepindex >= NUL2E) {
vm_pindex_t l0index, l1index;
pd_entry_t *l0, *l1;
pd_entry_t tl0;
l1index = ptepindex - NUL2E;
l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT;
l0 = &pmap->sp_l0[l0index];
tl0 = smmu_pmap_load(l0);
if (tl0 == 0) {
/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index)
== NULL) {
vm_page_unwire_noq(m);
vm_page_free_zero(m);
return (NULL);
}
} else {
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
l1pg->ref_count++;
}
l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK);
l1 = &l1[ptepindex & Ln_ADDR_MASK];
smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE);
} else {
vm_pindex_t l0index, l1index;
pd_entry_t *l0, *l1, *l2;
pd_entry_t tl0, tl1;
l1index = ptepindex >> IOMMU_Ln_ENTRIES_SHIFT;
l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT;
l0 = &pmap->sp_l0[l0index];
tl0 = smmu_pmap_load(l0);
if (tl0 == 0) {
/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) {
vm_page_unwire_noq(m);
vm_page_free_zero(m);
return (NULL);
}
tl0 = smmu_pmap_load(l0);
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
l1 = &l1[l1index & Ln_ADDR_MASK];
} else {
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
l1 = &l1[l1index & Ln_ADDR_MASK];
tl1 = smmu_pmap_load(l1);
if (tl1 == 0) {
/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + l1index)
== NULL) {
vm_page_unwire_noq(m);
vm_page_free_zero(m);
return (NULL);
}
} else {
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
l2pg->ref_count++;
}
}
l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK);
l2 = &l2[ptepindex & Ln_ADDR_MASK];
smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE);
}
smmu_pmap_resident_count_inc(pmap, 1);
return (m);
}
/***************************************************
* Pmap allocation/deallocation routines.
***************************************************/
/*
* Release any resources held by the given physical map.
* Called when a pmap initialized by pmap_pinit is being released.
* Should only be called if the map contains no valid mappings.
*/
void
smmu_pmap_release(struct smmu_pmap *pmap)
{
vm_page_t m;
KASSERT(pmap->sp_resident_count == 0,
("pmap_release: pmap resident count %ld != 0",
pmap->sp_resident_count));
m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr);
vm_page_unwire_noq(m);
vm_page_free_zero(m);
mtx_destroy(&pmap->sp_mtx);
}
/***************************************************
* page management routines.
***************************************************/
/*
* Add a single Mali GPU entry. This function does not sleep.
*/
int
pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,
vm_prot_t prot, u_int flags)
{
pd_entry_t *pde;
pt_entry_t new_l3;
pt_entry_t orig_l3 __diagused;
pt_entry_t *l3;
vm_page_t mpte;
pd_entry_t *l1p;
pd_entry_t *l2p;
int lvl;
int rv;
KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space"));
KASSERT((va & PAGE_MASK) == 0, ("va is misaligned"));
KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned"));
new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK);
if ((prot & VM_PROT_WRITE) != 0)
new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE);
if ((prot & VM_PROT_READ) != 0)
new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ);
if ((prot & VM_PROT_EXECUTE) == 0)
new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL);
CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa);
SMMU_PMAP_LOCK(pmap);
/*
* In the case that a page table page is not
* resident, we are creating it here.
*/
retry:
pde = smmu_pmap_pde(pmap, va, &lvl);
if (pde != NULL && lvl == 2) {
l3 = smmu_pmap_l2_to_l3(pde, va);
} else {
mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));
if (mpte == NULL) {
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
rv = KERN_RESOURCE_SHORTAGE;
goto out;
}
/*
* Ensure newly created l1, l2 are visible to GPU.
* l0 is already visible by similar call in panfrost driver.
* The cache entry for l3 handled below.
*/
l1p = smmu_pmap_l1(pmap, va);
l2p = smmu_pmap_l2(pmap, va);
- cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t));
- cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t));
+ cpu_dcache_wb_range(l1p, sizeof(pd_entry_t));
+ cpu_dcache_wb_range(l2p, sizeof(pd_entry_t));
goto retry;
}
orig_l3 = smmu_pmap_load(l3);
KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid"));
/* New mapping */
smmu_pmap_store(l3, new_l3);
- cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t));
+ cpu_dcache_wb_range(l3, sizeof(pt_entry_t));
smmu_pmap_resident_count_inc(pmap, 1);
dsb(ishst);
rv = KERN_SUCCESS;
out:
SMMU_PMAP_UNLOCK(pmap);
return (rv);
}
/*
* Remove a single Mali GPU entry.
*/
int
pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *pde;
pt_entry_t *pte;
int lvl;
int rc;
KASSERT((va & PAGE_MASK) == 0, ("va is misaligned"));
SMMU_PMAP_LOCK(pmap);
pde = smmu_pmap_pde(pmap, va, &lvl);
if (pde == NULL || lvl != 2) {
rc = KERN_FAILURE;
goto out;
}
pte = smmu_pmap_l2_to_l3(pde, va);
smmu_pmap_resident_count_dec(pmap, 1);
smmu_pmap_clear(pte);
- cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t));
+ cpu_dcache_wb_range(pte, sizeof(pt_entry_t));
rc = KERN_SUCCESS;
out:
SMMU_PMAP_UNLOCK(pmap);
return (rc);
}
/*
* Add a single SMMU entry. This function does not sleep.
*/
int
smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,
vm_prot_t prot, u_int flags)
{
pd_entry_t *pde;
pt_entry_t new_l3;
pt_entry_t orig_l3 __diagused;
pt_entry_t *l3;
vm_page_t mpte;
int lvl;
int rv;
KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space"));
va = trunc_page(va);
new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT |
ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE);
if ((prot & VM_PROT_WRITE) == 0)
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO);
new_l3 |= ATTR_S1_XN; /* Execute never. */
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER);
new_l3 |= ATTR_S1_nG; /* Non global. */
CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa);
SMMU_PMAP_LOCK(pmap);
/*
* In the case that a page table page is not
* resident, we are creating it here.
*/
retry:
pde = smmu_pmap_pde(pmap, va, &lvl);
if (pde != NULL && lvl == 2) {
l3 = smmu_pmap_l2_to_l3(pde, va);
} else {
mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va));
if (mpte == NULL) {
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
rv = KERN_RESOURCE_SHORTAGE;
goto out;
}
goto retry;
}
orig_l3 = smmu_pmap_load(l3);
KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid"));
/* New mapping */
smmu_pmap_store(l3, new_l3);
smmu_pmap_resident_count_inc(pmap, 1);
dsb(ishst);
rv = KERN_SUCCESS;
out:
SMMU_PMAP_UNLOCK(pmap);
return (rv);
}
/*
* Remove a single SMMU entry.
*/
int
smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va)
{
pt_entry_t *pte;
int lvl;
int rc;
SMMU_PMAP_LOCK(pmap);
pte = smmu_pmap_pte(pmap, va, &lvl);
KASSERT(lvl == 3,
("Invalid SMMU pagetable level: %d != 3", lvl));
if (pte != NULL) {
smmu_pmap_resident_count_dec(pmap, 1);
smmu_pmap_clear(pte);
rc = KERN_SUCCESS;
} else
rc = KERN_FAILURE;
SMMU_PMAP_UNLOCK(pmap);
return (rc);
}
/*
* Remove all the allocated L1, L2 pages from SMMU pmap.
* All the L3 entires must be cleared in advance, otherwise
* this function panics.
*/
void
smmu_pmap_remove_pages(struct smmu_pmap *pmap)
{
pd_entry_t l0e, *l1, l1e, *l2, l2e;
pt_entry_t *l3, l3e;
vm_page_t m, m0, m1;
vm_paddr_t pa;
vm_paddr_t pa0;
vm_paddr_t pa1;
int i, j, k, l;
SMMU_PMAP_LOCK(pmap);
for (i = 0; i < IOMMU_L0_ENTRIES; i++) {
l0e = pmap->sp_l0[i];
if ((l0e & ATTR_DESCR_VALID) == 0) {
continue;
}
pa0 = l0e & ~ATTR_MASK;
m0 = PHYS_TO_VM_PAGE(pa0);
l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0);
for (j = 0; j < IOMMU_Ln_ENTRIES; j++) {
l1e = l1[j];
if ((l1e & ATTR_DESCR_VALID) == 0) {
continue;
}
if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) {
continue;
}
pa1 = l1e & ~ATTR_MASK;
m1 = PHYS_TO_VM_PAGE(pa1);
l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1);
for (k = 0; k < IOMMU_Ln_ENTRIES; k++) {
l2e = l2[k];
if ((l2e & ATTR_DESCR_VALID) == 0) {
continue;
}
pa = l2e & ~ATTR_MASK;
m = PHYS_TO_VM_PAGE(pa);
l3 = (pt_entry_t *)PHYS_TO_DMAP(pa);
for (l = 0; l < IOMMU_Ln_ENTRIES; l++) {
l3e = l3[l];
if ((l3e & ATTR_DESCR_VALID) == 0)
continue;
panic(
"%s: l3e found (indexes %d %d %d %d)",
__func__, i, j, k, l);
}
vm_page_unwire_noq(m1);
vm_page_unwire_noq(m);
smmu_pmap_resident_count_dec(pmap, 1);
vm_page_free(m);
smmu_pmap_clear(&l2[k]);
}
vm_page_unwire_noq(m0);
smmu_pmap_resident_count_dec(pmap, 1);
vm_page_free(m1);
smmu_pmap_clear(&l1[j]);
}
smmu_pmap_resident_count_dec(pmap, 1);
vm_page_free(m0);
smmu_pmap_clear(&pmap->sp_l0[i]);
}
KASSERT(pmap->sp_resident_count == 0,
("Invalid resident count %jd", pmap->sp_resident_count));
SMMU_PMAP_UNLOCK(pmap);
}