Page MenuHomeFreeBSD
Files F164665524

D58561.diff
No OneTemporary
Actions

D58561.diff
View Options

diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist
--- a/etc/mtree/BSD.include.dist
+++ b/etc/mtree/BSD.include.dist
@@ -215,6 +215,8 @@
..
firewire
..
+ gasket
+ ..
hid
..
hwpmc
diff --git a/include/Makefile b/include/Makefile
--- a/include/Makefile
+++ b/include/Makefile
@@ -183,6 +183,10 @@
OPENCRYPTO= cryptodev.h
OPENCRYPTODIR= ${INCLUDEDIR}/crypto
+.PATH: ${SRCTOP}/sys/dev/gasket
+GASKET= gasket_ioctl.h
+GASKETDIR= ${INCLUDEDIR}/dev/gasket
+
.PATH: ${SRCTOP}/sys/dev/nvme
NVME= nvme.h
NVMEDIR= ${INCLUDEDIR}/dev/nvme
@@ -262,6 +266,7 @@
CRYPTO \
EVDEV \
FS9660 \
+ GASKET \
HID \
HYPERV \
OPENCRYPTO \
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -191,6 +191,7 @@
fwohci.4 \
fxp.4 \
gdb.4 \
+ gasket.4 \
gem.4 \
genet.4 \
genetlink.4 \
diff --git a/share/man/man4/gasket.4 b/share/man/man4/gasket.4
new file mode 100644
--- /dev/null
+++ b/share/man/man4/gasket.4
@@ -0,0 +1,53 @@
+.\"
+.\" Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+.\"
+.\" SPDX-License-Identifier: BSD-2-Clause
+.\"
+.Dd July 30, 2026
+.Dt GASKET 4
+.Os
+.Sh NAME
+.Nm gasket
+.Nd Google Edge TPU accelerator framework
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following line in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device gasket"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+framework provides the character device, the mapping policy and the on-chip
+page table shared by Google accelerator ASICs.
+It does not attach to hardware; a chip driver such as
+.Xr apex 4
+describes its registers and implements the
+.Nm
+kobj interface.
+.Pp
+Only the register ranges a chip driver declares mappable are reachable from
+user space; the rest of the window, including the page table, stays private.
+.Sh SYSCTL VARIABLES
+.Bl -tag -width indent
+.It Va hw.gasket.max_wired_pages
+Maximum pages one accelerator may have mapped, 131072 by default.
+These are not charged against
+.Xr getrlimit 2
+.Dv RLIMIT_MEMLOCK ,
+so this is the only bound on what a consumer can pin.
+.El
+.Sh SEE ALSO
+.Xr apex 4 ,
+.Xr pci 4
+.Sh HISTORY
+The
+.Nm
+framework first appeared in
+.Fx 16.0 .
+The name comes from the framework of the same name that Google distributed
+for Linux, where it stands for Google ASIC Software, Kernel Extensions, and
+Tools.
+.Sh AUTHORS
+.An Abdelkader Boudih Aq Mt freebsd@seuros.com
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -65,6 +65,11 @@
device thunderbolt
options THUNDERBOLT_DEBUG
+#
+# Framework for Google accelerator ASICs; see gasket(4).
+#
+device gasket
+
#
# Microsemi smartpqi controllers.
# These controllers have a SCSI-like interface, and require the
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -112,6 +112,9 @@
crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl
crypto/openssl/ossl_aes_gcm.c optional ossl
dev/amdgpio/amdgpio.c optional amdgpio
+dev/gasket/gasket_core.c optional gasket
+dev/gasket/gasket_page_table.c optional gasket
+dev/gasket/gasket_if.m optional gasket
dev/apple_bce/apple_bce.c optional apple_bce pci
dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci
dev/apple_bce/apple_bce_queue.c optional apple_bce pci
diff --git a/sys/dev/gasket/gasket.h b/sys/dev/gasket/gasket.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _DEV_GASKET_GASKET_H_
+#define _DEV_GASKET_GASKET_H_
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+
+#include <machine/bus.h>
+
+#define GASKET_STATUS_DEAD 0
+#define GASKET_STATUS_ALIVE 1
+
+struct gasket_region {
+ bus_size_t gr_offset;
+ bus_size_t gr_size;
+};
+
+struct gasket_page_table_config {
+ u_int gpt_id;
+ u_int gpt_total_entries;
+ bus_size_t gpt_base_reg;
+ bus_size_t gpt_extended_reg;
+ u_int gpt_extended_bit;
+};
+
+/* Apex packs up to four vector control fields per register. */
+enum gasket_interrupt_packing {
+ GASKET_INTR_UNPACKED = 0,
+ GASKET_INTR_PACK_0,
+ GASKET_INTR_PACK_1,
+ GASKET_INTR_PACK_2,
+ GASKET_INTR_PACK_3,
+};
+
+struct gasket_interrupt_desc {
+ u_int gid_index;
+ bus_size_t gid_reg;
+ enum gasket_interrupt_packing gid_packing;
+};
+
+struct gasket_desc {
+ const char *gd_name;
+ struct resource *gd_csr;
+
+ const struct gasket_region *gd_regions;
+ u_int gd_nregions;
+
+ const struct gasket_page_table_config *gd_page_tables;
+ u_int gd_npage_tables;
+
+ const struct gasket_interrupt_desc *gd_interrupts;
+ u_int gd_ninterrupts;
+ u_int gd_interrupt_pack_width;
+
+ bus_size_t gd_coherent_base;
+ bus_size_t gd_coherent_size;
+};
+
+struct gasket_softc;
+
+int gasket_attach(device_t dev, const struct gasket_desc *desc,
+ struct gasket_softc **scp);
+
+int gasket_suspend_begin(struct gasket_softc *sc);
+void gasket_resume(struct gasket_softc *sc);
+
+/* Two phase: do not call finish() until no handler can reach the softc. */
+int gasket_detach_begin(struct gasket_softc *sc);
+void gasket_detach_abort(struct gasket_softc *sc);
+void gasket_detach_finish(struct gasket_softc *sc);
+
+bool gasket_is_open(struct gasket_softc *sc);
+
+/* Safe to call from an interrupt handler. */
+void gasket_interrupt_notify(struct gasket_softc *sc, u_int index);
+
+/* False when no buffer is allocated; zero is a legal bus address. */
+bool gasket_coherent_bus_addr(struct gasket_softc *sc, bus_addr_t *addrp);
+
+struct gasket_page_table *gasket_page_table_get(struct gasket_softc *sc,
+ u_int index);
+
+/*
+ * A page table entry carries its valid bit in bit 0, so a split 64 bit write
+ * publishes a valid entry whose upper address bits are still zero.
+ */
+static inline uint32_t
+gasket_rd4(struct resource *csr, bus_size_t off)
+{
+ return (bus_read_4(csr, off));
+}
+
+static inline void
+gasket_wr4(struct resource *csr, bus_size_t off, uint32_t val)
+{
+ bus_write_4(csr, off, val);
+}
+
+static inline uint64_t
+gasket_rd8(struct resource *csr, bus_size_t off)
+{
+ return (bus_read_8(csr, off));
+}
+
+static inline void
+gasket_wr8(struct resource *csr, bus_size_t off, uint64_t val)
+{
+ bus_write_8(csr, off, val);
+}
+
+static inline void
+gasket_rmw4(struct resource *csr, bus_size_t off, uint32_t val, u_int width,
+ u_int shift)
+{
+ uint32_t mask, tmp;
+
+ mask = (uint32_t)(((1ULL << width) - 1) << shift);
+ tmp = bus_read_4(csr, off);
+ tmp = (tmp & ~mask) | ((val << shift) & mask);
+ bus_write_4(csr, off, tmp);
+}
+
+static inline void
+gasket_rmw8(struct resource *csr, bus_size_t off, uint64_t val, u_int width,
+ u_int shift)
+{
+ uint64_t mask, tmp;
+
+ mask = ((1ULL << width) - 1) << shift;
+ tmp = gasket_rd8(csr, off);
+ tmp = (tmp & ~mask) | ((val << shift) & mask);
+ gasket_wr8(csr, off, tmp);
+}
+
+int gasket_wait8(struct resource *csr, bus_size_t off, uint64_t mask,
+ uint64_t val, int retries, int delay_ms);
+
+#endif /* _DEV_GASKET_GASKET_H_ */
diff --git a/sys/dev/gasket/gasket_core.c b/sys/dev/gasket/gasket_core.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket_core.c
@@ -0,0 +1,1346 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/proc.h>
+#include <sys/rwlock.h>
+#include <sys/sx.h>
+#include <sys/vnode.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/vm_pager.h>
+
+#include <dev/gasket/gasket_ioctl.h>
+#include <dev/gasket/gasket.h>
+#include <dev/gasket/gasket_page_table.h>
+#include "gasket_if.h"
+
+static MALLOC_DEFINE(M_GASKET, "gasket", "Gasket accelerator framework");
+
+#define GASKET_EVENT_RING 256
+
+struct gasket_softc;
+
+/* Lock order: gs_lock -> gpt_lock, vm_map -> VM object; never together. */
+
+struct gasket_maphandle {
+ struct gasket_softc *gm_sc;
+ bool gm_coherent;
+ vm_paddr_t gm_bar_pa;
+ bus_size_t gm_base;
+ const vm_paddr_t *gm_pages;
+ u_int gm_npages;
+ uint64_t gm_gen;
+};
+
+struct gasket_softc {
+ device_t gs_dev;
+ const struct gasket_desc *gs_desc;
+ struct cdev *gs_cdev;
+ struct sx gs_lock;
+ int gs_opens;
+ bool gs_dying;
+
+ struct gasket_page_table **gs_page_tables;
+
+ bus_dma_tag_t gs_coherent_tag;
+ bus_dmamap_t gs_coherent_map;
+ void *gs_coherent_vaddr;
+ vm_paddr_t *gs_coherent_pages; /* CPU side */
+ bus_addr_t gs_coherent_paddr; /* device */
+ bus_size_t gs_coherent_size;
+ u_int gs_coherent_npages;
+ uint64_t gs_coherent_gen;
+
+ u_int gs_coherent_maps;
+ u_int gs_bar_maps;
+ u_int gs_coherent_pt;
+ bool gs_coherent_defer;
+ bool gs_suspended;
+
+ struct proc *gs_owner;
+ u_int gs_writers;
+ /* Set when a quiesce failed and the mappings had to be retained. */
+ bool gs_stuck;
+
+ struct mtx gs_evlock;
+ struct knlist gs_note;
+ struct gasket_interrupt_event *gs_ring;
+ uint64_t *gs_counts;
+ u_int gs_ev_head;
+ u_int gs_ev_tail;
+ uint32_t gs_ev_dropped;
+ bool gs_ev_waiting;
+};
+
+static int gasket_coherent_alloc(struct gasket_softc *sc, u_int index,
+ bus_size_t size);
+static const struct cdev_pager_ops gasket_pager_ops;
+static int gasket_coherent_free(struct gasket_softc *sc);
+static void gasket_map_release(struct gasket_maphandle *mh);
+static int gasket_map_buffer(struct gasket_softc *sc,
+ struct gasket_page_table *pt, vm_offset_t host_addr,
+ uint64_t dev_addr, size_t size, vm_prot_t prot);
+static int gasket_classify_and_hold(struct gasket_softc *sc,
+ vm_offset_t addr, vm_size_t size, vm_prot_t prot,
+ enum gasket_map_kind *kindp, vm_ooffset_t *coh_offp,
+ uint64_t *genp, vm_page_t *pages, u_int npages);
+
+static d_open_t gasket_cdev_open;
+static d_read_t gasket_cdev_read;
+static d_kqfilter_t gasket_cdev_kqfilter;
+static d_close_t gasket_cdev_close;
+static d_ioctl_t gasket_cdev_ioctl;
+static d_mmap_single_t gasket_cdev_mmap_single;
+
+static struct cdevsw gasket_cdevsw = {
+ .d_version = D_VERSION,
+ .d_open = gasket_cdev_open,
+ .d_read = gasket_cdev_read,
+ .d_kqfilter = gasket_cdev_kqfilter,
+ .d_close = gasket_cdev_close,
+ .d_ioctl = gasket_cdev_ioctl,
+ .d_mmap_single = gasket_cdev_mmap_single,
+ .d_flags = D_TRACKCLOSE,
+ .d_name = "gasket",
+};
+
+int
+gasket_wait8(struct resource *csr, bus_size_t off, uint64_t mask, uint64_t val,
+ int retries, int delay_ms)
+{
+ int i;
+
+ for (i = 0; i < retries; i++) {
+ if ((gasket_rd8(csr, off) & mask) == val)
+ return (0);
+ pause_sbt("gskwait", mstosbt(delay_ms), 0, C_HARDCLOCK);
+ }
+ return (ETIMEDOUT);
+}
+
+bool
+gasket_is_open(struct gasket_softc *sc)
+{
+ bool open;
+
+ sx_slock(&sc->gs_lock);
+ open = sc->gs_opens > 0;
+ sx_sunlock(&sc->gs_lock);
+
+ return (open);
+}
+
+int
+gasket_suspend_begin(struct gasket_softc *sc)
+{
+
+ if (sc == NULL)
+ return (0);
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_opens > 0 || sc->gs_coherent_maps != 0 ||
+ sc->gs_bar_maps != 0) {
+ sx_xunlock(&sc->gs_lock);
+ return (EBUSY);
+ }
+ sc->gs_suspended = true;
+ sx_xunlock(&sc->gs_lock);
+
+ return (0);
+}
+
+void
+gasket_resume(struct gasket_softc *sc)
+{
+
+ if (sc == NULL)
+ return;
+
+ sx_xlock(&sc->gs_lock);
+ sc->gs_suspended = false;
+ sx_xunlock(&sc->gs_lock);
+}
+
+static int
+gasket_cdev_open(struct cdev *cdev, int oflags, int devtype __unused,
+ struct thread *td)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+ bool writable = (oflags & FWRITE) != 0;
+ int error = 0;
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_dying) {
+ sx_xunlock(&sc->gs_lock);
+ return (ENXIO);
+ }
+ if (sc->gs_suspended) {
+ sx_xunlock(&sc->gs_lock);
+ return (EBUSY);
+ }
+ if (sc->gs_stuck) {
+ sx_xunlock(&sc->gs_lock);
+ return (ENXIO);
+ }
+ if (writable && sc->gs_writers != 0 &&
+ sc->gs_owner != td->td_proc) {
+ sx_xunlock(&sc->gs_lock);
+ return (EBUSY);
+ }
+
+ if (sc->gs_opens == 0) {
+ error = GASKET_RESET(sc->gs_dev);
+ if (error != 0) {
+ (void)GASKET_ENTER_RESET(sc->gs_dev);
+ } else {
+ error = GASKET_OPEN(sc->gs_dev);
+ if (error != 0) {
+ (void)GASKET_ENTER_RESET(sc->gs_dev);
+ }
+ }
+ }
+ if (error == 0) {
+ if (sc->gs_opens == 0) {
+ mtx_lock(&sc->gs_evlock);
+ sc->gs_ev_head = sc->gs_ev_tail = 0;
+ sc->gs_ev_dropped = 0;
+ mtx_unlock(&sc->gs_evlock);
+ }
+ sc->gs_coherent_defer = false;
+ if (writable) {
+ sc->gs_owner = td->td_proc;
+ sc->gs_writers++;
+ }
+ sc->gs_opens++;
+ }
+ sx_xunlock(&sc->gs_lock);
+
+ return (error);
+}
+
+static int
+gasket_cdev_close(struct cdev *cdev, int fflag, int devtype __unused,
+ struct thread *td __unused)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+ struct gasket_page_table *pt;
+ u_int i;
+ int error;
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_opens > 0)
+ sc->gs_opens--;
+ if ((fflag & FWRITE) != 0 && sc->gs_writers > 0 &&
+ --sc->gs_writers == 0)
+ sc->gs_owner = NULL;
+ if (sc->gs_opens == 0) {
+ if (!sc->gs_dying) {
+ GASKET_CLOSE(sc->gs_dev);
+ error = GASKET_QUIESCE(sc->gs_dev);
+ if (error != 0) {
+ device_printf(sc->gs_dev,
+ "device did not quiesce (%d), resetting\n",
+ error);
+
+ if (GASKET_RESET(sc->gs_dev) != 0)
+ device_printf(sc->gs_dev,
+ "reset also failed, asking the "
+ "engine anyway\n");
+ error = GASKET_QUIESCE(sc->gs_dev);
+ }
+ if (error != 0) {
+ device_printf(sc->gs_dev,
+ "device will not stop (%d), retaining its "
+ "mappings and refusing to detach\n", error);
+ sc->gs_stuck = true;
+ sx_xunlock(&sc->gs_lock);
+ return (0);
+ }
+ (void)GASKET_ENTER_RESET(sc->gs_dev);
+ }
+
+ for (i = 0; i < sc->gs_desc->gd_npage_tables; i++) {
+ pt = sc->gs_page_tables[i];
+ gasket_page_table_unmap_all(pt);
+ (void)gasket_page_table_partition(pt,
+ gasket_page_table_num_entries(pt));
+ }
+
+ if (gasket_coherent_free(sc) == EBUSY)
+ sc->gs_coherent_defer = true;
+ }
+ sx_xunlock(&sc->gs_lock);
+
+ return (0);
+}
+
+static int
+gasket_cdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
+ struct thread *td __unused)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+ u_int i;
+ int error;
+
+ switch (cmd) {
+ case GASKET_IOCTL_NUMBER_PAGE_TABLES:
+ case GASKET_IOCTL_PAGE_TABLE_SIZE:
+ case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE:
+ break;
+ default:
+ if ((fflag & FWRITE) == 0)
+ return (EPERM);
+ break;
+ }
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_dying) {
+ sx_xunlock(&sc->gs_lock);
+ return (ENXIO);
+ }
+
+ switch (cmd) {
+ case GASKET_IOCTL_PAGE_TABLE_SIZE:
+ case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: {
+ const struct gasket_page_table_ioctl *pti = (const void *)data;
+
+ if (pti->page_table_index > UINT_MAX) {
+ sx_xunlock(&sc->gs_lock);
+ return (EINVAL);
+ }
+ break;
+ }
+ case GASKET_IOCTL_PARTITION_PAGE_TABLE:
+ case GASKET_IOCTL_UNMAP_BUFFER: {
+ const struct gasket_page_table_ioctl *pti = (const void *)data;
+
+ if (pti->page_table_index > UINT_MAX ||
+ pti->size > UINT_MAX) {
+ sx_xunlock(&sc->gs_lock);
+ return (EINVAL);
+ }
+ break;
+ }
+ case GASKET_IOCTL_MAP_BUFFER: {
+ const struct gasket_page_table_ioctl *pti = (const void *)data;
+
+ if (pti->page_table_index > UINT_MAX ||
+ pti->size > UINT_MAX ||
+ pti->host_address > (uint64_t)VM_MAX_ADDRESS) {
+ sx_xunlock(&sc->gs_lock);
+ return (EINVAL);
+ }
+ break;
+ }
+ case GASKET_IOCTL_MAP_BUFFER_FLAGS: {
+ const struct gasket_page_table_ioctl_flags *ptf =
+ (const void *)data;
+
+ if (ptf->base.page_table_index > UINT_MAX ||
+ ptf->base.size > UINT_MAX ||
+ ptf->base.host_address > (uint64_t)VM_MAX_ADDRESS) {
+ sx_xunlock(&sc->gs_lock);
+ return (EINVAL);
+ }
+ break;
+ }
+ case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: {
+ const struct gasket_coherent_alloc_config_ioctl *cfg =
+ (const void *)data;
+
+ if (cfg->page_table_index >= sc->gs_desc->gd_npage_tables) {
+ sx_xunlock(&sc->gs_lock);
+ return (EINVAL);
+ }
+ break;
+ }
+ }
+
+ switch (cmd) {
+ case GASKET_IOCTL_RESET:
+ error = GASKET_QUIESCE(sc->gs_dev);
+ if (error != 0)
+ break;
+ for (i = 0; i < sc->gs_desc->gd_npage_tables; i++)
+ gasket_page_table_unmap_all(sc->gs_page_tables[i]);
+ error = GASKET_RESET(sc->gs_dev);
+ break;
+ case GASKET_IOCTL_NUMBER_PAGE_TABLES:
+ *(uint64_t *)data = sc->gs_desc->gd_npage_tables;
+ error = 0;
+ break;
+ case GASKET_IOCTL_PAGE_TABLE_SIZE:
+ case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: {
+ struct gasket_page_table_ioctl *pti = (void *)data;
+ struct gasket_page_table *pt;
+
+ pt = gasket_page_table_get(sc, pti->page_table_index);
+ if (pt == NULL) {
+ error = EINVAL;
+ break;
+ }
+ if (cmd == GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE)
+ pti->size = gasket_page_table_num_simple_entries(pt);
+ else
+ pti->size = gasket_page_table_num_entries(pt);
+ error = 0;
+ break;
+ }
+ case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS:
+ mtx_lock(&sc->gs_evlock);
+ memset(sc->gs_counts, 0, sc->gs_desc->gd_ninterrupts *
+ sizeof(*sc->gs_counts));
+ sc->gs_ev_head = sc->gs_ev_tail = 0;
+ sc->gs_ev_dropped = 0;
+ mtx_unlock(&sc->gs_evlock);
+ error = 0;
+ break;
+ case GASKET_IOCTL_PARTITION_PAGE_TABLE: {
+ struct gasket_page_table_ioctl *pti = (void *)data;
+ struct gasket_page_table *pt;
+
+ pt = gasket_page_table_get(sc, pti->page_table_index);
+ if (pt == NULL) {
+ error = EINVAL;
+ break;
+ }
+ error = gasket_page_table_partition(pt, pti->size);
+ break;
+ }
+ case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: {
+ struct gasket_coherent_alloc_config_ioctl *cfg = (void *)data;
+
+ if (cfg->enable != 0) {
+ error = gasket_coherent_alloc(sc,
+ (u_int)cfg->page_table_index, cfg->size);
+ if (error == 0) {
+ cfg->dma_address =
+ sc->gs_desc->gd_coherent_base;
+ }
+ } else {
+ error = gasket_coherent_free(sc);
+ if (error == 0)
+ cfg->dma_address = 0;
+ }
+ break;
+ }
+ case GASKET_IOCTL_MAP_BUFFER: {
+ struct gasket_page_table_ioctl *pti = (void *)data;
+ struct gasket_page_table *pt;
+
+ pt = gasket_page_table_get(sc, pti->page_table_index);
+ if (pt == NULL) {
+ error = EINVAL;
+ break;
+ }
+ error = gasket_map_buffer(sc, pt,
+ (vm_offset_t)pti->host_address, pti->device_address,
+ pti->size, VM_PROT_READ | VM_PROT_WRITE);
+ break;
+ }
+ case GASKET_IOCTL_MAP_BUFFER_FLAGS: {
+ struct gasket_page_table_ioctl_flags *ptf = (void *)data;
+ struct gasket_page_table *pt;
+ int prot;
+
+ pt = gasket_page_table_get(sc, ptf->base.page_table_index);
+ if (pt == NULL) {
+ error = EINVAL;
+ break;
+ }
+
+ switch (ptf->flags & GASKET_DMA_NONE) {
+ case GASKET_DMA_BIDIRECTIONAL:
+ prot = VM_PROT_READ | VM_PROT_WRITE;
+ break;
+ case GASKET_DMA_TO_DEVICE:
+ prot = VM_PROT_READ;
+ break;
+ case GASKET_DMA_FROM_DEVICE:
+ prot = VM_PROT_WRITE;
+ break;
+ default:
+ error = EINVAL;
+ goto done;
+ }
+
+ error = gasket_map_buffer(sc, pt,
+ (vm_offset_t)ptf->base.host_address,
+ ptf->base.device_address, ptf->base.size, prot);
+ break;
+ }
+ case GASKET_IOCTL_UNMAP_BUFFER: {
+ struct gasket_page_table_ioctl *pti = (void *)data;
+ struct gasket_page_table *pt;
+
+ pt = gasket_page_table_get(sc, pti->page_table_index);
+ if (pt == NULL) {
+ error = EINVAL;
+ break;
+ }
+ error = gasket_page_table_unmap(pt, pti->device_address,
+ pti->size);
+ break;
+ }
+ default:
+ error = GASKET_IOCTL(sc->gs_dev, cmd, data, fflag);
+ break;
+ }
+done:
+ sx_xunlock(&sc->gs_lock);
+
+ return (error);
+}
+
+void
+gasket_interrupt_notify(struct gasket_softc *sc, u_int index)
+{
+ struct gasket_interrupt_event *ev;
+ u_int next;
+
+ if (sc == NULL || index >= sc->gs_desc->gd_ninterrupts)
+ return;
+
+ mtx_lock(&sc->gs_evlock);
+ sc->gs_counts[index]++;
+
+ next = (sc->gs_ev_head + 1) % GASKET_EVENT_RING;
+ if (next == sc->gs_ev_tail) {
+ sc->gs_ev_dropped++;
+ } else {
+ ev = &sc->gs_ring[sc->gs_ev_head];
+ ev->gie_interrupt = index;
+ ev->gie_dropped = sc->gs_ev_dropped;
+ ev->gie_count = sc->gs_counts[index];
+ sc->gs_ev_head = next;
+ sc->gs_ev_dropped = 0;
+ }
+
+ if (sc->gs_ev_waiting) {
+ sc->gs_ev_waiting = false;
+ wakeup(&sc->gs_ring);
+ }
+ KNOTE_LOCKED(&sc->gs_note, 0);
+ mtx_unlock(&sc->gs_evlock);
+}
+
+bool
+gasket_coherent_bus_addr(struct gasket_softc *sc, bus_addr_t *addrp)
+{
+ bool present;
+
+ if (sc == NULL)
+ return (false);
+
+ sx_slock(&sc->gs_lock);
+ present = sc->gs_coherent_vaddr != NULL;
+ if (present)
+ *addrp = sc->gs_coherent_paddr;
+ sx_sunlock(&sc->gs_lock);
+
+ return (present);
+}
+
+struct gasket_page_table *
+gasket_page_table_get(struct gasket_softc *sc, u_int index)
+{
+
+ if (sc == NULL || index >= sc->gs_desc->gd_npage_tables)
+ return (NULL);
+ return (sc->gs_page_tables[index]);
+}
+
+static int
+gasket_cdev_read(struct cdev *cdev, struct uio *uio, int ioflag)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+ struct gasket_interrupt_event ev;
+ int error;
+
+ if (uio->uio_resid < (ssize_t)sizeof(ev))
+ return (EINVAL);
+
+ error = 0;
+ mtx_lock(&sc->gs_evlock);
+ while (sc->gs_ev_head == sc->gs_ev_tail) {
+ if (sc->gs_dying) {
+ mtx_unlock(&sc->gs_evlock);
+ return (ENXIO);
+ }
+ if ((ioflag & IO_NDELAY) != 0) {
+ mtx_unlock(&sc->gs_evlock);
+ return (EWOULDBLOCK);
+ }
+ sc->gs_ev_waiting = true;
+ error = msleep(&sc->gs_ring, &sc->gs_evlock, PCATCH, "gskev",
+ 0);
+ if (error != 0 && error != EWOULDBLOCK) {
+ mtx_unlock(&sc->gs_evlock);
+ return (error);
+ }
+ error = 0;
+ }
+
+ while (sc->gs_ev_head != sc->gs_ev_tail &&
+ uio->uio_resid >= (ssize_t)sizeof(ev)) {
+ ev = sc->gs_ring[sc->gs_ev_tail];
+ sc->gs_ev_tail = (sc->gs_ev_tail + 1) % GASKET_EVENT_RING;
+ mtx_unlock(&sc->gs_evlock);
+
+ error = uiomove(&ev, sizeof(ev), uio);
+
+ mtx_lock(&sc->gs_evlock);
+ if (error != 0)
+ break;
+ }
+ mtx_unlock(&sc->gs_evlock);
+
+ return (error);
+}
+
+static void
+gasket_kq_detach(struct knote *kn)
+{
+ struct gasket_softc *sc = kn->kn_hook;
+
+ knlist_remove(&sc->gs_note, kn, 0);
+}
+
+static int
+gasket_kq_event(struct knote *kn, long hint __unused)
+{
+ struct gasket_softc *sc = kn->kn_hook;
+ u_int pending;
+
+ pending = (sc->gs_ev_head + GASKET_EVENT_RING - sc->gs_ev_tail) %
+ GASKET_EVENT_RING;
+ kn->kn_data = (int64_t)pending * sizeof(struct gasket_interrupt_event);
+
+ if (sc->gs_dying) {
+ kn->kn_flags |= EV_EOF;
+ return (1);
+ }
+ return (pending != 0);
+}
+
+static const struct filterops gasket_read_filterops = {
+ .f_isfd = 1,
+ .f_detach = gasket_kq_detach,
+ .f_event = gasket_kq_event,
+};
+
+static int
+gasket_cdev_kqfilter(struct cdev *cdev, struct knote *kn)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+
+ if (kn->kn_filter != EVFILT_READ)
+ return (EINVAL);
+
+ kn->kn_fop = &gasket_read_filterops;
+ kn->kn_hook = sc;
+ knlist_add(&sc->gs_note, kn, 0);
+
+ return (0);
+}
+
+struct gasket_coherent_load {
+ bus_addr_t gcl_addr;
+ int gcl_error;
+};
+
+static void
+gasket_coherent_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+ struct gasket_coherent_load *load = arg;
+
+ if (error != 0) {
+ load->gcl_error = error;
+ return;
+ }
+ if (nsegs != 1) {
+ load->gcl_error = EFBIG;
+ return;
+ }
+ load->gcl_addr = segs[0].ds_addr;
+ load->gcl_error = 0;
+}
+
+static int
+gasket_coherent_free(struct gasket_softc *sc)
+{
+ struct gasket_page_table *pt;
+
+ sx_assert(&sc->gs_lock, SA_XLOCKED);
+
+ if (sc->gs_coherent_vaddr == NULL)
+ return (0);
+ if (sc->gs_coherent_maps != 0)
+ return (EBUSY);
+
+ pt = sc->gs_page_tables[sc->gs_coherent_pt];
+ gasket_page_table_unmap_coherent(pt);
+ gasket_page_table_set_coherent(pt, NULL, 0, 0);
+
+ if (sc->gs_coherent_vaddr != NULL) {
+ bus_dmamap_unload(sc->gs_coherent_tag, sc->gs_coherent_map);
+ bus_dmamem_free(sc->gs_coherent_tag, sc->gs_coherent_vaddr,
+ sc->gs_coherent_map);
+ sc->gs_coherent_vaddr = NULL;
+ sc->gs_coherent_paddr = 0;
+ sc->gs_coherent_size = 0;
+ free(sc->gs_coherent_pages, M_GASKET);
+ sc->gs_coherent_pages = NULL;
+ sc->gs_coherent_npages = 0;
+ }
+ if (sc->gs_coherent_tag != NULL) {
+ bus_dma_tag_destroy(sc->gs_coherent_tag);
+ sc->gs_coherent_tag = NULL;
+ }
+
+ return (0);
+}
+
+static int
+gasket_coherent_alloc(struct gasket_softc *sc, u_int index, bus_size_t size)
+{
+ struct gasket_coherent_load load;
+ bus_addr_t addr;
+ u_int i;
+ int error;
+
+ sx_assert(&sc->gs_lock, SA_XLOCKED);
+
+ if (size == 0 || size > sc->gs_desc->gd_coherent_size)
+ return (EINVAL);
+ if (sc->gs_coherent_vaddr != NULL)
+ return (EBUSY);
+
+ size = roundup2(size, PAGE_SIZE);
+
+ error = bus_dma_tag_create(bus_get_dma_tag(sc->gs_dev), PAGE_SIZE, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, size, 1, size,
+ 0, NULL, NULL, &sc->gs_coherent_tag);
+ if (error != 0)
+ return (ENOMEM);
+
+ error = bus_dmamem_alloc(sc->gs_coherent_tag, &sc->gs_coherent_vaddr,
+ BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,
+ &sc->gs_coherent_map);
+ if (error != 0) {
+ bus_dma_tag_destroy(sc->gs_coherent_tag);
+ sc->gs_coherent_tag = NULL;
+ return (ENOMEM);
+ }
+
+ /* Zero is a legal bus address, so failure is reported separately. */
+ load.gcl_addr = 0;
+ load.gcl_error = ENOMEM;
+ error = bus_dmamap_load(sc->gs_coherent_tag, sc->gs_coherent_map,
+ sc->gs_coherent_vaddr, size, gasket_coherent_cb, &load,
+ BUS_DMA_NOWAIT);
+ if (error == 0 && load.gcl_error != 0)
+ error = load.gcl_error;
+ if (error != 0) {
+ if (load.gcl_error == 0)
+ bus_dmamap_unload(sc->gs_coherent_tag,
+ sc->gs_coherent_map);
+ bus_dmamem_free(sc->gs_coherent_tag, sc->gs_coherent_vaddr,
+ sc->gs_coherent_map);
+ bus_dma_tag_destroy(sc->gs_coherent_tag);
+ sc->gs_coherent_tag = NULL;
+ sc->gs_coherent_vaddr = NULL;
+ return (ENOMEM);
+ }
+ addr = load.gcl_addr;
+
+ sc->gs_coherent_paddr = addr;
+ sc->gs_coherent_size = size;
+ sc->gs_coherent_npages = size / PAGE_SIZE;
+ sc->gs_coherent_gen++;
+ sc->gs_coherent_pages = mallocarray(sc->gs_coherent_npages,
+ sizeof(*sc->gs_coherent_pages), M_GASKET, M_WAITOK | M_ZERO);
+ for (i = 0; i < sc->gs_coherent_npages; i++) {
+ sc->gs_coherent_pages[i] = vtophys((char *)sc->gs_coherent_vaddr
+ +
+ (size_t)i * PAGE_SIZE);
+ }
+
+ sc->gs_coherent_pt = index;
+ gasket_page_table_set_coherent(sc->gs_page_tables[index],
+ sc->gs_coherent_pages, sc->gs_coherent_npages, addr);
+
+ return (0);
+}
+
+static bool
+gasket_region_contains(const struct gasket_desc *desc, vm_ooffset_t offset,
+ vm_size_t size)
+{
+ const struct gasket_region *gr;
+ u_int i;
+
+ for (i = 0; i < desc->gd_nregions; i++) {
+ gr = &desc->gd_regions[i];
+ if (offset >= gr->gr_offset &&
+ size <= gr->gr_size &&
+ offset - gr->gr_offset <= gr->gr_size - size)
+ return (true);
+ }
+ return (false);
+}
+
+/*
+ * Validated here, not in d_mmap(): the device pager keys its object on the
+ * cdev, so later mmap(2) calls reuse it and never reach d_mmap().
+ */
+static bool
+gasket_coherent_contains(struct gasket_softc *sc, vm_ooffset_t offset,
+ vm_size_t size)
+{
+ const bus_size_t base = sc->gs_desc->gd_coherent_base;
+
+ if (sc->gs_coherent_vaddr == NULL)
+ return (false);
+ return (offset >= base && size <= sc->gs_coherent_size &&
+ (bus_size_t)offset - base <= sc->gs_coherent_size - size);
+}
+
+static int
+gasket_cdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset,
+ vm_size_t size, struct vm_object **objp, int nprot)
+{
+ struct gasket_softc *sc = cdev->si_drv1;
+ struct gasket_maphandle *mh;
+ vm_object_t obj;
+ bool coherent;
+
+ if (size == 0 || (*offset & PAGE_MASK) != 0)
+ return (EINVAL);
+
+ mh = malloc(sizeof(*mh), M_GASKET, M_WAITOK | M_ZERO);
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_dying) {
+ sx_xunlock(&sc->gs_lock);
+ free(mh, M_GASKET);
+ return (ENXIO);
+ }
+ mh->gm_sc = sc;
+ if (gasket_region_contains(sc->gs_desc, *offset, size)) {
+ coherent = false;
+ mh->gm_coherent = false;
+ mh->gm_bar_pa = (vm_paddr_t)rman_get_start(sc->gs_desc->gd_csr);
+ sc->gs_bar_maps++;
+ } else if (gasket_coherent_contains(sc, *offset, size)) {
+ coherent = true;
+ mh->gm_coherent = true;
+ mh->gm_base = sc->gs_desc->gd_coherent_base;
+ mh->gm_pages = sc->gs_coherent_pages;
+ mh->gm_npages = sc->gs_coherent_npages;
+ mh->gm_gen = sc->gs_coherent_gen;
+ sc->gs_coherent_maps++;
+ } else {
+ sx_xunlock(&sc->gs_lock);
+ free(mh, M_GASKET);
+ return (EINVAL);
+ }
+ sx_xunlock(&sc->gs_lock);
+
+ obj = cdev_pager_allocate(mh, OBJT_DEVICE, &gasket_pager_ops, size,
+ nprot, *offset, curthread->td_ucred);
+ if (obj == NULL) {
+ gasket_map_release(mh);
+ return (ENXIO);
+ }
+
+ /* Control registers must not be cached; coherent memory stays WB. */
+ if (!coherent) {
+ VM_OBJECT_WLOCK(obj);
+ vm_object_set_memattr(obj, VM_MEMATTR_UNCACHEABLE);
+ VM_OBJECT_WUNLOCK(obj);
+ }
+
+ *objp = obj;
+
+ return (0);
+}
+
+static bool
+gasket_paddr_for_offset(const struct gasket_maphandle *mh, vm_ooffset_t offset,
+ vm_paddr_t *paddr)
+{
+ u_int idx;
+
+ if (!mh->gm_coherent) {
+ if (!gasket_region_contains(mh->gm_sc->gs_desc, offset,
+ PAGE_SIZE))
+ return (false);
+ *paddr = mh->gm_bar_pa + offset;
+ return (true);
+ }
+
+ if (mh->gm_pages == NULL || offset < mh->gm_base)
+ return (false);
+ idx = (u_int)((offset - mh->gm_base) / PAGE_SIZE);
+ if (idx >= mh->gm_npages)
+ return (false);
+ *paddr = mh->gm_pages[idx];
+
+ return (true);
+}
+
+static void
+gasket_map_release(struct gasket_maphandle *mh)
+{
+ struct gasket_softc *sc = mh->gm_sc;
+ bool finish_free;
+
+ sx_xlock(&sc->gs_lock);
+ if (mh->gm_coherent) {
+ KASSERT(sc->gs_coherent_maps != 0,
+ ("gasket: coherent mapping count underflow"));
+ sc->gs_coherent_maps--;
+ finish_free = sc->gs_coherent_maps == 0 &&
+ sc->gs_coherent_defer;
+ } else {
+ KASSERT(sc->gs_bar_maps != 0,
+ ("gasket: register mapping count underflow"));
+ sc->gs_bar_maps--;
+ finish_free = false;
+ }
+ if (finish_free) {
+ sc->gs_coherent_defer = false;
+ (void)gasket_coherent_free(sc);
+ }
+ sx_xunlock(&sc->gs_lock);
+
+ free(mh, M_GASKET);
+}
+
+static int
+gasket_pager_ctor(void *handle __unused, vm_ooffset_t size __unused,
+ vm_prot_t prot __unused, vm_ooffset_t foff __unused,
+ struct ucred *cred __unused, u_short *color)
+{
+
+ *color = 0;
+
+ return (0);
+}
+
+static void
+gasket_pager_dtor(void *handle)
+{
+
+ gasket_map_release(handle);
+}
+
+static int
+gasket_pager_fault(vm_object_t obj, vm_ooffset_t offset, int prot __unused,
+ vm_page_t *mres)
+{
+ const struct gasket_maphandle *mh = obj->handle;
+ vm_paddr_t paddr;
+ vm_page_t page;
+
+ if (!gasket_paddr_for_offset(mh, offset, &paddr))
+ return (VM_PAGER_FAIL);
+
+ if (((*mres)->flags & PG_FICTITIOUS) != 0) {
+ vm_page_updatefake(*mres, paddr, obj->memattr);
+ vm_page_valid(*mres);
+ return (VM_PAGER_OK);
+ }
+
+ VM_OBJECT_WUNLOCK(obj);
+ page = vm_page_getfake(paddr, obj->memattr);
+ VM_OBJECT_WLOCK(obj);
+ vm_page_replace(page, obj, (*mres)->pindex, *mres);
+ *mres = page;
+ vm_page_valid(page);
+
+ return (VM_PAGER_OK);
+}
+
+static const struct cdev_pager_ops gasket_pager_ops = {
+ .cdev_pg_ctor = gasket_pager_ctor,
+ .cdev_pg_dtor = gasket_pager_dtor,
+ .cdev_pg_fault = gasket_pager_fault,
+};
+
+static bool
+gasket_object_is_coherent(struct gasket_softc *sc, vm_object_t obj,
+ uint64_t *genp)
+{
+ const struct gasket_maphandle *mh;
+
+ if (obj == NULL || obj->type != OBJT_DEVICE ||
+ obj->un_pager.devp.ops != &gasket_pager_ops)
+ return (false);
+
+ mh = obj->un_pager.devp.handle;
+ if (mh == NULL || mh->gm_sc != sc || !mh->gm_coherent)
+ return (false);
+ if (genp != NULL)
+ *genp = mh->gm_gen;
+
+ return (true);
+}
+
+/*
+ * Classify from the address space, not from the pages a fault returns: a fake
+ * page is freed by its own pager regardless of wiring.
+ */
+static int
+gasket_classify_range_locked(struct gasket_softc *sc, vm_offset_t addr,
+ vm_size_t size, enum gasket_map_kind *kindp, vm_ooffset_t *coh_offp,
+ uint64_t *genp)
+{
+ vm_map_t map = &curproc->p_vmspace->vm_map;
+ vm_map_entry_t entry;
+ vm_object_t obj;
+ vm_offset_t end, va;
+ int error = 0;
+
+ end = addr + size;
+ *kindp = GASKET_MAP_NONE;
+ *coh_offp = 0;
+
+ if (!vm_map_lookup_entry(map, addr, &entry))
+ return (EFAULT);
+
+ /* A submap entry's union holds a vm_map, not a vm_object. */
+ if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0)
+ return (EINVAL);
+
+ if (gasket_object_is_coherent(sc, entry->object.vm_object, genp)) {
+ if (end > entry->end)
+ return (EINVAL);
+ *kindp = GASKET_MAP_COHERENT;
+ *coh_offp = entry->offset + (addr - entry->start) -
+ sc->gs_desc->gd_coherent_base;
+ return (0);
+ }
+
+ for (va = addr; va < end; va = entry->end) {
+ if (!vm_map_lookup_entry(map, va, &entry)) {
+ error = EFAULT;
+ break;
+ }
+ if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) {
+ error = EINVAL;
+ break;
+ }
+ obj = entry->object.vm_object;
+ if (gasket_object_is_coherent(sc, obj, NULL)) {
+ error = EINVAL;
+ break;
+ }
+ if (obj != NULL && (obj->type == OBJT_DEVICE ||
+ obj->type == OBJT_MGTDEVICE || obj->type == OBJT_PHYS ||
+ obj->type == OBJT_SG)) {
+ error = EINVAL;
+ break;
+ }
+ }
+ if (error == 0)
+ *kindp = GASKET_MAP_HOST;
+
+ return (error);
+}
+
+static int
+gasket_map_buffer(struct gasket_softc *sc, struct gasket_page_table *pt,
+ vm_offset_t host_addr, uint64_t dev_addr, size_t size, vm_prot_t prot)
+{
+ enum gasket_map_kind kind;
+ vm_ooffset_t coh_off;
+ vm_page_t *pages;
+ uint64_t gen = 0;
+ u_int i, npages;
+ int error;
+
+ if (size == 0 || (size & PAGE_MASK) != 0 ||
+ (host_addr & PAGE_MASK) != 0 || host_addr + size < host_addr)
+ return (EINVAL);
+ npages = (u_int)(size >> PAGE_SHIFT);
+ if ((size >> PAGE_SHIFT) != npages)
+ return (EINVAL);
+
+ if (npages > gasket_page_table_max_pages(pt))
+ return (EINVAL);
+
+ pages = mallocarray(npages, sizeof(*pages), M_GASKET,
+ M_WAITOK | M_ZERO);
+
+ sx_xunlock(&sc->gs_lock);
+ error = gasket_classify_and_hold(sc, host_addr, size, prot, &kind,
+ &coh_off, &gen, pages, npages);
+ sx_xlock(&sc->gs_lock);
+
+ if (error == 0) {
+ if (sc->gs_dying || sc->gs_stuck)
+ error = ENXIO;
+ else if (kind == GASKET_MAP_COHERENT &&
+ (sc->gs_coherent_vaddr == NULL ||
+ gen != sc->gs_coherent_gen)) {
+ error = ENXIO;
+ } else
+ error = gasket_page_table_map(pt, host_addr, dev_addr,
+ size, prot, kind, coh_off, pages);
+
+ for (i = 0; i < npages; i++) {
+ if (pages[i] != NULL)
+ vm_page_unhold_pages(&pages[i], 1);
+ }
+ }
+
+ free(pages, M_GASKET);
+
+ return (error);
+}
+
+#define GASKET_HOLD_ATTEMPTS 8
+
+static int
+gasket_classify_and_hold(struct gasket_softc *sc, vm_offset_t addr,
+ vm_size_t size, vm_prot_t prot, enum gasket_map_kind *kindp,
+ vm_ooffset_t *coh_offp, uint64_t *genp, vm_page_t *pages, u_int npages)
+{
+ vm_map_t map = &curproc->p_vmspace->vm_map;
+ pmap_t pmap = vmspace_pmap(curproc->p_vmspace);
+ vm_prot_t fault_prot;
+ vm_offset_t va;
+ u_int i;
+ int attempt, error;
+
+ fault_prot = prot;
+ if ((fault_prot & VM_PROT_WRITE) != 0)
+ fault_prot |= VM_PROT_READ;
+
+ for (attempt = 0; attempt < GASKET_HOLD_ATTEMPTS; attempt++) {
+ for (va = addr; va < addr + size; va += PAGE_SIZE) {
+ if (vm_fault(map, va, fault_prot, VM_FAULT_NORMAL,
+ NULL) != KERN_SUCCESS)
+ return (EFAULT);
+ }
+
+ vm_map_lock_read(map);
+ error = gasket_classify_range_locked(sc, addr, size, kindp,
+ coh_offp, genp);
+ if (error != 0) {
+ vm_map_unlock_read(map);
+ return (error);
+ }
+ if (*kindp == GASKET_MAP_COHERENT) {
+ vm_map_unlock_read(map);
+ return (0);
+ }
+
+ for (i = 0; i < npages; i++) {
+ pages[i] = pmap_extract_and_hold(pmap,
+ addr + (vm_offset_t)i * PAGE_SIZE, fault_prot);
+ if (pages[i] == NULL)
+ break;
+ if ((pages[i]->oflags & VPO_UNMANAGED) != 0 ||
+ (pages[i]->flags & PG_FICTITIOUS) != 0) {
+ /* Classified, but not ours to keep. */
+ vm_page_unhold_pages(pages, i + 1);
+ memset(pages, 0, npages * sizeof(*pages));
+ vm_map_unlock_read(map);
+ return (EINVAL);
+ }
+
+ /* DMA leaves no CPU dirty bit behind. */
+ if ((prot & VM_PROT_WRITE) != 0 &&
+ pages[i]->dirty != VM_PAGE_BITS_ALL)
+ vm_page_dirty(pages[i]);
+ }
+ vm_map_unlock_read(map);
+
+ if (i == npages)
+ return (0);
+
+ vm_page_unhold_pages(pages, i);
+ memset(pages, 0, npages * sizeof(*pages));
+ }
+
+ return (EFAULT);
+}
+
+int
+gasket_attach(device_t dev, const struct gasket_desc *desc,
+ struct gasket_softc **scp)
+{
+ struct make_dev_args mda;
+ struct gasket_softc *sc;
+ u_int i;
+ int error;
+
+ KASSERT(desc->gd_csr != NULL, ("gasket: no control register window"));
+ KASSERT(desc->gd_name != NULL, ("gasket: no device name"));
+
+ sc = malloc(sizeof(*sc), M_GASKET, M_WAITOK | M_ZERO);
+ sc->gs_dev = dev;
+ sc->gs_desc = desc;
+ sx_init(&sc->gs_lock, "gasket");
+ mtx_init(&sc->gs_evlock, "gasket ev", NULL, MTX_DEF);
+
+ knlist_init_mtx(&sc->gs_note, &sc->gs_evlock);
+ sc->gs_ring = mallocarray(GASKET_EVENT_RING, sizeof(*sc->gs_ring),
+ M_GASKET, M_WAITOK | M_ZERO);
+ sc->gs_counts = mallocarray(MAX(desc->gd_ninterrupts, 1),
+ sizeof(*sc->gs_counts), M_GASKET, M_WAITOK | M_ZERO);
+
+ if (desc->gd_npage_tables != 0) {
+ sc->gs_page_tables = mallocarray(desc->gd_npage_tables,
+ sizeof(*sc->gs_page_tables), M_GASKET, M_WAITOK | M_ZERO);
+ for (i = 0; i < desc->gd_npage_tables; i++) {
+ error = gasket_page_table_init(dev, desc->gd_csr,
+ &desc->gd_page_tables[i], &sc->gs_page_tables[i]);
+ if (error != 0) {
+ device_printf(dev,
+ "cannot init page table %u: %d\n", i,
+ error);
+ goto fail;
+ }
+ }
+ }
+
+ make_dev_args_init(&mda);
+ mda.mda_devsw = &gasket_cdevsw;
+ mda.mda_unit = device_get_unit(dev);
+ mda.mda_uid = UID_ROOT;
+ mda.mda_gid = GID_WHEEL;
+ mda.mda_mode = 0660;
+ mda.mda_si_drv1 = sc;
+ error = make_dev_s(&mda, &sc->gs_cdev, "%s%d", desc->gd_name,
+ device_get_unit(dev));
+ if (error != 0) {
+ device_printf(dev, "cannot create device node: %d\n", error);
+ goto fail;
+ }
+
+ *scp = sc;
+ return (0);
+
+fail:
+ for (i = 0; i < desc->gd_npage_tables; i++) {
+ if (sc->gs_page_tables != NULL &&
+ sc->gs_page_tables[i] != NULL)
+ gasket_page_table_free(sc->gs_page_tables[i]);
+ }
+ free(sc->gs_page_tables, M_GASKET);
+ free(sc->gs_counts, M_GASKET);
+ free(sc->gs_ring, M_GASKET);
+ knlist_destroy(&sc->gs_note);
+ mtx_destroy(&sc->gs_evlock);
+ sx_destroy(&sc->gs_lock);
+ free(sc, M_GASKET);
+ return (error);
+}
+
+int
+gasket_detach_begin(struct gasket_softc *sc)
+{
+
+ if (sc == NULL)
+ return (0);
+
+ sx_xlock(&sc->gs_lock);
+ if (sc->gs_opens > 0 || sc->gs_coherent_maps != 0 ||
+ sc->gs_bar_maps != 0) {
+ sx_xunlock(&sc->gs_lock);
+ return (EBUSY);
+ }
+ if (sc->gs_stuck) {
+ sx_xunlock(&sc->gs_lock);
+ return (EBUSY);
+ }
+ sc->gs_dying = true;
+ sx_xunlock(&sc->gs_lock);
+
+ return (0);
+}
+
+void
+gasket_detach_abort(struct gasket_softc *sc)
+{
+
+ if (sc == NULL)
+ return;
+
+ sx_xlock(&sc->gs_lock);
+ sc->gs_dying = false;
+ sx_xunlock(&sc->gs_lock);
+}
+
+void
+gasket_detach_finish(struct gasket_softc *sc)
+{
+ u_int i;
+
+ if (sc == NULL)
+ return;
+
+ mtx_lock(&sc->gs_evlock);
+ sc->gs_ev_waiting = false;
+ wakeup(&sc->gs_ring);
+ KNOTE_LOCKED(&sc->gs_note, 0);
+ mtx_unlock(&sc->gs_evlock);
+
+ destroy_dev(sc->gs_cdev);
+
+ /* Not knlist_detach(): it frees a knlist embedded in the softc. */
+ knlist_clear(&sc->gs_note, 0);
+ knlist_destroy(&sc->gs_note);
+
+ sx_xlock(&sc->gs_lock);
+ gasket_coherent_free(sc);
+ sx_xunlock(&sc->gs_lock);
+
+ for (i = 0; i < sc->gs_desc->gd_npage_tables; i++)
+ gasket_page_table_free(sc->gs_page_tables[i]);
+ free(sc->gs_page_tables, M_GASKET);
+ free(sc->gs_counts, M_GASKET);
+ free(sc->gs_ring, M_GASKET);
+ mtx_destroy(&sc->gs_evlock);
+ sx_destroy(&sc->gs_lock);
+ free(sc, M_GASKET);
+}
+
+static int
+gasket_modevent(module_t mod __unused, int type, void *data __unused)
+{
+ switch (type) {
+ case MOD_LOAD:
+ case MOD_UNLOAD:
+ return (0);
+ default:
+ return (EOPNOTSUPP);
+ }
+}
+
+static moduledata_t gasket_mod = {
+ "gasket",
+ gasket_modevent,
+ NULL
+};
+
+DECLARE_MODULE(gasket, gasket_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
+MODULE_VERSION(gasket, 1);
diff --git a/sys/dev/gasket/gasket_if.m b/sys/dev/gasket/gasket_if.m
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket_if.m
@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+# Every method is called with the core's device lock held, from a context
+# that is allowed to sleep.
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+
+#include <dev/gasket/gasket.h>
+
+INTERFACE gasket;
+
+CODE {
+ static int
+ gasket_default_reset(device_t dev)
+ {
+ return (0);
+ }
+
+ static int
+ gasket_default_open(device_t dev)
+ {
+ return (0);
+ }
+
+ static void
+ gasket_default_close(device_t dev)
+ {
+ }
+
+ static int
+ gasket_default_ioctl(device_t dev, u_long cmd, caddr_t data, int fflag)
+ {
+ return (ENOTTY);
+ }
+
+ static int
+ gasket_default_status(device_t dev)
+ {
+ return (GASKET_STATUS_ALIVE);
+ }
+};
+
+#
+# Called on first open and from GASKET_IOCTL_RESET.
+#
+METHOD int reset {
+ device_t dev;
+} DEFAULT gasket_default_reset;
+
+#
+# Stop all DMA durably: a register window mapping outlives the descriptor it
+# was made through, so pausing the engine is not enough, the core must be
+# held in reset. The core releases no host pages if this fails.
+#
+METHOD int quiesce {
+ device_t dev;
+} DEFAULT gasket_default_reset;
+
+#
+# Power policy only; a driver may make this a no-op, so it is never relied
+# on for DMA safety.
+#
+METHOD int enter_reset {
+ device_t dev;
+} DEFAULT gasket_default_reset;
+
+METHOD int quit_reset {
+ device_t dev;
+} DEFAULT gasket_default_reset;
+
+METHOD int status {
+ device_t dev;
+} DEFAULT gasket_default_status;
+
+METHOD int open {
+ device_t dev;
+} DEFAULT gasket_default_open;
+
+METHOD void close {
+ device_t dev;
+} DEFAULT gasket_default_close;
+
+METHOD int ioctl {
+ device_t dev;
+ u_long cmd;
+ caddr_t data;
+ int fflag;
+} DEFAULT gasket_default_ioctl;
diff --git a/sys/dev/gasket/gasket_ioctl.h b/sys/dev/gasket/gasket_ioctl.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket_ioctl.h
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Structure layouts match Google's Linux gasket driver so a ported user space
+ * library can share them. The command numbers use native FreeBSD encoding.
+ */
+
+#ifndef _DEV_GASKET_GASKET_IOCTL_H_
+#define _DEV_GASKET_GASKET_IOCTL_H_
+
+#include <sys/ioccom.h>
+#include <sys/types.h>
+
+struct gasket_page_table_ioctl {
+ uint64_t page_table_index;
+ uint64_t size;
+ uint64_t host_address;
+ uint64_t device_address;
+};
+
+#define GASKET_PTE_INUSE 0x1
+#define GASKET_DMA_BIDIRECTIONAL (0 << 1)
+#define GASKET_DMA_TO_DEVICE (1 << 1)
+#define GASKET_DMA_FROM_DEVICE (2 << 1)
+#define GASKET_DMA_NONE (3 << 1)
+
+/*
+ * The padding is deliberate: without it this structure is 40 bytes where
+ * uint64_t aligns to 8 and 36 where it aligns to 4, and since the size is
+ * encoded into the command number, a 32 bit process would compute a different
+ * GASKET_IOCTL_MAP_BUFFER_FLAGS and get ENOTTY.
+ */
+struct gasket_page_table_ioctl_flags {
+ struct gasket_page_table_ioctl base;
+ uint32_t flags;
+ uint32_t _pad;
+};
+
+struct gasket_coherent_alloc_config_ioctl {
+ uint64_t page_table_index;
+ uint64_t enable;
+ uint64_t size;
+ uint64_t dma_address;
+};
+
+struct gasket_interrupt_event {
+ uint32_t gie_interrupt;
+ uint32_t gie_dropped; /* lost before this one */
+ uint64_t gie_count; /* cumulative for this interrupt */
+};
+
+/*
+ * Same macros and group as Google's shared gasket_ioctl.h, so a user space
+ * library including that header computes identical values. Only the encoding
+ * differs from Linux, and that falls out of <sys/ioccom.h>.
+ */
+#define GASKET_IOC_GROUP 0xDC
+
+#define GASKET_IOCTL_RESET _IO(GASKET_IOC_GROUP, 0)
+#define GASKET_IOCTL_NUMBER_PAGE_TABLES _IOR(GASKET_IOC_GROUP, 4, uint64_t)
+#define GASKET_IOCTL_PAGE_TABLE_SIZE \
+ _IOWR(GASKET_IOC_GROUP, 5, struct gasket_page_table_ioctl)
+#define GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE \
+ _IOWR(GASKET_IOC_GROUP, 6, struct gasket_page_table_ioctl)
+#define GASKET_IOCTL_PARTITION_PAGE_TABLE \
+ _IOW(GASKET_IOC_GROUP, 7, struct gasket_page_table_ioctl)
+#define GASKET_IOCTL_MAP_BUFFER \
+ _IOW(GASKET_IOC_GROUP, 8, struct gasket_page_table_ioctl)
+#define GASKET_IOCTL_UNMAP_BUFFER \
+ _IOW(GASKET_IOC_GROUP, 9, struct gasket_page_table_ioctl)
+#define GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS _IO(GASKET_IOC_GROUP, 10)
+#define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR \
+ _IOWR(GASKET_IOC_GROUP, 11, struct gasket_coherent_alloc_config_ioctl)
+#define GASKET_IOCTL_MAP_BUFFER_FLAGS \
+ _IOW(GASKET_IOC_GROUP, 12, struct gasket_page_table_ioctl_flags)
+
+#ifdef _KERNEL
+_Static_assert(sizeof(struct gasket_page_table_ioctl) == 32,
+ "gasket_page_table_ioctl changed size, breaking the ioctl ABI");
+_Static_assert(sizeof(struct gasket_page_table_ioctl_flags) == 40,
+ "gasket_page_table_ioctl_flags changed size, breaking the ioctl ABI");
+_Static_assert(sizeof(struct gasket_coherent_alloc_config_ioctl) == 32,
+ "gasket_coherent_alloc_config_ioctl changed size, breaking the ioctl ABI");
+_Static_assert(sizeof(struct gasket_interrupt_event) == 16,
+ "gasket_interrupt_event changed size, breaking the read(2) ABI");
+#endif
+
+#endif /* _DEV_GASKET_GASKET_IOCTL_H_ */
diff --git a/sys/dev/gasket/gasket_page_table.h b/sys/dev/gasket/gasket_page_table.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket_page_table.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _DEV_GASKET_GASKET_PAGE_TABLE_H_
+#define _DEV_GASKET_GASKET_PAGE_TABLE_H_
+
+#include <sys/types.h>
+#include <sys/bus.h>
+
+#include <vm/vm.h>
+
+#include <dev/gasket/gasket.h>
+
+struct gasket_page_table;
+
+enum gasket_map_kind {
+ GASKET_MAP_NONE = 0,
+ GASKET_MAP_HOST, /* pinned user page, released on unmap */
+ GASKET_MAP_COHERENT, /* driver's coherent buffer, not ours to free */
+};
+
+int gasket_page_table_init(device_t dev, struct resource *csr,
+ const struct gasket_page_table_config *cfg,
+ struct gasket_page_table **ptp);
+void gasket_page_table_free(struct gasket_page_table *pt);
+
+/* On success the entries own the pages and the array is cleared. */
+int gasket_page_table_map(struct gasket_page_table *pt,
+ vm_offset_t host_addr, uint64_t dev_addr, size_t size, int prot,
+ enum gasket_map_kind kind, vm_ooffset_t coh_off, vm_page_t *pages);
+int gasket_page_table_unmap(struct gasket_page_table *pt, uint64_t dev_addr,
+ size_t size);
+void gasket_page_table_unmap_all(struct gasket_page_table *pt);
+int gasket_page_table_partition(struct gasket_page_table *pt,
+ u_int num_simple);
+
+void gasket_page_table_set_coherent(struct gasket_page_table *pt,
+ const vm_paddr_t *pages, u_int npages, bus_addr_t bus);
+
+void gasket_page_table_unmap_coherent(struct gasket_page_table *pt);
+u_int gasket_page_table_num_entries(struct gasket_page_table *pt);
+u_int gasket_page_table_num_simple_entries(struct gasket_page_table *pt);
+
+/* Larger than the entry count: each extended entry covers a subtable. */
+u_int gasket_page_table_max_pages(struct gasket_page_table *pt);
+u_int gasket_page_table_num_active_pages(struct gasket_page_table *pt);
+
+#endif /* _DEV_GASKET_GASKET_PAGE_TABLE_H_ */
diff --git a/sys/dev/gasket/gasket_page_table.c b/sys/dev/gasket/gasket_page_table.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/gasket/gasket_page_table.c
@@ -0,0 +1,1033 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/sx.h>
+#include <sys/sysctl.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_page.h>
+
+#include <dev/gasket/gasket.h>
+#include <dev/gasket/gasket_page_table.h>
+
+#define GASKET_PAGE_SHIFT 12
+#define GASKET_EXTENDED_LVL0_SHIFT 21
+#define GASKET_EXTENDED_LVL0_WIDTH 13
+#define GASKET_SUBTABLE_ENTRIES 512
+#define GASKET_PTE_VALID 0x1
+
+CTASSERT(PAGE_SIZE == (1 << GASKET_PAGE_SHIFT));
+CTASSERT(GASKET_SUBTABLE_ENTRIES * sizeof(uint64_t) == PAGE_SIZE);
+
+static void gasket_garbage_collect_locked(struct gasket_page_table *pt);
+
+static MALLOC_DEFINE(M_GASKET_PT, "gasket_pt", "Gasket device page tables");
+
+static SYSCTL_NODE(_hw, OID_AUTO, gasket, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Gasket accelerator framework");
+
+static u_int gasket_max_wired_pages = 131072;
+SYSCTL_UINT(_hw_gasket, OID_AUTO, max_wired_pages, CTLFLAG_RWTUN,
+ &gasket_max_wired_pages, 0,
+ "Maximum pages one device page table may have mapped");
+
+struct gasket_mapping {
+ vm_page_t gm_page;
+ bus_dmamap_t gm_dma_map;
+ bus_addr_t gm_dma_addr;
+ int gm_prot;
+ enum gasket_map_kind gm_kind;
+ bool gm_mapped;
+};
+
+struct gasket_subtable {
+ uint64_t *gst_slots;
+ bus_dmamap_t gst_dma_map;
+ bus_addr_t gst_dma_addr;
+ struct gasket_mapping gst_entries[GASKET_SUBTABLE_ENTRIES];
+};
+
+struct gasket_l0_entry {
+ struct gasket_mapping gle_simple;
+ struct gasket_subtable *gle_subtable;
+};
+
+struct gasket_page_table {
+ device_t gpt_dev;
+ struct resource *gpt_csr;
+ struct gasket_page_table_config gpt_cfg;
+ struct gasket_l0_entry *gpt_entries;
+ bus_dma_tag_t gpt_dma_tag;
+ struct sx gpt_lock;
+ uint64_t gpt_extended_flag;
+ u_int gpt_num_simple;
+ u_int gpt_num_extended;
+ u_int gpt_num_active;
+
+ const vm_paddr_t *gpt_coherent_pages;
+ u_int gpt_coherent_npages;
+ bus_addr_t gpt_coherent_bus;
+};
+
+struct gasket_dma_load {
+ bus_addr_t gdl_addr;
+ int gdl_error;
+};
+
+static void
+gasket_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+ struct gasket_dma_load *load;
+
+ load = arg;
+ if (error != 0) {
+ load->gdl_error = error;
+ return;
+ }
+ if (nsegs != 1) {
+ load->gdl_error = EFBIG;
+ return;
+ }
+ load->gdl_addr = segs[0].ds_addr;
+ load->gdl_error = 0;
+}
+
+static int
+gasket_dma_load_page(struct gasket_page_table *pt, bus_dmamap_t map,
+ vm_page_t page, bus_addr_t *addrp)
+{
+ struct gasket_dma_load load;
+ struct memdesc mem;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ load.gdl_addr = 0;
+ load.gdl_error = ENOMEM;
+
+ /* memdesc_vmpages() would keep a pointer into this stack frame. */
+ mem = memdesc_paddr(VM_PAGE_TO_PHYS(page), PAGE_SIZE);
+ error = bus_dmamap_load_mem(pt->gpt_dma_tag, map, &mem,
+ gasket_dma_load_cb, &load, BUS_DMA_NOWAIT);
+ if (error != 0)
+ return (ENOMEM);
+ if (load.gdl_error != 0) {
+ bus_dmamap_unload(pt->gpt_dma_tag, map);
+ return (ENOMEM);
+ }
+ if ((load.gdl_addr & PAGE_MASK) != 0) {
+ bus_dmamap_unload(pt->gpt_dma_tag, map);
+ return (EFAULT);
+ }
+
+ *addrp = load.gdl_addr;
+ return (0);
+}
+
+/* A page the host writes is one the device reads, and the other way round. */
+static bus_dmasync_op_t
+gasket_dma_sync_op(int prot, bus_dmasync_op_t write, bus_dmasync_op_t read)
+{
+ bus_dmasync_op_t op;
+
+ op = 0;
+ if ((prot & VM_PROT_READ) != 0)
+ op |= write;
+ if ((prot & VM_PROT_WRITE) != 0)
+ op |= read;
+ return (op);
+}
+
+#define gasket_dma_pre_sync(prot) \
+ gasket_dma_sync_op((prot), BUS_DMASYNC_PREWRITE, BUS_DMASYNC_PREREAD)
+#define gasket_dma_post_sync(prot) \
+ gasket_dma_sync_op((prot), BUS_DMASYNC_POSTWRITE, BUS_DMASYNC_POSTREAD)
+
+static int
+gasket_mapping_load_locked(struct gasket_page_table *pt,
+ struct gasket_mapping *mapping, vm_page_t page, bus_addr_t bus, int prot,
+ enum gasket_map_kind kind)
+{
+ bus_dmasync_op_t op;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+ KASSERT(!mapping->gm_mapped, ("gasket: mapping already in use"));
+ KASSERT(kind != GASKET_MAP_NONE, ("gasket: mapping with no kind"));
+
+ if (kind == GASKET_MAP_COHERENT) {
+ mapping->gm_page = NULL;
+ mapping->gm_dma_addr = bus;
+ mapping->gm_prot = prot;
+ mapping->gm_kind = kind;
+ return (0);
+ }
+
+ error = bus_dmamap_create(pt->gpt_dma_tag, 0,
+ &mapping->gm_dma_map);
+ if (error != 0)
+ return (ENOMEM);
+ error = gasket_dma_load_page(pt, mapping->gm_dma_map, page,
+ &mapping->gm_dma_addr);
+ if (error != 0) {
+ bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map);
+ memset(mapping, 0, sizeof(*mapping));
+ return (error);
+ }
+
+ op = gasket_dma_pre_sync(prot);
+ if (op != 0)
+ bus_dmamap_sync(pt->gpt_dma_tag, mapping->gm_dma_map, op);
+ mapping->gm_page = page;
+ mapping->gm_prot = prot;
+ mapping->gm_kind = kind;
+ return (0);
+}
+
+static void
+gasket_mapping_discard_locked(struct gasket_page_table *pt,
+ struct gasket_mapping *mapping)
+{
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+ KASSERT(!mapping->gm_mapped,
+ ("gasket: discarding a published mapping"));
+
+ if (mapping->gm_kind == GASKET_MAP_HOST) {
+ bus_dmamap_unload(pt->gpt_dma_tag, mapping->gm_dma_map);
+ bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map);
+ }
+ memset(mapping, 0, sizeof(*mapping));
+}
+
+static void
+gasket_mapping_unload_locked(struct gasket_page_table *pt,
+ struct gasket_mapping *mapping)
+{
+ vm_page_t page;
+ bus_dmasync_op_t op;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ if (!mapping->gm_mapped)
+ return;
+
+ if (mapping->gm_kind == GASKET_MAP_COHERENT) {
+ KASSERT(pt->gpt_num_active != 0,
+ ("gasket: active page count underflow"));
+ pt->gpt_num_active--;
+ memset(mapping, 0, sizeof(*mapping));
+ return;
+ }
+
+ op = gasket_dma_post_sync(mapping->gm_prot);
+ if (op != 0)
+ bus_dmamap_sync(pt->gpt_dma_tag, mapping->gm_dma_map, op);
+ bus_dmamap_unload(pt->gpt_dma_tag, mapping->gm_dma_map);
+ bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map);
+
+ /* Dirtied at acquisition; busying it here trips an assertion. */
+ page = mapping->gm_page;
+ vm_page_unhold_pages(&page, 1);
+
+ KASSERT(pt->gpt_num_active != 0,
+ ("gasket: active page count underflow"));
+ pt->gpt_num_active--;
+ memset(mapping, 0, sizeof(*mapping));
+}
+
+static bus_size_t
+gasket_l0_slot_offset(struct gasket_page_table *pt, u_int slot)
+{
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+ return (pt->gpt_cfg.gpt_base_reg + (bus_size_t)slot * sizeof(uint64_t));
+}
+
+static bool
+gasket_mapping_range_free_locked(struct gasket_page_table *pt,
+ struct gasket_mapping *entries, u_int nentries)
+{
+ u_int i;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ for (i = 0; i < nentries; i++) {
+ if (entries[i].gm_mapped)
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+gasket_simple_range_free_locked(struct gasket_page_table *pt, u_int first,
+ u_int nentries)
+{
+ u_int i;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ for (i = 0; i < nentries; i++) {
+ if (pt->gpt_entries[first + i].gle_simple.gm_mapped)
+ return (false);
+ }
+ return (true);
+}
+
+static int
+gasket_alloc_subtable_locked(struct gasket_page_table *pt, u_int lvl0)
+{
+ struct gasket_dma_load load;
+ struct gasket_l0_entry *l0e;
+ struct gasket_subtable *subtable;
+ bus_size_t slot;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+ KASSERT(lvl0 < pt->gpt_num_extended,
+ ("gasket: extended level 0 index out of range"));
+
+ l0e = &pt->gpt_entries[pt->gpt_num_simple + lvl0];
+ if (l0e->gle_subtable != NULL)
+ return (0);
+
+ subtable = malloc(sizeof(*subtable), M_GASKET_PT, M_WAITOK | M_ZERO);
+ error = bus_dmamem_alloc(pt->gpt_dma_tag,
+ (void **)&subtable->gst_slots,
+ BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO,
+ &subtable->gst_dma_map);
+ if (error != 0) {
+ free(subtable, M_GASKET_PT);
+ return (ENOMEM);
+ }
+
+ load.gdl_addr = 0;
+ load.gdl_error = ENOMEM;
+ error = bus_dmamap_load(pt->gpt_dma_tag, subtable->gst_dma_map,
+ subtable->gst_slots, PAGE_SIZE, gasket_dma_load_cb, &load,
+ BUS_DMA_NOWAIT);
+ if (error != 0 || load.gdl_error != 0 ||
+ (load.gdl_addr & PAGE_MASK) != 0) {
+ if (error == 0)
+ bus_dmamap_unload(pt->gpt_dma_tag,
+ subtable->gst_dma_map);
+ bus_dmamem_free(pt->gpt_dma_tag, subtable->gst_slots,
+ subtable->gst_dma_map);
+ free(subtable, M_GASKET_PT);
+ return (ENOMEM);
+ }
+ subtable->gst_dma_addr = load.gdl_addr;
+
+ bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map,
+ BUS_DMASYNC_PREWRITE);
+ slot = gasket_l0_slot_offset(pt, pt->gpt_num_simple + lvl0);
+ gasket_wr8(pt->gpt_csr, slot,
+ subtable->gst_dma_addr | GASKET_PTE_VALID);
+ l0e->gle_subtable = subtable;
+ return (0);
+}
+
+static void
+gasket_free_subtable_locked(struct gasket_page_table *pt, u_int lvl0)
+{
+ struct gasket_l0_entry *l0e;
+ struct gasket_subtable *subtable;
+ bus_size_t slot;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+ KASSERT(lvl0 < pt->gpt_num_extended,
+ ("gasket: extended level 0 index out of range"));
+
+ l0e = &pt->gpt_entries[pt->gpt_num_simple + lvl0];
+ subtable = l0e->gle_subtable;
+ if (subtable == NULL)
+ return;
+ KASSERT(gasket_mapping_range_free_locked(pt, subtable->gst_entries,
+ GASKET_SUBTABLE_ENTRIES),
+ ("gasket: freeing active subtable"));
+
+ /* Posted write: read back before the subtable page is freed. */
+ slot = gasket_l0_slot_offset(pt, pt->gpt_num_simple + lvl0);
+ gasket_wr8(pt->gpt_csr, slot, 0);
+ (void)gasket_rd8(pt->gpt_csr, slot);
+ bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(pt->gpt_dma_tag, subtable->gst_dma_map);
+ bus_dmamem_free(pt->gpt_dma_tag, subtable->gst_slots,
+ subtable->gst_dma_map);
+ free(subtable, M_GASKET_PT);
+ l0e->gle_subtable = NULL;
+}
+
+static int
+gasket_validate_dev_range_locked(struct gasket_page_table *pt,
+ uint64_t dev_addr, u_int npages, bool *extendedp, u_int *indexp)
+{
+ uint64_t global, limit, raw;
+ u_int lvl0;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ if ((dev_addr & PAGE_MASK) != 0)
+ return (EINVAL);
+
+ if ((dev_addr & pt->gpt_extended_flag) == 0) {
+ global = dev_addr >> GASKET_PAGE_SHIFT;
+ if (global >= pt->gpt_num_simple ||
+ npages > pt->gpt_num_simple - global)
+ return (EINVAL);
+ *extendedp = false;
+ *indexp = (u_int)global;
+ return (0);
+ }
+
+ raw = dev_addr & ~pt->gpt_extended_flag;
+ global = raw >> GASKET_PAGE_SHIFT;
+ limit = (uint64_t)pt->gpt_num_extended *
+ GASKET_SUBTABLE_ENTRIES;
+ lvl0 = (u_int)(global / GASKET_SUBTABLE_ENTRIES);
+ if ((raw & PAGE_MASK) != 0 ||
+ raw >> (GASKET_EXTENDED_LVL0_WIDTH +
+ GASKET_EXTENDED_LVL0_SHIFT) != 0 ||
+ lvl0 >= pt->gpt_num_extended || global >= limit ||
+ npages > limit - global)
+ return (EINVAL);
+
+ *extendedp = true;
+ *indexp = (u_int)global;
+ return (0);
+}
+
+static int
+gasket_prepare_extended_locked(struct gasket_page_table *pt, u_int first,
+ u_int npages)
+{
+ struct gasket_subtable *subtable;
+ u_int len, lvl0, lvl1, remain;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ remain = npages;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+ if (subtable != NULL &&
+ !gasket_mapping_range_free_locked(pt,
+ &subtable->gst_entries[lvl1], len))
+ return (EBUSY);
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+
+ remain = npages;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ error = gasket_alloc_subtable_locked(pt, lvl0);
+ if (error != 0)
+ return (error);
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+ return (0);
+}
+
+static int
+gasket_map_simple_locked(struct gasket_page_table *pt, u_int first,
+ vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot,
+ enum gasket_map_kind kind, u_int *nmappedp)
+{
+ struct gasket_mapping *mapping;
+ uint64_t pte;
+ u_int i;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ for (i = 0; i < npages; i++) {
+ mapping = &pt->gpt_entries[first + i].gle_simple;
+ error = gasket_mapping_load_locked(pt, mapping,
+ pages != NULL ? pages[i] : NULL, bus[i], prot, kind);
+ if (error != 0) {
+ while (i-- != 0)
+ gasket_mapping_discard_locked(pt,
+ &pt->gpt_entries[first + i].gle_simple);
+ *nmappedp = 0;
+ return (error);
+ }
+ }
+
+ for (i = 0; i < npages; i++) {
+ mapping = &pt->gpt_entries[first + i].gle_simple;
+ pte = mapping->gm_dma_addr | GASKET_PTE_VALID;
+ gasket_wr8(pt->gpt_csr,
+ gasket_l0_slot_offset(pt, first + i), pte);
+ mapping->gm_mapped = true;
+ pt->gpt_num_active++;
+ if (pages != NULL)
+ pages[i] = NULL;
+ }
+ *nmappedp = npages;
+ return (0);
+}
+
+static void
+gasket_discard_extended_locked(struct gasket_page_table *pt, u_int first,
+ u_int nloaded)
+{
+ struct gasket_subtable *subtable;
+ u_int i, len, lvl0, lvl1, remain;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ remain = nloaded;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+ KASSERT(subtable != NULL, ("gasket: missing subtable"));
+
+ for (i = 0; i < len; i++)
+ gasket_mapping_discard_locked(pt,
+ &subtable->gst_entries[lvl1 + i]);
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+}
+
+static int
+gasket_map_extended_locked(struct gasket_page_table *pt, u_int first,
+ vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot,
+ enum gasket_map_kind kind, u_int *nmappedp)
+{
+ struct gasket_mapping *mapping;
+ struct gasket_subtable *subtable;
+ uint64_t pte;
+ u_int done, i, len, lvl0, lvl1, remain;
+ int error;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ done = 0;
+ remain = npages;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+ KASSERT(subtable != NULL, ("gasket: missing subtable"));
+
+ for (i = 0; i < len; i++) {
+ mapping = &subtable->gst_entries[lvl1 + i];
+ error = gasket_mapping_load_locked(pt, mapping,
+ pages != NULL ? pages[done] : NULL, bus[done], prot,
+ kind);
+ if (error != 0) {
+ gasket_discard_extended_locked(pt, first, done);
+ *nmappedp = 0;
+ return (error);
+ }
+ done++;
+ }
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+
+ done = 0;
+ remain = npages;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+
+ for (i = 0; i < len; i++) {
+ mapping = &subtable->gst_entries[lvl1 + i];
+ pte = mapping->gm_dma_addr | GASKET_PTE_VALID;
+ subtable->gst_slots[lvl1 + i] = pte;
+ mapping->gm_mapped = true;
+ pt->gpt_num_active++;
+ if (pages != NULL)
+ pages[done] = NULL;
+ done++;
+ }
+ bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map,
+ BUS_DMASYNC_PREWRITE);
+
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+ *nmappedp = npages;
+ return (0);
+}
+
+static void
+gasket_unmap_simple_locked(struct gasket_page_table *pt, u_int first,
+ u_int npages)
+{
+ struct gasket_mapping *mapping;
+ u_int i;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ /* MMIO writes are posted: flush before releasing the pages. */
+ for (i = 0; i < npages; i++) {
+ gasket_wr8(pt->gpt_csr,
+ gasket_l0_slot_offset(pt, first + i), 0);
+ }
+ if (npages != 0)
+ (void)gasket_rd8(pt->gpt_csr, gasket_l0_slot_offset(pt, first));
+
+ for (i = 0; i < npages; i++) {
+ mapping = &pt->gpt_entries[first + i].gle_simple;
+ gasket_mapping_unload_locked(pt, mapping);
+ }
+}
+
+static void
+gasket_unmap_extended_locked(struct gasket_page_table *pt, u_int first,
+ u_int npages)
+{
+ struct gasket_subtable *subtable;
+ u_int i, len, lvl0, lvl1, remain;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ remain = npages;
+ lvl0 = first / GASKET_SUBTABLE_ENTRIES;
+ lvl1 = first % GASKET_SUBTABLE_ENTRIES;
+ while (remain != 0) {
+ len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1);
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+ if (subtable != NULL) {
+ for (i = 0; i < len; i++)
+ subtable->gst_slots[lvl1 + i] = 0;
+ bus_dmamap_sync(pt->gpt_dma_tag,
+ subtable->gst_dma_map, BUS_DMASYNC_PREWRITE);
+ (void)gasket_rd8(pt->gpt_csr, gasket_l0_slot_offset(pt,
+ pt->gpt_num_simple + lvl0));
+
+ for (i = 0; i < len; i++)
+ gasket_mapping_unload_locked(pt,
+ &subtable->gst_entries[lvl1 + i]);
+ }
+ remain -= len;
+ lvl0++;
+ lvl1 = 0;
+ }
+}
+
+static void
+gasket_unmap_all_locked(struct gasket_page_table *pt)
+{
+ u_int i;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ gasket_unmap_simple_locked(pt, 0, pt->gpt_num_simple);
+ for (i = 0; i < pt->gpt_num_extended; i++) {
+ if (pt->gpt_entries[pt->gpt_num_simple +
+ i].gle_subtable != NULL)
+ gasket_unmap_extended_locked(pt,
+ i * GASKET_SUBTABLE_ENTRIES,
+ GASKET_SUBTABLE_ENTRIES);
+ }
+}
+
+int
+gasket_page_table_init(device_t dev, struct resource *csr,
+ const struct gasket_page_table_config *cfg,
+ struct gasket_page_table **ptp)
+{
+ struct gasket_page_table *pt;
+ bus_size_t csr_size, table_size;
+ u_int i;
+ int error;
+
+ if (ptp == NULL)
+ return (EINVAL);
+ *ptp = NULL;
+ if (dev == NULL || csr == NULL || cfg == NULL ||
+ cfg->gpt_total_entries == 0 ||
+ !powerof2(cfg->gpt_total_entries) ||
+ cfg->gpt_total_entries > (1U << GASKET_EXTENDED_LVL0_WIDTH) ||
+ cfg->gpt_extended_bit < GASKET_EXTENDED_LVL0_WIDTH +
+ GASKET_EXTENDED_LVL0_SHIFT ||
+ cfg->gpt_extended_bit >= 64)
+ return (EINVAL);
+
+ csr_size = rman_get_size(csr);
+ table_size = (bus_size_t)cfg->gpt_total_entries * sizeof(uint64_t);
+ if (cfg->gpt_base_reg > csr_size ||
+ table_size > csr_size - cfg->gpt_base_reg ||
+ cfg->gpt_extended_reg > csr_size ||
+ sizeof(uint64_t) > csr_size - cfg->gpt_extended_reg)
+ return (EINVAL);
+
+ pt = malloc(sizeof(*pt), M_GASKET_PT, M_WAITOK | M_ZERO);
+ pt->gpt_entries = mallocarray(cfg->gpt_total_entries,
+ sizeof(*pt->gpt_entries), M_GASKET_PT, M_WAITOK | M_ZERO);
+ pt->gpt_dev = dev;
+ pt->gpt_csr = csr;
+ pt->gpt_cfg = *cfg;
+ pt->gpt_extended_flag = 1ULL << cfg->gpt_extended_bit;
+ pt->gpt_num_simple = cfg->gpt_total_entries;
+ sx_init(&pt->gpt_lock, "gasket page table");
+
+ error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, PAGE_SIZE, 1,
+ PAGE_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &pt->gpt_dma_tag);
+ if (error != 0) {
+ sx_destroy(&pt->gpt_lock);
+ free(pt->gpt_entries, M_GASKET_PT);
+ free(pt, M_GASKET_PT);
+ return (ENOMEM);
+ }
+
+ sx_xlock(&pt->gpt_lock);
+ for (i = 0; i < cfg->gpt_total_entries; i++)
+ gasket_wr8(csr, gasket_l0_slot_offset(pt, i), 0);
+ gasket_wr8(csr, cfg->gpt_extended_reg, cfg->gpt_total_entries);
+ sx_xunlock(&pt->gpt_lock);
+
+ *ptp = pt;
+ return (0);
+}
+
+void
+gasket_page_table_free(struct gasket_page_table *pt)
+{
+ u_int i;
+
+ if (pt == NULL)
+ return;
+
+ sx_xlock(&pt->gpt_lock);
+ gasket_unmap_all_locked(pt);
+ for (i = 0; i < pt->gpt_num_extended; i++)
+ gasket_free_subtable_locked(pt, i);
+ gasket_wr8(pt->gpt_csr, pt->gpt_cfg.gpt_extended_reg,
+ pt->gpt_cfg.gpt_total_entries);
+ sx_xunlock(&pt->gpt_lock);
+
+ bus_dma_tag_destroy(pt->gpt_dma_tag);
+ sx_destroy(&pt->gpt_lock);
+ free(pt->gpt_entries, M_GASKET_PT);
+ free(pt, M_GASKET_PT);
+}
+
+int
+gasket_page_table_map(struct gasket_page_table *pt, vm_offset_t host_addr,
+ uint64_t dev_addr, size_t size, int prot, enum gasket_map_kind kind,
+ vm_ooffset_t coh_off, vm_page_t *pages)
+{
+ bus_addr_t *bus;
+ vm_ooffset_t limit;
+ bool coherent, extended;
+ u_int first, i, nmapped, npages;
+ int error;
+
+ if (kind != GASKET_MAP_HOST && kind != GASKET_MAP_COHERENT)
+ return (EINVAL);
+ if (kind == GASKET_MAP_HOST && pages == NULL)
+ return (EINVAL);
+#ifdef KMSAN
+ /*
+ * kmsan_bus_dmamap_sync() panics on anything but virtual address and
+ * mbuf descriptors, and a page mapped for a device is described
+ * physically.
+ */
+ if (kind == GASKET_MAP_HOST)
+ return (EOPNOTSUPP);
+#endif
+
+ if (pt != NULL && size / PAGE_SIZE > gasket_page_table_max_pages(pt))
+ return (EINVAL);
+
+ if (pt == NULL || size == 0 || (size & PAGE_MASK) != 0 ||
+ (host_addr & PAGE_MASK) != 0 || host_addr + size < host_addr ||
+ (prot & ~(VM_PROT_READ | VM_PROT_WRITE)) != 0 ||
+ (prot & (VM_PROT_READ | VM_PROT_WRITE)) == 0)
+ return (EINVAL);
+ npages = (u_int)(size >> GASKET_PAGE_SHIFT);
+ if ((size >> GASKET_PAGE_SHIFT) != npages)
+ return (EINVAL);
+
+ sx_xlock(&pt->gpt_lock);
+
+ if (pt->gpt_num_active + npages > gasket_max_wired_pages ||
+ pt->gpt_num_active + npages < pt->gpt_num_active) {
+ error = ENOMEM;
+ goto out;
+ }
+
+ error = gasket_validate_dev_range_locked(pt, dev_addr, npages,
+ &extended, &first);
+ if (error != 0)
+ goto out;
+
+ if (extended) {
+ error = gasket_prepare_extended_locked(pt, first, npages);
+ if (error != 0) {
+ gasket_garbage_collect_locked(pt);
+ goto out;
+ }
+ } else if (!gasket_simple_range_free_locked(pt, first, npages)) {
+ error = EBUSY;
+ goto out;
+ }
+
+ bus = mallocarray(npages, sizeof(*bus), M_GASKET_PT,
+ M_WAITOK | M_ZERO);
+
+ coherent = kind == GASKET_MAP_COHERENT;
+ if (coherent) {
+ limit = (vm_ooffset_t)pt->gpt_coherent_npages * PAGE_SIZE;
+ if (pt->gpt_coherent_pages == NULL || coh_off > limit ||
+ (vm_ooffset_t)size > limit - coh_off) {
+ error = EINVAL;
+ goto out_free;
+ }
+ for (i = 0; i < npages; i++)
+ bus[i] = pt->gpt_coherent_bus + (bus_addr_t)coh_off +
+ (bus_addr_t)i * PAGE_SIZE;
+ }
+
+ nmapped = 0;
+ if (extended)
+ error = gasket_map_extended_locked(pt, first, pages, bus,
+ npages, prot, kind, &nmapped);
+ else
+ error = gasket_map_simple_locked(pt, first, pages, bus,
+ npages, prot, kind, &nmapped);
+ if (error != 0) {
+ if (extended)
+ gasket_unmap_extended_locked(pt, first, nmapped);
+ else
+ gasket_unmap_simple_locked(pt, first, nmapped);
+ }
+out_free:
+ if (error != 0 && extended)
+ gasket_garbage_collect_locked(pt);
+ free(bus, M_GASKET_PT);
+
+out:
+ sx_xunlock(&pt->gpt_lock);
+ return (error);
+}
+
+int
+gasket_page_table_unmap(struct gasket_page_table *pt, uint64_t dev_addr,
+ size_t size)
+{
+ bool extended;
+ u_int first, npages;
+ int error;
+
+ if (pt == NULL || size == 0 || (size & PAGE_MASK) != 0)
+ return (EINVAL);
+ npages = (u_int)(size >> GASKET_PAGE_SHIFT);
+ if ((size >> GASKET_PAGE_SHIFT) != npages)
+ return (EINVAL);
+
+ sx_xlock(&pt->gpt_lock);
+ error = gasket_validate_dev_range_locked(pt, dev_addr, npages,
+ &extended, &first);
+ if (error == 0) {
+ if (extended)
+ gasket_unmap_extended_locked(pt, first, npages);
+ else
+ gasket_unmap_simple_locked(pt, first, npages);
+ }
+ sx_xunlock(&pt->gpt_lock);
+ return (error);
+}
+
+void
+gasket_page_table_unmap_all(struct gasket_page_table *pt)
+{
+
+ if (pt == NULL)
+ return;
+
+ sx_xlock(&pt->gpt_lock);
+ gasket_unmap_all_locked(pt);
+ sx_xunlock(&pt->gpt_lock);
+}
+
+void
+gasket_page_table_set_coherent(struct gasket_page_table *pt,
+ const vm_paddr_t *pages, u_int npages, bus_addr_t bus)
+{
+
+ if (pt == NULL)
+ return;
+
+ sx_xlock(&pt->gpt_lock);
+ pt->gpt_coherent_pages = pages;
+ pt->gpt_coherent_npages = npages;
+ pt->gpt_coherent_bus = bus;
+ sx_xunlock(&pt->gpt_lock);
+}
+
+void
+gasket_page_table_unmap_coherent(struct gasket_page_table *pt)
+{
+ struct gasket_subtable *subtable;
+ u_int i, j;
+
+ if (pt == NULL)
+ return;
+
+ sx_xlock(&pt->gpt_lock);
+ for (i = 0; i < pt->gpt_num_simple; i++) {
+ if (pt->gpt_entries[i].gle_simple.gm_mapped &&
+ pt->gpt_entries[i].gle_simple.gm_kind ==
+ GASKET_MAP_COHERENT)
+ gasket_unmap_simple_locked(pt, i, 1);
+ }
+ for (i = 0; i < pt->gpt_num_extended; i++) {
+ subtable = pt->gpt_entries[pt->gpt_num_simple + i].gle_subtable;
+ if (subtable == NULL)
+ continue;
+ for (j = 0; j < GASKET_SUBTABLE_ENTRIES; j++) {
+ if (subtable->gst_entries[j].gm_mapped &&
+ subtable->gst_entries[j].gm_kind ==
+ GASKET_MAP_COHERENT)
+ gasket_unmap_extended_locked(pt,
+ i * GASKET_SUBTABLE_ENTRIES + j, 1);
+ }
+ }
+ sx_xunlock(&pt->gpt_lock);
+}
+
+static void
+gasket_garbage_collect_locked(struct gasket_page_table *pt)
+{
+ struct gasket_subtable *subtable;
+ u_int lvl0;
+
+ sx_assert(&pt->gpt_lock, SA_XLOCKED);
+
+ for (lvl0 = 0; lvl0 < pt->gpt_num_extended; lvl0++) {
+ subtable = pt->gpt_entries[pt->gpt_num_simple +
+ lvl0].gle_subtable;
+ if (subtable == NULL)
+ continue;
+ if (gasket_mapping_range_free_locked(pt, subtable->gst_entries,
+ GASKET_SUBTABLE_ENTRIES))
+ gasket_free_subtable_locked(pt, lvl0);
+ }
+}
+
+int
+gasket_page_table_partition(struct gasket_page_table *pt, u_int num_simple)
+{
+ u_int i, start;
+
+ if (pt == NULL)
+ return (EINVAL);
+
+ sx_xlock(&pt->gpt_lock);
+ if (num_simple > pt->gpt_cfg.gpt_total_entries) {
+ sx_xunlock(&pt->gpt_lock);
+ return (EINVAL);
+ }
+
+ gasket_garbage_collect_locked(pt);
+
+ start = MIN(pt->gpt_num_simple, num_simple);
+ for (i = start; i < pt->gpt_cfg.gpt_total_entries; i++) {
+ if (pt->gpt_entries[i].gle_simple.gm_mapped ||
+ pt->gpt_entries[i].gle_subtable != NULL) {
+ device_printf(pt->gpt_dev,
+ "page table entry %u is still in use\n", i);
+ sx_xunlock(&pt->gpt_lock);
+ return (EBUSY);
+ }
+ }
+
+ pt->gpt_num_simple = num_simple;
+ pt->gpt_num_extended = pt->gpt_cfg.gpt_total_entries - num_simple;
+ gasket_wr8(pt->gpt_csr, pt->gpt_cfg.gpt_extended_reg, num_simple);
+ sx_xunlock(&pt->gpt_lock);
+
+ return (0);
+}
+
+u_int
+gasket_page_table_num_entries(struct gasket_page_table *pt)
+{
+
+ if (pt == NULL)
+ return (0);
+ return (pt->gpt_num_simple + pt->gpt_num_extended);
+}
+
+u_int
+gasket_page_table_num_simple_entries(struct gasket_page_table *pt)
+{
+ u_int n;
+
+ if (pt == NULL)
+ return (0);
+ sx_slock(&pt->gpt_lock);
+ n = pt->gpt_num_simple;
+ sx_sunlock(&pt->gpt_lock);
+
+ return (n);
+}
+
+u_int
+gasket_page_table_max_pages(struct gasket_page_table *pt)
+{
+ u_int n;
+
+ if (pt == NULL)
+ return (0);
+ sx_slock(&pt->gpt_lock);
+ n = pt->gpt_num_simple +
+ pt->gpt_num_extended * GASKET_SUBTABLE_ENTRIES;
+ sx_sunlock(&pt->gpt_lock);
+
+ return (n);
+}
+
+u_int
+gasket_page_table_num_active_pages(struct gasket_page_table *pt)
+{
+ u_int nactive;
+
+ if (pt == NULL)
+ return (0);
+
+ sx_slock(&pt->gpt_lock);
+ nactive = pt->gpt_num_active;
+ sx_sunlock(&pt->gpt_lock);
+ return (nactive);
+}
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -131,6 +131,7 @@
${_ftwd} \
fusefs \
${_fxp} \
+ ${_gasket} \
gem \
geom \
${_glxiic} \
@@ -550,6 +551,9 @@
.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
SUBDIR+= linux
.endif
+.if ${MACHINE_CPUARCH} == "amd64"
+_gasket= gasket
+.endif
.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64"
SUBDIR+= linux64
SUBDIR+= linux_common
diff --git a/sys/modules/gasket/Makefile b/sys/modules/gasket/Makefile
new file mode 100644
--- /dev/null
+++ b/sys/modules/gasket/Makefile
@@ -0,0 +1,13 @@
+.PATH: ${SRCTOP}/sys/dev/gasket
+
+KMOD= gasket
+SRCS= gasket_core.c gasket_page_table.c
+SRCS+= gasket_if.c gasket_if.h
+SRCS+= device_if.h bus_if.h pci_if.h vnode_if.h
+
+# The device page table needs atomic 64 bit MMIO and 4 KiB pages.
+.if ${MACHINE_CPUARCH} != "amd64"
+.error gasket requires an architecture with bus_read_8()/bus_write_8()
+.endif
+
+.include <bsd.kmod.mk>

File Metadata

Mime Type
text/plain
Expires
Tue, Aug 4, 1:41 AM (9 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35860402
Default Alt Text
D58561.diff (75 KB)

Event Timeline