diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist --- a/etc/mtree/BSD.include.dist +++ b/etc/mtree/BSD.include.dist @@ -215,6 +215,8 @@ .. firewire .. + gasket + .. hid .. hwpmc diff --git a/include/Makefile b/include/Makefile --- a/include/Makefile +++ b/include/Makefile @@ -185,6 +185,10 @@ OPENCRYPTO= cryptodev.h OPENCRYPTODIR= ${INCLUDEDIR}/crypto +.PATH: ${SRCTOP}/sys/dev/gasket +GASKET= gasket_ioctl.h +GASKETDIR= ${INCLUDEDIR}/dev/gasket + .PATH: ${SRCTOP}/sys/dev/nvme NVME= nvme.h NVMEDIR= ${INCLUDEDIR}/dev/nvme @@ -269,6 +273,7 @@ CRYPTO \ EVDEV \ FS9660 \ + GASKET \ HID \ HYPERV \ OPENCRYPTO \ diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -193,6 +193,7 @@ fwohci.4 \ fxp.4 \ gdb.4 \ + gasket.4 \ gem.4 \ genet.4 \ genetlink.4 \ diff --git a/share/man/man4/gasket.4 b/share/man/man4/gasket.4 new file mode 100644 --- /dev/null +++ b/share/man/man4/gasket.4 @@ -0,0 +1,48 @@ +.\" +.\" Copyright (c) 2026 Abdelkader Boudih +.\" +.\" SPDX-License-Identifier: BSD-2-Clause +.\" +.Dd July 30, 2026 +.Dt GASKET 4 +.Os +.Sh NAME +.Nm gasket +.Nd Google Edge TPU accelerator framework +.Sh SYNOPSIS +.Cd device gasket +.Sh DESCRIPTION +The +.Nm +framework provides the character device, the mapping policy and the on-chip +page table shared by Google accelerator ASICs. +It does not attach to hardware; a chip driver such as +.Xr apex 4 +describes its registers and implements the +.Nm +kobj interface. +.Pp +Only the register ranges a chip driver declares mappable are reachable from +user space; the rest of the window, including the page table, stays private. +.Sh SYSCTL VARIABLES +.Bl -tag -width indent +.It Va hw.gasket.max_wired_pages +Maximum pages one accelerator may have mapped, 131072 by default. +These are not charged against +.Xr getrlimit 2 +.Dv RLIMIT_MEMLOCK , +so this is the only bound on what a consumer can pin. +.El +.Sh SEE ALSO +.Xr apex 4 , +.Xr pci 4 +.Sh HISTORY +The +.Nm +framework first appeared in +.Fx 16.0 . +The name comes from the framework of the same name that Google distributed +for Linux, where it stands for Google ASIC Software, Kernel Extensions, and +Tools. +.Sh AUTHORS +.An Abdelkader Boudih Aq Mt seuros@FreeBSD.org diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -65,6 +65,11 @@ device thunderbolt options THUNDERBOLT_DEBUG +# +# Framework for Google accelerator ASICs; see gasket(4). +# +device gasket + # # Microsemi smartpqi controllers. # These controllers have a SCSI-like interface, and require the diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -112,6 +112,9 @@ crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl crypto/openssl/ossl_aes_gcm.c optional ossl dev/amdgpio/amdgpio.c optional amdgpio +dev/gasket/gasket_core.c optional gasket +dev/gasket/gasket_page_table.c optional gasket +dev/gasket/gasket_if.m optional gasket dev/apple_bce/apple_bce.c optional apple_bce pci dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci dev/apple_bce/apple_bce_queue.c optional apple_bce pci diff --git a/sys/dev/gasket/gasket.h b/sys/dev/gasket/gasket.h new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _DEV_GASKET_GASKET_H_ +#define _DEV_GASKET_GASKET_H_ + +#include +#include +#include + +#include + +#define GASKET_STATUS_DEAD 0 +#define GASKET_STATUS_ALIVE 1 + +struct gasket_region { + bus_size_t gr_offset; + bus_size_t gr_size; +}; + +struct gasket_page_table_config { + u_int gpt_id; + u_int gpt_total_entries; + bus_size_t gpt_base_reg; + bus_size_t gpt_extended_reg; + u_int gpt_extended_bit; +}; + +/* Apex packs up to four vector control fields per register. */ +enum gasket_interrupt_packing { + GASKET_INTR_UNPACKED = 0, + GASKET_INTR_PACK_0, + GASKET_INTR_PACK_1, + GASKET_INTR_PACK_2, + GASKET_INTR_PACK_3, +}; + +struct gasket_interrupt_desc { + u_int gid_index; + bus_size_t gid_reg; + enum gasket_interrupt_packing gid_packing; +}; + +struct gasket_desc { + const char *gd_name; + struct resource *gd_csr; + + const struct gasket_region *gd_regions; + u_int gd_nregions; + + const struct gasket_page_table_config *gd_page_tables; + u_int gd_npage_tables; + + const struct gasket_interrupt_desc *gd_interrupts; + u_int gd_ninterrupts; + u_int gd_interrupt_pack_width; + + bus_size_t gd_coherent_base; + bus_size_t gd_coherent_size; +}; + +struct gasket_softc; + +int gasket_attach(device_t dev, const struct gasket_desc *desc, + struct gasket_softc **scp); + +int gasket_suspend_begin(struct gasket_softc *sc); +void gasket_resume(struct gasket_softc *sc); + +/* Two phase: do not call finish() until no handler can reach the softc. */ +int gasket_detach_begin(struct gasket_softc *sc); +void gasket_detach_abort(struct gasket_softc *sc); +void gasket_detach_finish(struct gasket_softc *sc); + +bool gasket_is_open(struct gasket_softc *sc); + +/* Safe to call from an interrupt handler. */ +void gasket_interrupt_notify(struct gasket_softc *sc, u_int index); + +/* False when no buffer is allocated; zero is a legal bus address. */ +bool gasket_coherent_bus_addr(struct gasket_softc *sc, bus_addr_t *addrp); + +struct gasket_page_table *gasket_page_table_get(struct gasket_softc *sc, + u_int index); + +/* + * A page table entry carries its valid bit in bit 0, so a split 64 bit write + * publishes a valid entry whose upper address bits are still zero. + */ +static inline uint32_t +gasket_rd4(struct resource *csr, bus_size_t off) +{ + return (bus_read_4(csr, off)); +} + +static inline void +gasket_wr4(struct resource *csr, bus_size_t off, uint32_t val) +{ + bus_write_4(csr, off, val); +} + +static inline uint64_t +gasket_rd8(struct resource *csr, bus_size_t off) +{ + return (bus_read_8(csr, off)); +} + +static inline void +gasket_wr8(struct resource *csr, bus_size_t off, uint64_t val) +{ + bus_write_8(csr, off, val); +} + +static inline void +gasket_rmw4(struct resource *csr, bus_size_t off, uint32_t val, u_int width, + u_int shift) +{ + uint32_t mask, tmp; + + mask = (uint32_t)(((1ULL << width) - 1) << shift); + tmp = bus_read_4(csr, off); + tmp = (tmp & ~mask) | ((val << shift) & mask); + bus_write_4(csr, off, tmp); +} + +static inline void +gasket_rmw8(struct resource *csr, bus_size_t off, uint64_t val, u_int width, + u_int shift) +{ + uint64_t mask, tmp; + + mask = ((1ULL << width) - 1) << shift; + tmp = gasket_rd8(csr, off); + tmp = (tmp & ~mask) | ((val << shift) & mask); + gasket_wr8(csr, off, tmp); +} + +int gasket_wait8(struct resource *csr, bus_size_t off, uint64_t mask, + uint64_t val, int retries, int delay_ms); + +#endif /* _DEV_GASKET_GASKET_H_ */ diff --git a/sys/dev/gasket/gasket_chardev.c b/sys/dev/gasket/gasket_chardev.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_chardev.c @@ -0,0 +1,502 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include "gasket_priv.h" +#include "gasket_if.h" + +int +gasket_cdev_open(struct cdev *cdev, int oflags, int devtype __unused, + struct thread *td) +{ + struct gasket_softc *sc = cdev->si_drv1; + bool writable = (oflags & FWRITE) != 0; + int error = 0; + + sx_xlock(&sc->gs_lock); + if (sc->gs_dying) { + sx_xunlock(&sc->gs_lock); + return (ENXIO); + } + if (sc->gs_suspended) { + sx_xunlock(&sc->gs_lock); + return (EBUSY); + } + if (sc->gs_stuck) { + sx_xunlock(&sc->gs_lock); + return (ENXIO); + } + if (writable && sc->gs_writers != 0 && + sc->gs_owner != td->td_proc) { + sx_xunlock(&sc->gs_lock); + return (EBUSY); + } + + if (sc->gs_opens == 0) { + error = GASKET_RESET(sc->gs_dev); + if (error != 0) { + (void)GASKET_ENTER_RESET(sc->gs_dev); + } else { + error = GASKET_OPEN(sc->gs_dev); + if (error != 0) { + (void)GASKET_ENTER_RESET(sc->gs_dev); + } + } + } + if (error == 0) { + if (sc->gs_opens == 0) { + mtx_lock(&sc->gs_evlock); + sc->gs_ev_head = sc->gs_ev_tail = 0; + sc->gs_ev_dropped = 0; + mtx_unlock(&sc->gs_evlock); + } + sc->gs_coherent_defer = false; + if (writable) { + sc->gs_owner = td->td_proc; + sc->gs_writers++; + } + sc->gs_opens++; + } + sx_xunlock(&sc->gs_lock); + + return (error); +} + +int +gasket_cdev_close(struct cdev *cdev, int fflag, int devtype __unused, + struct thread *td __unused) +{ + struct gasket_softc *sc = cdev->si_drv1; + struct gasket_page_table *pt; + u_int i; + int error; + + sx_xlock(&sc->gs_lock); + if (sc->gs_opens > 0) + sc->gs_opens--; + if ((fflag & FWRITE) != 0 && sc->gs_writers > 0 && + --sc->gs_writers == 0) + sc->gs_owner = NULL; + if (sc->gs_opens == 0) { + if (!sc->gs_dying) { + GASKET_CLOSE(sc->gs_dev); + error = GASKET_QUIESCE(sc->gs_dev); + if (error != 0) { + device_printf(sc->gs_dev, + "device did not quiesce (%d), resetting\n", + error); + + if (GASKET_RESET(sc->gs_dev) != 0) + device_printf(sc->gs_dev, + "reset also failed, asking the " + "engine anyway\n"); + error = GASKET_QUIESCE(sc->gs_dev); + } + if (error != 0) { + device_printf(sc->gs_dev, + "device will not stop (%d), retaining its " + "mappings and refusing to detach\n", error); + sc->gs_stuck = true; + sx_xunlock(&sc->gs_lock); + return (0); + } + (void)GASKET_ENTER_RESET(sc->gs_dev); + } + + for (i = 0; i < sc->gs_desc->gd_npage_tables; i++) { + pt = sc->gs_page_tables[i]; + gasket_page_table_unmap_all(pt); + (void)gasket_page_table_partition(pt, + gasket_page_table_num_entries(pt)); + } + + if (gasket_coherent_free(sc) == EBUSY) + sc->gs_coherent_defer = true; + } + sx_xunlock(&sc->gs_lock); + + return (0); +} + +int +gasket_cdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, + struct thread *td __unused) +{ + struct gasket_softc *sc = cdev->si_drv1; + u_int i; + int error; + + switch (cmd) { + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: + break; + default: + if ((fflag & FWRITE) == 0) + return (EPERM); + break; + } + + sx_xlock(&sc->gs_lock); + if (sc->gs_dying) { + sx_xunlock(&sc->gs_lock); + return (ENXIO); + } + + switch (cmd) { + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: { + const struct gasket_page_table_ioctl *pti = (const void *)data; + + if (pti->page_table_index > UINT_MAX) { + sx_xunlock(&sc->gs_lock); + return (EINVAL); + } + break; + } + case GASKET_IOCTL_PARTITION_PAGE_TABLE: + case GASKET_IOCTL_UNMAP_BUFFER: { + const struct gasket_page_table_ioctl *pti = (const void *)data; + + if (pti->page_table_index > UINT_MAX || + pti->size > UINT_MAX) { + sx_xunlock(&sc->gs_lock); + return (EINVAL); + } + break; + } + case GASKET_IOCTL_MAP_BUFFER: { + const struct gasket_page_table_ioctl *pti = (const void *)data; + + if (pti->page_table_index > UINT_MAX || + pti->size > UINT_MAX || + pti->host_address > (uint64_t)VM_MAX_ADDRESS) { + sx_xunlock(&sc->gs_lock); + return (EINVAL); + } + break; + } + case GASKET_IOCTL_MAP_BUFFER_FLAGS: { + const struct gasket_page_table_ioctl_flags *ptf = + (const void *)data; + + if (ptf->base.page_table_index > UINT_MAX || + ptf->base.size > UINT_MAX || + ptf->base.host_address > (uint64_t)VM_MAX_ADDRESS) { + sx_xunlock(&sc->gs_lock); + return (EINVAL); + } + break; + } + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: { + const struct gasket_coherent_alloc_config_ioctl *cfg = + (const void *)data; + + if (cfg->page_table_index >= sc->gs_desc->gd_npage_tables) { + sx_xunlock(&sc->gs_lock); + return (EINVAL); + } + break; + } + } + + switch (cmd) { + case GASKET_IOCTL_RESET: + error = GASKET_QUIESCE(sc->gs_dev); + if (error != 0) + break; + for (i = 0; i < sc->gs_desc->gd_npage_tables; i++) + gasket_page_table_unmap_all(sc->gs_page_tables[i]); + error = GASKET_RESET(sc->gs_dev); + break; + case GASKET_IOCTL_NUMBER_PAGE_TABLES: + *(uint64_t *)data = sc->gs_desc->gd_npage_tables; + error = 0; + break; + case GASKET_IOCTL_PAGE_TABLE_SIZE: + case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: { + struct gasket_page_table_ioctl *pti = (void *)data; + struct gasket_page_table *pt; + + pt = gasket_page_table_get(sc, pti->page_table_index); + if (pt == NULL) { + error = EINVAL; + break; + } + if (cmd == GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE) + pti->size = gasket_page_table_num_simple_entries(pt); + else + pti->size = gasket_page_table_num_entries(pt); + error = 0; + break; + } + case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: + mtx_lock(&sc->gs_evlock); + memset(sc->gs_counts, 0, sc->gs_desc->gd_ninterrupts * + sizeof(*sc->gs_counts)); + sc->gs_ev_head = sc->gs_ev_tail = 0; + sc->gs_ev_dropped = 0; + mtx_unlock(&sc->gs_evlock); + error = 0; + break; + case GASKET_IOCTL_PARTITION_PAGE_TABLE: { + struct gasket_page_table_ioctl *pti = (void *)data; + struct gasket_page_table *pt; + + pt = gasket_page_table_get(sc, pti->page_table_index); + if (pt == NULL) { + error = EINVAL; + break; + } + error = gasket_page_table_partition(pt, pti->size); + break; + } + case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: { + struct gasket_coherent_alloc_config_ioctl *cfg = (void *)data; + + if (cfg->enable != 0) { + error = gasket_coherent_alloc(sc, + (u_int)cfg->page_table_index, cfg->size); + if (error == 0) { + cfg->dma_address = + sc->gs_desc->gd_coherent_base; + } + } else { + error = gasket_coherent_free(sc); + if (error == 0) + cfg->dma_address = 0; + } + break; + } + case GASKET_IOCTL_MAP_BUFFER: { + struct gasket_page_table_ioctl *pti = (void *)data; + struct gasket_page_table *pt; + + pt = gasket_page_table_get(sc, pti->page_table_index); + if (pt == NULL) { + error = EINVAL; + break; + } + error = gasket_map_buffer(sc, pt, + (vm_offset_t)pti->host_address, pti->device_address, + pti->size, VM_PROT_READ | VM_PROT_WRITE); + break; + } + case GASKET_IOCTL_MAP_BUFFER_FLAGS: { + struct gasket_page_table_ioctl_flags *ptf = (void *)data; + struct gasket_page_table *pt; + int prot; + + pt = gasket_page_table_get(sc, ptf->base.page_table_index); + if (pt == NULL) { + error = EINVAL; + break; + } + + switch (ptf->flags & GASKET_DMA_NONE) { + case GASKET_DMA_BIDIRECTIONAL: + prot = VM_PROT_READ | VM_PROT_WRITE; + break; + case GASKET_DMA_TO_DEVICE: + prot = VM_PROT_READ; + break; + case GASKET_DMA_FROM_DEVICE: + prot = VM_PROT_WRITE; + break; + default: + error = EINVAL; + goto done; + } + + error = gasket_map_buffer(sc, pt, + (vm_offset_t)ptf->base.host_address, + ptf->base.device_address, ptf->base.size, prot); + break; + } + case GASKET_IOCTL_UNMAP_BUFFER: { + struct gasket_page_table_ioctl *pti = (void *)data; + struct gasket_page_table *pt; + + pt = gasket_page_table_get(sc, pti->page_table_index); + if (pt == NULL) { + error = EINVAL; + break; + } + error = gasket_page_table_unmap(pt, pti->device_address, + pti->size); + break; + } + default: + error = GASKET_IOCTL(sc->gs_dev, cmd, data, fflag); + break; + } +done: + sx_xunlock(&sc->gs_lock); + + return (error); +} + +void +gasket_interrupt_notify(struct gasket_softc *sc, u_int index) +{ + struct gasket_interrupt_event *ev; + u_int next; + + if (sc == NULL || index >= sc->gs_desc->gd_ninterrupts) + return; + + mtx_lock(&sc->gs_evlock); + sc->gs_counts[index]++; + + next = (sc->gs_ev_head + 1) % GASKET_EVENT_RING; + if (next == sc->gs_ev_tail) { + sc->gs_ev_dropped++; + } else { + ev = &sc->gs_ring[sc->gs_ev_head]; + ev->gie_interrupt = index; + ev->gie_dropped = sc->gs_ev_dropped; + ev->gie_count = sc->gs_counts[index]; + sc->gs_ev_head = next; + sc->gs_ev_dropped = 0; + } + + if (sc->gs_ev_waiting) { + sc->gs_ev_waiting = false; + wakeup(&sc->gs_ring); + } + KNOTE_LOCKED(&sc->gs_note, 0); + mtx_unlock(&sc->gs_evlock); +} + +bool +gasket_coherent_bus_addr(struct gasket_softc *sc, bus_addr_t *addrp) +{ + bool present; + + if (sc == NULL) + return (false); + + sx_slock(&sc->gs_lock); + present = sc->gs_coherent_vaddr != NULL; + if (present) + *addrp = sc->gs_coherent_paddr; + sx_sunlock(&sc->gs_lock); + + return (present); +} + +struct gasket_page_table * +gasket_page_table_get(struct gasket_softc *sc, u_int index) +{ + + if (sc == NULL || index >= sc->gs_desc->gd_npage_tables) + return (NULL); + return (sc->gs_page_tables[index]); +} + +int +gasket_cdev_read(struct cdev *cdev, struct uio *uio, int ioflag) +{ + struct gasket_softc *sc = cdev->si_drv1; + struct gasket_interrupt_event ev; + int error; + + if (uio->uio_resid < (ssize_t)sizeof(ev)) + return (EINVAL); + + error = 0; + mtx_lock(&sc->gs_evlock); + while (sc->gs_ev_head == sc->gs_ev_tail) { + if (sc->gs_dying) { + mtx_unlock(&sc->gs_evlock); + return (ENXIO); + } + if ((ioflag & IO_NDELAY) != 0) { + mtx_unlock(&sc->gs_evlock); + return (EWOULDBLOCK); + } + sc->gs_ev_waiting = true; + error = msleep(&sc->gs_ring, &sc->gs_evlock, PCATCH, "gskev", + 0); + if (error != 0 && error != EWOULDBLOCK) { + mtx_unlock(&sc->gs_evlock); + return (error); + } + error = 0; + } + + while (sc->gs_ev_head != sc->gs_ev_tail && + uio->uio_resid >= (ssize_t)sizeof(ev)) { + ev = sc->gs_ring[sc->gs_ev_tail]; + sc->gs_ev_tail = (sc->gs_ev_tail + 1) % GASKET_EVENT_RING; + mtx_unlock(&sc->gs_evlock); + + error = uiomove(&ev, sizeof(ev), uio); + + mtx_lock(&sc->gs_evlock); + if (error != 0) + break; + } + mtx_unlock(&sc->gs_evlock); + + return (error); +} + +static void +gasket_kq_detach(struct knote *kn) +{ + struct gasket_softc *sc = kn->kn_hook; + + knlist_remove(&sc->gs_note, kn, 0); +} + +static int +gasket_kq_event(struct knote *kn, long hint __unused) +{ + struct gasket_softc *sc = kn->kn_hook; + u_int pending; + + pending = (sc->gs_ev_head + GASKET_EVENT_RING - sc->gs_ev_tail) % + GASKET_EVENT_RING; + kn->kn_data = (int64_t)pending * sizeof(struct gasket_interrupt_event); + + if (sc->gs_dying) { + kn->kn_flags |= EV_EOF; + return (1); + } + return (pending != 0); +} + +static const struct filterops gasket_read_filterops = { + .f_isfd = 1, + .f_detach = gasket_kq_detach, + .f_event = gasket_kq_event, +}; + +int +gasket_cdev_kqfilter(struct cdev *cdev, struct knote *kn) +{ + struct gasket_softc *sc = cdev->si_drv1; + + if (kn->kn_filter != EVFILT_READ) + return (EINVAL); + + kn->kn_fop = &gasket_read_filterops; + kn->kn_hook = sc; + knlist_add(&sc->gs_note, kn, 0); + + return (0); +} diff --git a/sys/dev/gasket/gasket_coherent.c b/sys/dev/gasket/gasket_coherent.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_coherent.c @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include "gasket_priv.h" + +struct gasket_coherent_load { + vm_paddr_t *pages; + bus_addr_t paddr; + int error; +}; + +static void +gasket_coherent_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct gasket_coherent_load *load = arg; + int i; + + if (error != 0) { + load->error = error; + return; + } + KASSERT(nsegs == 1, ("gasket: coherent alloc not contiguous")); + load->paddr = segs[0].ds_addr; + for (i = 0; i < nsegs; i++) + load->pages[i] = segs[0].ds_addr + i * PAGE_SIZE; +} + +int +gasket_coherent_free(struct gasket_softc *sc) +{ + + if (sc->gs_coherent_vaddr == NULL) + return (0); + + if (sc->gs_coherent_maps != 0) { + sc->gs_coherent_defer = true; + return (EBUSY); + } + + bus_dmamap_unload(sc->gs_coherent_tag, sc->gs_coherent_map); + bus_dmamem_free(sc->gs_coherent_tag, sc->gs_coherent_vaddr, + sc->gs_coherent_map); + bus_dma_tag_destroy(sc->gs_coherent_tag); + + free(sc->gs_coherent_pages, M_GASKET); + + sc->gs_coherent_tag = NULL; + sc->gs_coherent_map = NULL; + sc->gs_coherent_vaddr = NULL; + sc->gs_coherent_pages = NULL; + sc->gs_coherent_paddr = 0; + sc->gs_coherent_size = 0; + sc->gs_coherent_npages = 0; + sc->gs_coherent_gen++; + + return (0); +} + +int +gasket_coherent_alloc(struct gasket_softc *sc, u_int index, bus_size_t size) +{ + struct gasket_coherent_load load; + int error; + + if (size == 0 || (size & PAGE_MASK) != 0) + return (EINVAL); + + if (sc->gs_coherent_vaddr != NULL) { + error = gasket_coherent_free(sc); + if (error != 0) + return (error); + } + + sc->gs_coherent_npages = (u_int)(size >> PAGE_SHIFT); + sc->gs_coherent_pages = mallocarray(sc->gs_coherent_npages, + sizeof(*sc->gs_coherent_pages), M_GASKET, M_WAITOK | M_ZERO); + + error = bus_dma_tag_create(bus_get_dma_tag(sc->gs_dev), PAGE_SIZE, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, size, 1, size, + 0, NULL, NULL, &sc->gs_coherent_tag); + if (error != 0) + goto fail; + + error = bus_dmamem_alloc(sc->gs_coherent_tag, &sc->gs_coherent_vaddr, + BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, + &sc->gs_coherent_map); + if (error != 0) + goto fail; + + load.pages = sc->gs_coherent_pages; + load.error = 0; + error = bus_dmamap_load(sc->gs_coherent_tag, sc->gs_coherent_map, + sc->gs_coherent_vaddr, size, gasket_coherent_cb, &load, + BUS_DMA_NOWAIT); + if (error == 0) + error = load.error; + if (error != 0) + goto fail; + + sc->gs_coherent_paddr = load.paddr; + sc->gs_coherent_size = size; + sc->gs_coherent_pt = index; + sc->gs_coherent_gen++; + + return (0); + +fail: + if (sc->gs_coherent_vaddr != NULL) { + bus_dmamem_free(sc->gs_coherent_tag, sc->gs_coherent_vaddr, + sc->gs_coherent_map); + sc->gs_coherent_vaddr = NULL; + } + if (sc->gs_coherent_tag != NULL) { + bus_dma_tag_destroy(sc->gs_coherent_tag); + sc->gs_coherent_tag = NULL; + } + free(sc->gs_coherent_pages, M_GASKET); + sc->gs_coherent_pages = NULL; + sc->gs_coherent_npages = 0; + + return (error); +} diff --git a/sys/dev/gasket/gasket_core.c b/sys/dev/gasket/gasket_core.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_core.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include "gasket_priv.h" + +MALLOC_DEFINE(M_GASKET, "gasket", "Gasket accelerator framework"); + +static struct cdevsw gasket_cdevsw = { + .d_version = D_VERSION, + .d_open = gasket_cdev_open, + .d_read = gasket_cdev_read, + .d_kqfilter = gasket_cdev_kqfilter, + .d_close = gasket_cdev_close, + .d_ioctl = gasket_cdev_ioctl, + .d_mmap_single = gasket_cdev_mmap_single, + .d_flags = D_TRACKCLOSE, + .d_name = "gasket", +}; + +int +gasket_wait8(struct resource *csr, bus_size_t off, uint64_t mask, uint64_t val, + int retries, int delay_ms) +{ + int i; + + for (i = 0; i < retries; i++) { + if ((gasket_rd8(csr, off) & mask) == val) + return (0); + pause_sbt("gskwait", mstosbt(delay_ms), 0, C_HARDCLOCK); + } + return (ETIMEDOUT); +} + +bool +gasket_is_open(struct gasket_softc *sc) +{ + bool open; + + sx_slock(&sc->gs_lock); + open = sc->gs_opens > 0; + sx_sunlock(&sc->gs_lock); + + return (open); +} + +int +gasket_suspend_begin(struct gasket_softc *sc) +{ + + if (sc == NULL) + return (0); + + sx_xlock(&sc->gs_lock); + if (sc->gs_opens > 0 || sc->gs_coherent_maps != 0 || + sc->gs_bar_maps != 0) { + sx_xunlock(&sc->gs_lock); + return (EBUSY); + } + sc->gs_suspended = true; + sx_xunlock(&sc->gs_lock); + + return (0); +} + +void +gasket_resume(struct gasket_softc *sc) +{ + + if (sc == NULL) + return; + + sx_xlock(&sc->gs_lock); + sc->gs_suspended = false; + sx_xunlock(&sc->gs_lock); +} + +int +gasket_attach(device_t dev, const struct gasket_desc *desc, + struct gasket_softc **scp) +{ + struct make_dev_args mda; + struct gasket_softc *sc; + u_int i; + int error; + + KASSERT(desc->gd_csr != NULL, ("gasket: no control register window")); + KASSERT(desc->gd_name != NULL, ("gasket: no device name")); + + sc = malloc(sizeof(*sc), M_GASKET, M_WAITOK | M_ZERO); + sc->gs_dev = dev; + sc->gs_desc = desc; + sx_init(&sc->gs_lock, "gasket"); + mtx_init(&sc->gs_evlock, "gasket ev", NULL, MTX_DEF); + + knlist_init_mtx(&sc->gs_note, &sc->gs_evlock); + sc->gs_ring = mallocarray(GASKET_EVENT_RING, sizeof(*sc->gs_ring), + M_GASKET, M_WAITOK | M_ZERO); + sc->gs_counts = mallocarray(MAX(desc->gd_ninterrupts, 1), + sizeof(*sc->gs_counts), M_GASKET, M_WAITOK | M_ZERO); + + if (desc->gd_npage_tables != 0) { + sc->gs_page_tables = mallocarray(desc->gd_npage_tables, + sizeof(*sc->gs_page_tables), M_GASKET, M_WAITOK | M_ZERO); + for (i = 0; i < desc->gd_npage_tables; i++) { + error = gasket_page_table_init(dev, desc->gd_csr, + &desc->gd_page_tables[i], &sc->gs_page_tables[i]); + if (error != 0) { + device_printf(dev, + "cannot init page table %u: %d\n", i, + error); + goto fail; + } + } + } + + make_dev_args_init(&mda); + mda.mda_devsw = &gasket_cdevsw; + mda.mda_unit = device_get_unit(dev); + mda.mda_uid = UID_ROOT; + mda.mda_gid = GID_WHEEL; + mda.mda_mode = 0660; + mda.mda_si_drv1 = sc; + error = make_dev_s(&mda, &sc->gs_cdev, "%s%d", desc->gd_name, + device_get_unit(dev)); + if (error != 0) { + device_printf(dev, "cannot create device node: %d\n", error); + goto fail; + } + + *scp = sc; + return (0); + +fail: + for (i = 0; i < desc->gd_npage_tables; i++) { + if (sc->gs_page_tables != NULL && + sc->gs_page_tables[i] != NULL) + gasket_page_table_free(sc->gs_page_tables[i]); + } + free(sc->gs_page_tables, M_GASKET); + free(sc->gs_counts, M_GASKET); + free(sc->gs_ring, M_GASKET); + knlist_destroy(&sc->gs_note); + mtx_destroy(&sc->gs_evlock); + sx_destroy(&sc->gs_lock); + free(sc, M_GASKET); + return (error); +} + +int +gasket_detach_begin(struct gasket_softc *sc) +{ + + if (sc == NULL) + return (0); + + sx_xlock(&sc->gs_lock); + if (sc->gs_opens > 0 || sc->gs_coherent_maps != 0 || + sc->gs_bar_maps != 0) { + sx_xunlock(&sc->gs_lock); + return (EBUSY); + } + if (sc->gs_stuck) { + sx_xunlock(&sc->gs_lock); + return (EBUSY); + } + sc->gs_dying = true; + sx_xunlock(&sc->gs_lock); + + return (0); +} + +void +gasket_detach_abort(struct gasket_softc *sc) +{ + + if (sc == NULL) + return; + + sx_xlock(&sc->gs_lock); + sc->gs_dying = false; + sx_xunlock(&sc->gs_lock); +} + +void +gasket_detach_finish(struct gasket_softc *sc) +{ + u_int i; + + if (sc == NULL) + return; + + mtx_lock(&sc->gs_evlock); + sc->gs_ev_waiting = false; + wakeup(&sc->gs_ring); + KNOTE_LOCKED(&sc->gs_note, 0); + mtx_unlock(&sc->gs_evlock); + + destroy_dev(sc->gs_cdev); + + /* Not knlist_detach(): it frees a knlist embedded in the softc. */ + knlist_clear(&sc->gs_note, 0); + knlist_destroy(&sc->gs_note); + + sx_xlock(&sc->gs_lock); + gasket_coherent_free(sc); + sx_xunlock(&sc->gs_lock); + + for (i = 0; i < sc->gs_desc->gd_npage_tables; i++) + gasket_page_table_free(sc->gs_page_tables[i]); + free(sc->gs_page_tables, M_GASKET); + free(sc->gs_counts, M_GASKET); + free(sc->gs_ring, M_GASKET); + mtx_destroy(&sc->gs_evlock); + sx_destroy(&sc->gs_lock); + free(sc, M_GASKET); +} + +static int +gasket_modevent(module_t mod __unused, int type, void *data __unused) +{ + switch (type) { + case MOD_LOAD: + case MOD_UNLOAD: + return (0); + default: + return (EOPNOTSUPP); + } +} + +static moduledata_t gasket_mod = { + "gasket", + gasket_modevent, + NULL +}; + +DECLARE_MODULE(gasket, gasket_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); +MODULE_VERSION(gasket, 1); diff --git a/sys/dev/gasket/gasket_dma.c b/sys/dev/gasket/gasket_dma.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_dma.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include +#include + +#include "gasket_pt_priv.h" + +void +gasket_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct gasket_dma_load *load; + + load = arg; + if (error != 0) { + load->gdl_error = error; + return; + } + if (nsegs != 1) { + load->gdl_error = EFBIG; + return; + } + load->gdl_addr = segs[0].ds_addr; + load->gdl_error = 0; +} + +int +gasket_dma_load_page(struct gasket_page_table *pt, bus_dmamap_t map, + vm_page_t page, bus_addr_t *addrp) +{ + struct gasket_dma_load load; + struct memdesc mem; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + load.gdl_addr = 0; + load.gdl_error = ENOMEM; + + /* memdesc_vmpages() would keep a pointer into this stack frame. */ + mem = memdesc_paddr(VM_PAGE_TO_PHYS(page), PAGE_SIZE); + error = bus_dmamap_load_mem(pt->gpt_dma_tag, map, &mem, + gasket_dma_load_cb, &load, BUS_DMA_NOWAIT); + if (error != 0) + return (ENOMEM); + if (load.gdl_error != 0) { + bus_dmamap_unload(pt->gpt_dma_tag, map); + return (ENOMEM); + } + if ((load.gdl_addr & PAGE_MASK) != 0) { + bus_dmamap_unload(pt->gpt_dma_tag, map); + return (EFAULT); + } + + *addrp = load.gdl_addr; + return (0); +} + +/* A page the host writes is one the device reads, and the other way round. */ +static bus_dmasync_op_t +gasket_dma_sync_op(int prot, bus_dmasync_op_t write, bus_dmasync_op_t read) +{ + bus_dmasync_op_t op; + + op = 0; + if ((prot & VM_PROT_READ) != 0) + op |= write; + if ((prot & VM_PROT_WRITE) != 0) + op |= read; + return (op); +} + +#define gasket_dma_pre_sync(prot) \ + gasket_dma_sync_op((prot), BUS_DMASYNC_PREWRITE, BUS_DMASYNC_PREREAD) +#define gasket_dma_post_sync(prot) \ + gasket_dma_sync_op((prot), BUS_DMASYNC_POSTWRITE, BUS_DMASYNC_POSTREAD) + +int +gasket_mapping_load_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping, vm_page_t page, bus_addr_t bus, int prot, + enum gasket_map_kind kind) +{ + bus_dmasync_op_t op; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + KASSERT(!mapping->gm_mapped, ("gasket: mapping already in use")); + KASSERT(kind != GASKET_MAP_NONE, ("gasket: mapping with no kind")); + + if (kind == GASKET_MAP_COHERENT) { + mapping->gm_page = NULL; + mapping->gm_dma_addr = bus; + mapping->gm_prot = prot; + mapping->gm_kind = kind; + return (0); + } + + error = bus_dmamap_create(pt->gpt_dma_tag, 0, + &mapping->gm_dma_map); + if (error != 0) + return (ENOMEM); + error = gasket_dma_load_page(pt, mapping->gm_dma_map, page, + &mapping->gm_dma_addr); + if (error != 0) { + bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map); + memset(mapping, 0, sizeof(*mapping)); + return (error); + } + + op = gasket_dma_pre_sync(prot); + if (op != 0) + bus_dmamap_sync(pt->gpt_dma_tag, mapping->gm_dma_map, op); + mapping->gm_page = page; + mapping->gm_prot = prot; + mapping->gm_kind = kind; + return (0); +} + +void +gasket_mapping_discard_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping) +{ + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + KASSERT(!mapping->gm_mapped, + ("gasket: discarding a published mapping")); + + if (mapping->gm_kind == GASKET_MAP_HOST) { + bus_dmamap_unload(pt->gpt_dma_tag, mapping->gm_dma_map); + bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map); + } + memset(mapping, 0, sizeof(*mapping)); +} + +void +gasket_mapping_unload_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping) +{ + vm_page_t page; + bus_dmasync_op_t op; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + if (!mapping->gm_mapped) + return; + + if (mapping->gm_kind == GASKET_MAP_COHERENT) { + KASSERT(pt->gpt_num_active != 0, + ("gasket: active page count underflow")); + pt->gpt_num_active--; + memset(mapping, 0, sizeof(*mapping)); + return; + } + + op = gasket_dma_post_sync(mapping->gm_prot); + if (op != 0) + bus_dmamap_sync(pt->gpt_dma_tag, mapping->gm_dma_map, op); + bus_dmamap_unload(pt->gpt_dma_tag, mapping->gm_dma_map); + bus_dmamap_destroy(pt->gpt_dma_tag, mapping->gm_dma_map); + + /* Dirtied at acquisition; busying it here trips an assertion. */ + page = mapping->gm_page; + vm_page_unhold_pages(&page, 1); + + KASSERT(pt->gpt_num_active != 0, + ("gasket: active page count underflow")); + pt->gpt_num_active--; + memset(mapping, 0, sizeof(*mapping)); +} diff --git a/sys/dev/gasket/gasket_if.m b/sys/dev/gasket/gasket_if.m new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_if.m @@ -0,0 +1,94 @@ +# +# Copyright (c) 2026 Abdelkader Boudih +# +# SPDX-License-Identifier: BSD-2-Clause +# + +# Every method is called with the core's device lock held, from a context +# that is allowed to sleep. + +#include +#include +#include + +#include + +INTERFACE gasket; + +CODE { + static int + gasket_default_reset(device_t dev) + { + return (0); + } + + static int + gasket_default_open(device_t dev) + { + return (0); + } + + static void + gasket_default_close(device_t dev) + { + } + + static int + gasket_default_ioctl(device_t dev, u_long cmd, caddr_t data, int fflag) + { + return (ENOTTY); + } + + static int + gasket_default_status(device_t dev) + { + return (GASKET_STATUS_ALIVE); + } +}; + +# +# Called on first open and from GASKET_IOCTL_RESET. +# +METHOD int reset { + device_t dev; +} DEFAULT gasket_default_reset; + +# +# Stop all DMA durably: a register window mapping outlives the descriptor it +# was made through, so pausing the engine is not enough, the core must be +# held in reset. The core releases no host pages if this fails. +# +METHOD int quiesce { + device_t dev; +} DEFAULT gasket_default_reset; + +# +# Power policy only; a driver may make this a no-op, so it is never relied +# on for DMA safety. +# +METHOD int enter_reset { + device_t dev; +} DEFAULT gasket_default_reset; + +METHOD int quit_reset { + device_t dev; +} DEFAULT gasket_default_reset; + +METHOD int status { + device_t dev; +} DEFAULT gasket_default_status; + +METHOD int open { + device_t dev; +} DEFAULT gasket_default_open; + +METHOD void close { + device_t dev; +} DEFAULT gasket_default_close; + +METHOD int ioctl { + device_t dev; + u_long cmd; + caddr_t data; + int fflag; +} DEFAULT gasket_default_ioctl; diff --git a/sys/dev/gasket/gasket_ioctl.h b/sys/dev/gasket/gasket_ioctl.h new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_ioctl.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * Structure layouts match Google's Linux gasket driver so a ported user space + * library can share them. The command numbers use native FreeBSD encoding. + */ + +#ifndef _DEV_GASKET_GASKET_IOCTL_H_ +#define _DEV_GASKET_GASKET_IOCTL_H_ + +#include +#include + +struct gasket_page_table_ioctl { + uint64_t page_table_index; + uint64_t size; + uint64_t host_address; + uint64_t device_address; +}; + +#define GASKET_PTE_INUSE 0x1 +#define GASKET_DMA_BIDIRECTIONAL (0 << 1) +#define GASKET_DMA_TO_DEVICE (1 << 1) +#define GASKET_DMA_FROM_DEVICE (2 << 1) +#define GASKET_DMA_NONE (3 << 1) + +/* + * The padding is deliberate: without it this structure is 40 bytes where + * uint64_t aligns to 8 and 36 where it aligns to 4, and since the size is + * encoded into the command number, a 32 bit process would compute a different + * GASKET_IOCTL_MAP_BUFFER_FLAGS and get ENOTTY. + */ +struct gasket_page_table_ioctl_flags { + struct gasket_page_table_ioctl base; + uint32_t flags; + uint32_t _pad; +}; + +struct gasket_coherent_alloc_config_ioctl { + uint64_t page_table_index; + uint64_t enable; + uint64_t size; + uint64_t dma_address; +}; + +struct gasket_interrupt_event { + uint32_t gie_interrupt; + uint32_t gie_dropped; /* lost before this one */ + uint64_t gie_count; /* cumulative for this interrupt */ +}; + +/* + * Same macros and group as Google's shared gasket_ioctl.h, so a user space + * library including that header computes identical values. Only the encoding + * differs from Linux, and that falls out of . + */ +#define GASKET_IOC_GROUP 0xDC + +#define GASKET_IOCTL_RESET _IO(GASKET_IOC_GROUP, 0) +#define GASKET_IOCTL_NUMBER_PAGE_TABLES _IOR(GASKET_IOC_GROUP, 4, uint64_t) +#define GASKET_IOCTL_PAGE_TABLE_SIZE \ + _IOWR(GASKET_IOC_GROUP, 5, struct gasket_page_table_ioctl) +#define GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE \ + _IOWR(GASKET_IOC_GROUP, 6, struct gasket_page_table_ioctl) +#define GASKET_IOCTL_PARTITION_PAGE_TABLE \ + _IOW(GASKET_IOC_GROUP, 7, struct gasket_page_table_ioctl) +#define GASKET_IOCTL_MAP_BUFFER \ + _IOW(GASKET_IOC_GROUP, 8, struct gasket_page_table_ioctl) +#define GASKET_IOCTL_UNMAP_BUFFER \ + _IOW(GASKET_IOC_GROUP, 9, struct gasket_page_table_ioctl) +#define GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS _IO(GASKET_IOC_GROUP, 10) +#define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR \ + _IOWR(GASKET_IOC_GROUP, 11, struct gasket_coherent_alloc_config_ioctl) +#define GASKET_IOCTL_MAP_BUFFER_FLAGS \ + _IOW(GASKET_IOC_GROUP, 12, struct gasket_page_table_ioctl_flags) + +#ifdef _KERNEL +_Static_assert(sizeof(struct gasket_page_table_ioctl) == 32, + "gasket_page_table_ioctl changed size, breaking the ioctl ABI"); +_Static_assert(sizeof(struct gasket_page_table_ioctl_flags) == 40, + "gasket_page_table_ioctl_flags changed size, breaking the ioctl ABI"); +_Static_assert(sizeof(struct gasket_coherent_alloc_config_ioctl) == 32, + "gasket_coherent_alloc_config_ioctl changed size, breaking the ioctl ABI"); +_Static_assert(sizeof(struct gasket_interrupt_event) == 16, + "gasket_interrupt_event changed size, breaking the read(2) ABI"); +#endif + +#endif /* _DEV_GASKET_GASKET_IOCTL_H_ */ diff --git a/sys/dev/gasket/gasket_mmap.c b/sys/dev/gasket/gasket_mmap.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_mmap.c @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gasket_priv.h" + +static const struct cdev_pager_ops gasket_pager_ops; +static void gasket_map_release(struct gasket_maphandle *mh); +static int gasket_classify_and_hold(struct gasket_softc *sc, + vm_offset_t addr, vm_size_t size, vm_prot_t prot, + enum gasket_map_kind *kindp, vm_ooffset_t *coh_offp, + uint64_t *genp, vm_page_t *pages, u_int npages); + +static bool +gasket_region_contains(const struct gasket_desc *desc, vm_ooffset_t offset, + vm_size_t size) +{ + const struct gasket_region *gr; + u_int i; + + for (i = 0; i < desc->gd_nregions; i++) { + gr = &desc->gd_regions[i]; + if (offset >= gr->gr_offset && + size <= gr->gr_size && + offset - gr->gr_offset <= gr->gr_size - size) + return (true); + } + return (false); +} + +/* + * Validated here, not in d_mmap(): the device pager keys its object on the + * cdev, so later mmap(2) calls reuse it and never reach d_mmap(). + */ +static bool +gasket_coherent_contains(struct gasket_softc *sc, vm_ooffset_t offset, + vm_size_t size) +{ + const bus_size_t base = sc->gs_desc->gd_coherent_base; + + if (sc->gs_coherent_vaddr == NULL) + return (false); + return (offset >= base && size <= sc->gs_coherent_size && + (bus_size_t)offset - base <= sc->gs_coherent_size - size); +} + +int +gasket_cdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **objp, int nprot) +{ + struct gasket_softc *sc = cdev->si_drv1; + struct gasket_maphandle *mh; + vm_object_t obj; + bool coherent; + + if (size == 0 || (*offset & PAGE_MASK) != 0) + return (EINVAL); + + mh = malloc(sizeof(*mh), M_GASKET, M_WAITOK | M_ZERO); + + sx_xlock(&sc->gs_lock); + if (sc->gs_dying) { + sx_xunlock(&sc->gs_lock); + free(mh, M_GASKET); + return (ENXIO); + } + mh->gm_sc = sc; + if (gasket_region_contains(sc->gs_desc, *offset, size)) { + coherent = false; + mh->gm_coherent = false; + mh->gm_bar_pa = (vm_paddr_t)rman_get_start(sc->gs_desc->gd_csr); + sc->gs_bar_maps++; + } else if (gasket_coherent_contains(sc, *offset, size)) { + coherent = true; + mh->gm_coherent = true; + mh->gm_base = sc->gs_desc->gd_coherent_base; + mh->gm_pages = sc->gs_coherent_pages; + mh->gm_npages = sc->gs_coherent_npages; + mh->gm_gen = sc->gs_coherent_gen; + sc->gs_coherent_maps++; + } else { + sx_xunlock(&sc->gs_lock); + free(mh, M_GASKET); + return (EINVAL); + } + sx_xunlock(&sc->gs_lock); + + obj = cdev_pager_allocate(mh, OBJT_DEVICE, &gasket_pager_ops, size, + nprot, *offset, curthread->td_ucred); + if (obj == NULL) { + gasket_map_release(mh); + return (ENXIO); + } + + /* Control registers must not be cached; coherent memory stays WB. */ + if (!coherent) { + VM_OBJECT_WLOCK(obj); + vm_object_set_memattr(obj, VM_MEMATTR_UNCACHEABLE); + VM_OBJECT_WUNLOCK(obj); + } + + *objp = obj; + + return (0); +} + +static bool +gasket_paddr_for_offset(const struct gasket_maphandle *mh, vm_ooffset_t offset, + vm_paddr_t *paddr) +{ + u_int idx; + + if (!mh->gm_coherent) { + if (!gasket_region_contains(mh->gm_sc->gs_desc, offset, + PAGE_SIZE)) + return (false); + *paddr = mh->gm_bar_pa + offset; + return (true); + } + + if (mh->gm_pages == NULL || offset < mh->gm_base) + return (false); + idx = (u_int)((offset - mh->gm_base) / PAGE_SIZE); + if (idx >= mh->gm_npages) + return (false); + *paddr = mh->gm_pages[idx]; + + return (true); +} + +static void +gasket_map_release(struct gasket_maphandle *mh) +{ + struct gasket_softc *sc = mh->gm_sc; + bool finish_free; + + sx_xlock(&sc->gs_lock); + if (mh->gm_coherent) { + KASSERT(sc->gs_coherent_maps != 0, + ("gasket: coherent mapping count underflow")); + sc->gs_coherent_maps--; + finish_free = sc->gs_coherent_maps == 0 && + sc->gs_coherent_defer; + } else { + KASSERT(sc->gs_bar_maps != 0, + ("gasket: register mapping count underflow")); + sc->gs_bar_maps--; + finish_free = false; + } + if (finish_free) { + sc->gs_coherent_defer = false; + (void)gasket_coherent_free(sc); + } + sx_xunlock(&sc->gs_lock); + + free(mh, M_GASKET); +} + +static int +gasket_pager_ctor(void *handle __unused, vm_ooffset_t size __unused, + vm_prot_t prot __unused, vm_ooffset_t foff __unused, + struct ucred *cred __unused, u_short *color) +{ + + *color = 0; + + return (0); +} + +static void +gasket_pager_dtor(void *handle) +{ + + gasket_map_release(handle); +} + +static int +gasket_pager_fault(vm_object_t obj, vm_ooffset_t offset, int prot __unused, + vm_page_t *mres) +{ + const struct gasket_maphandle *mh = obj->handle; + vm_paddr_t paddr; + vm_page_t page; + + if (!gasket_paddr_for_offset(mh, offset, &paddr)) + return (VM_PAGER_FAIL); + + if (((*mres)->flags & PG_FICTITIOUS) != 0) { + vm_page_updatefake(*mres, paddr, obj->memattr); + vm_page_valid(*mres); + return (VM_PAGER_OK); + } + + VM_OBJECT_WUNLOCK(obj); + page = vm_page_getfake(paddr, obj->memattr); + VM_OBJECT_WLOCK(obj); + vm_page_replace(page, obj, (*mres)->pindex, *mres); + *mres = page; + vm_page_valid(page); + + return (VM_PAGER_OK); +} + +static const struct cdev_pager_ops gasket_pager_ops = { + .cdev_pg_ctor = gasket_pager_ctor, + .cdev_pg_dtor = gasket_pager_dtor, + .cdev_pg_fault = gasket_pager_fault, +}; + +static bool +gasket_object_is_coherent(struct gasket_softc *sc, vm_object_t obj, + uint64_t *genp) +{ + const struct gasket_maphandle *mh; + + if (obj == NULL || obj->type != OBJT_DEVICE || + obj->un_pager.devp.ops != &gasket_pager_ops) + return (false); + + mh = obj->un_pager.devp.handle; + if (mh == NULL || mh->gm_sc != sc || !mh->gm_coherent) + return (false); + if (genp != NULL) + *genp = mh->gm_gen; + + return (true); +} + +/* + * Classify from the address space, not from the pages a fault returns: a fake + * page is freed by its own pager regardless of wiring. + */ +static int +gasket_classify_range_locked(struct gasket_softc *sc, vm_offset_t addr, + vm_size_t size, enum gasket_map_kind *kindp, vm_ooffset_t *coh_offp, + uint64_t *genp) +{ + vm_map_t map = &curproc->p_vmspace->vm_map; + vm_map_entry_t entry; + vm_object_t obj; + vm_offset_t end, va; + int error = 0; + + end = addr + size; + *kindp = GASKET_MAP_NONE; + *coh_offp = 0; + + if (!vm_map_lookup_entry(map, addr, &entry)) + return (EFAULT); + + /* A submap entry's union holds a vm_map, not a vm_object. */ + if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) + return (EINVAL); + + if (gasket_object_is_coherent(sc, entry->object.vm_object, genp)) { + if (end > entry->end) + return (EINVAL); + *kindp = GASKET_MAP_COHERENT; + *coh_offp = entry->offset + (addr - entry->start) - + sc->gs_desc->gd_coherent_base; + return (0); + } + + for (va = addr; va < end; va = entry->end) { + if (!vm_map_lookup_entry(map, va, &entry)) { + error = EFAULT; + break; + } + if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) != 0) { + error = EINVAL; + break; + } + obj = entry->object.vm_object; + if (gasket_object_is_coherent(sc, obj, NULL)) { + error = EINVAL; + break; + } + if (obj != NULL && (obj->type == OBJT_DEVICE || + obj->type == OBJT_MGTDEVICE || obj->type == OBJT_PHYS || + obj->type == OBJT_SG)) { + error = EINVAL; + break; + } + } + if (error == 0) + *kindp = GASKET_MAP_HOST; + + return (error); +} + +int +gasket_map_buffer(struct gasket_softc *sc, struct gasket_page_table *pt, + vm_offset_t host_addr, uint64_t dev_addr, size_t size, vm_prot_t prot) +{ + enum gasket_map_kind kind; + vm_ooffset_t coh_off; + vm_page_t *pages; + uint64_t gen = 0; + u_int i, npages; + int error; + + if (size == 0 || (size & PAGE_MASK) != 0 || + (host_addr & PAGE_MASK) != 0 || host_addr + size < host_addr) + return (EINVAL); + npages = (u_int)(size >> PAGE_SHIFT); + if ((size >> PAGE_SHIFT) != npages) + return (EINVAL); + + if (npages > gasket_page_table_max_pages(pt)) + return (EINVAL); + + pages = mallocarray(npages, sizeof(*pages), M_GASKET, + M_WAITOK | M_ZERO); + + sx_xunlock(&sc->gs_lock); + error = gasket_classify_and_hold(sc, host_addr, size, prot, &kind, + &coh_off, &gen, pages, npages); + sx_xlock(&sc->gs_lock); + + if (error == 0) { + if (sc->gs_dying || sc->gs_stuck) + error = ENXIO; + else if (kind == GASKET_MAP_COHERENT && + (sc->gs_coherent_vaddr == NULL || + gen != sc->gs_coherent_gen)) { + error = ENXIO; + } else + error = gasket_page_table_map(pt, host_addr, dev_addr, + size, prot, kind, coh_off, pages); + + for (i = 0; i < npages; i++) { + if (pages[i] != NULL) + vm_page_unhold_pages(&pages[i], 1); + } + } + + free(pages, M_GASKET); + + return (error); +} + +#define GASKET_HOLD_ATTEMPTS 8 + +static int +gasket_classify_and_hold(struct gasket_softc *sc, vm_offset_t addr, + vm_size_t size, vm_prot_t prot, enum gasket_map_kind *kindp, + vm_ooffset_t *coh_offp, uint64_t *genp, vm_page_t *pages, u_int npages) +{ + vm_map_t map = &curproc->p_vmspace->vm_map; + pmap_t pmap = vmspace_pmap(curproc->p_vmspace); + vm_prot_t fault_prot; + vm_offset_t va; + u_int i; + int attempt, error; + + fault_prot = prot; + if ((fault_prot & VM_PROT_WRITE) != 0) + fault_prot |= VM_PROT_READ; + + for (attempt = 0; attempt < GASKET_HOLD_ATTEMPTS; attempt++) { + for (va = addr; va < addr + size; va += PAGE_SIZE) { + if (vm_fault(map, va, fault_prot, VM_FAULT_NORMAL, + NULL) != KERN_SUCCESS) + return (EFAULT); + } + + vm_map_lock_read(map); + error = gasket_classify_range_locked(sc, addr, size, kindp, + coh_offp, genp); + if (error != 0) { + vm_map_unlock_read(map); + return (error); + } + if (*kindp == GASKET_MAP_COHERENT) { + vm_map_unlock_read(map); + return (0); + } + + for (i = 0; i < npages; i++) { + pages[i] = pmap_extract_and_hold(pmap, + addr + (vm_offset_t)i * PAGE_SIZE, fault_prot); + if (pages[i] == NULL) + break; + if ((pages[i]->oflags & VPO_UNMANAGED) != 0 || + (pages[i]->flags & PG_FICTITIOUS) != 0) { + /* Classified, but not ours to keep. */ + vm_page_unhold_pages(pages, i + 1); + memset(pages, 0, npages * sizeof(*pages)); + vm_map_unlock_read(map); + return (EINVAL); + } + + /* DMA leaves no CPU dirty bit behind. */ + if ((prot & VM_PROT_WRITE) != 0 && + pages[i]->dirty != VM_PAGE_BITS_ALL) + vm_page_dirty(pages[i]); + } + vm_map_unlock_read(map); + + if (i == npages) + return (0); + + vm_page_unhold_pages(pages, i); + memset(pages, 0, npages * sizeof(*pages)); + } + + return (EFAULT); +} diff --git a/sys/dev/gasket/gasket_page_table.h b/sys/dev/gasket/gasket_page_table.h new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_page_table.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _DEV_GASKET_GASKET_PAGE_TABLE_H_ +#define _DEV_GASKET_GASKET_PAGE_TABLE_H_ + +#include +#include + +#include + +#include + +struct gasket_page_table; + +enum gasket_map_kind { + GASKET_MAP_NONE = 0, + GASKET_MAP_HOST, /* pinned user page, released on unmap */ + GASKET_MAP_COHERENT, /* driver's coherent buffer, not ours to free */ +}; + +int gasket_page_table_init(device_t dev, struct resource *csr, + const struct gasket_page_table_config *cfg, + struct gasket_page_table **ptp); +void gasket_page_table_free(struct gasket_page_table *pt); + +/* On success the entries own the pages and the array is cleared. */ +int gasket_page_table_map(struct gasket_page_table *pt, + vm_offset_t host_addr, uint64_t dev_addr, size_t size, int prot, + enum gasket_map_kind kind, vm_ooffset_t coh_off, vm_page_t *pages); +int gasket_page_table_unmap(struct gasket_page_table *pt, uint64_t dev_addr, + size_t size); +void gasket_page_table_unmap_all(struct gasket_page_table *pt); +int gasket_page_table_partition(struct gasket_page_table *pt, + u_int num_simple); + +void gasket_page_table_set_coherent(struct gasket_page_table *pt, + const vm_paddr_t *pages, u_int npages, bus_addr_t bus); + +void gasket_page_table_unmap_coherent(struct gasket_page_table *pt); +u_int gasket_page_table_num_entries(struct gasket_page_table *pt); +u_int gasket_page_table_num_simple_entries(struct gasket_page_table *pt); + +/* Larger than the entry count: each extended entry covers a subtable. */ +u_int gasket_page_table_max_pages(struct gasket_page_table *pt); +u_int gasket_page_table_num_active_pages(struct gasket_page_table *pt); + +#endif /* _DEV_GASKET_GASKET_PAGE_TABLE_H_ */ diff --git a/sys/dev/gasket/gasket_page_table.c b/sys/dev/gasket/gasket_page_table.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_page_table.c @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "gasket_pt_priv.h" + +CTASSERT(PAGE_SIZE == (1 << GASKET_PAGE_SHIFT)); +CTASSERT(GASKET_SUBTABLE_ENTRIES * sizeof(uint64_t) == PAGE_SIZE); + +static void gasket_garbage_collect_locked(struct gasket_page_table *pt); + +MALLOC_DEFINE(M_GASKET_PT, "gasket_pt", "Gasket device page tables"); + +static SYSCTL_NODE(_hw, OID_AUTO, gasket, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Gasket accelerator framework"); + +static u_int gasket_max_wired_pages = 131072; +SYSCTL_UINT(_hw_gasket, OID_AUTO, max_wired_pages, CTLFLAG_RWTUN, + &gasket_max_wired_pages, 0, + "Maximum pages one device page table may have mapped"); + +int +gasket_page_table_init(device_t dev, struct resource *csr, + const struct gasket_page_table_config *cfg, + struct gasket_page_table **ptp) +{ + struct gasket_page_table *pt; + bus_size_t csr_size, table_size; + u_int i; + int error; + + if (ptp == NULL) + return (EINVAL); + *ptp = NULL; + if (dev == NULL || csr == NULL || cfg == NULL || + cfg->gpt_total_entries == 0 || + !powerof2(cfg->gpt_total_entries) || + cfg->gpt_total_entries > (1U << GASKET_EXTENDED_LVL0_WIDTH) || + cfg->gpt_extended_bit < GASKET_EXTENDED_LVL0_WIDTH + + GASKET_EXTENDED_LVL0_SHIFT || + cfg->gpt_extended_bit >= 64) + return (EINVAL); + + csr_size = rman_get_size(csr); + table_size = (bus_size_t)cfg->gpt_total_entries * sizeof(uint64_t); + if (cfg->gpt_base_reg > csr_size || + table_size > csr_size - cfg->gpt_base_reg || + cfg->gpt_extended_reg > csr_size || + sizeof(uint64_t) > csr_size - cfg->gpt_extended_reg) + return (EINVAL); + + pt = malloc(sizeof(*pt), M_GASKET_PT, M_WAITOK | M_ZERO); + pt->gpt_entries = mallocarray(cfg->gpt_total_entries, + sizeof(*pt->gpt_entries), M_GASKET_PT, M_WAITOK | M_ZERO); + pt->gpt_dev = dev; + pt->gpt_csr = csr; + pt->gpt_cfg = *cfg; + pt->gpt_extended_flag = 1ULL << cfg->gpt_extended_bit; + pt->gpt_num_simple = cfg->gpt_total_entries; + sx_init(&pt->gpt_lock, "gasket page table"); + + error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, PAGE_SIZE, 1, + PAGE_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &pt->gpt_dma_tag); + if (error != 0) { + sx_destroy(&pt->gpt_lock); + free(pt->gpt_entries, M_GASKET_PT); + free(pt, M_GASKET_PT); + return (ENOMEM); + } + + sx_xlock(&pt->gpt_lock); + for (i = 0; i < cfg->gpt_total_entries; i++) + gasket_wr8(csr, gasket_l0_slot_offset(pt, i), 0); + gasket_wr8(csr, cfg->gpt_extended_reg, cfg->gpt_total_entries); + sx_xunlock(&pt->gpt_lock); + + *ptp = pt; + return (0); +} + +void +gasket_page_table_free(struct gasket_page_table *pt) +{ + u_int i; + + if (pt == NULL) + return; + + sx_xlock(&pt->gpt_lock); + gasket_unmap_all_locked(pt); + for (i = 0; i < pt->gpt_num_extended; i++) + gasket_free_subtable_locked(pt, i); + gasket_wr8(pt->gpt_csr, pt->gpt_cfg.gpt_extended_reg, + pt->gpt_cfg.gpt_total_entries); + sx_xunlock(&pt->gpt_lock); + + bus_dma_tag_destroy(pt->gpt_dma_tag); + sx_destroy(&pt->gpt_lock); + free(pt->gpt_entries, M_GASKET_PT); + free(pt, M_GASKET_PT); +} + +int +gasket_page_table_map(struct gasket_page_table *pt, vm_offset_t host_addr, + uint64_t dev_addr, size_t size, int prot, enum gasket_map_kind kind, + vm_ooffset_t coh_off, vm_page_t *pages) +{ + bus_addr_t *bus; + vm_ooffset_t limit; + bool coherent, extended; + u_int first, i, nmapped, npages; + int error; + + if (kind != GASKET_MAP_HOST && kind != GASKET_MAP_COHERENT) + return (EINVAL); + if (kind == GASKET_MAP_HOST && pages == NULL) + return (EINVAL); +#ifdef KMSAN + /* + * kmsan_bus_dmamap_sync() panics on anything but virtual address and + * mbuf descriptors, and a page mapped for a device is described + * physically. + */ + if (kind == GASKET_MAP_HOST) + return (EOPNOTSUPP); +#endif + + if (pt != NULL && size / PAGE_SIZE > gasket_page_table_max_pages(pt)) + return (EINVAL); + + if (pt == NULL || size == 0 || (size & PAGE_MASK) != 0 || + (host_addr & PAGE_MASK) != 0 || host_addr + size < host_addr || + (prot & ~(VM_PROT_READ | VM_PROT_WRITE)) != 0 || + (prot & (VM_PROT_READ | VM_PROT_WRITE)) == 0) + return (EINVAL); + npages = (u_int)(size >> GASKET_PAGE_SHIFT); + if ((size >> GASKET_PAGE_SHIFT) != npages) + return (EINVAL); + + sx_xlock(&pt->gpt_lock); + + if (pt->gpt_num_active + npages > gasket_max_wired_pages || + pt->gpt_num_active + npages < pt->gpt_num_active) { + error = ENOMEM; + goto out; + } + + error = gasket_validate_dev_range_locked(pt, dev_addr, npages, + &extended, &first); + if (error != 0) + goto out; + + if (extended) { + error = gasket_prepare_extended_locked(pt, first, npages); + if (error != 0) { + gasket_garbage_collect_locked(pt); + goto out; + } + } else if (!gasket_simple_range_free_locked(pt, first, npages)) { + error = EBUSY; + goto out; + } + + bus = mallocarray(npages, sizeof(*bus), M_GASKET_PT, + M_WAITOK | M_ZERO); + + coherent = kind == GASKET_MAP_COHERENT; + if (coherent) { + limit = (vm_ooffset_t)pt->gpt_coherent_npages * PAGE_SIZE; + if (pt->gpt_coherent_pages == NULL || coh_off > limit || + (vm_ooffset_t)size > limit - coh_off) { + error = EINVAL; + goto out_free; + } + for (i = 0; i < npages; i++) + bus[i] = pt->gpt_coherent_bus + (bus_addr_t)coh_off + + (bus_addr_t)i * PAGE_SIZE; + } + + nmapped = 0; + if (extended) + error = gasket_map_extended_locked(pt, first, pages, bus, + npages, prot, kind, &nmapped); + else + error = gasket_map_simple_locked(pt, first, pages, bus, + npages, prot, kind, &nmapped); + if (error != 0) { + if (extended) + gasket_unmap_extended_locked(pt, first, nmapped); + else + gasket_unmap_simple_locked(pt, first, nmapped); + } +out_free: + if (error != 0 && extended) + gasket_garbage_collect_locked(pt); + free(bus, M_GASKET_PT); + +out: + sx_xunlock(&pt->gpt_lock); + return (error); +} + +int +gasket_page_table_unmap(struct gasket_page_table *pt, uint64_t dev_addr, + size_t size) +{ + bool extended; + u_int first, npages; + int error; + + if (pt == NULL || size == 0 || (size & PAGE_MASK) != 0) + return (EINVAL); + npages = (u_int)(size >> GASKET_PAGE_SHIFT); + if ((size >> GASKET_PAGE_SHIFT) != npages) + return (EINVAL); + + sx_xlock(&pt->gpt_lock); + error = gasket_validate_dev_range_locked(pt, dev_addr, npages, + &extended, &first); + if (error == 0) { + if (extended) + gasket_unmap_extended_locked(pt, first, npages); + else + gasket_unmap_simple_locked(pt, first, npages); + } + sx_xunlock(&pt->gpt_lock); + return (error); +} + +void +gasket_page_table_unmap_all(struct gasket_page_table *pt) +{ + + if (pt == NULL) + return; + + sx_xlock(&pt->gpt_lock); + gasket_unmap_all_locked(pt); + sx_xunlock(&pt->gpt_lock); +} + +void +gasket_page_table_set_coherent(struct gasket_page_table *pt, + const vm_paddr_t *pages, u_int npages, bus_addr_t bus) +{ + + if (pt == NULL) + return; + + sx_xlock(&pt->gpt_lock); + pt->gpt_coherent_pages = pages; + pt->gpt_coherent_npages = npages; + pt->gpt_coherent_bus = bus; + sx_xunlock(&pt->gpt_lock); +} + +void +gasket_page_table_unmap_coherent(struct gasket_page_table *pt) +{ + struct gasket_subtable *subtable; + u_int i, j; + + if (pt == NULL) + return; + + sx_xlock(&pt->gpt_lock); + for (i = 0; i < pt->gpt_num_simple; i++) { + if (pt->gpt_entries[i].gle_simple.gm_mapped && + pt->gpt_entries[i].gle_simple.gm_kind == + GASKET_MAP_COHERENT) + gasket_unmap_simple_locked(pt, i, 1); + } + for (i = 0; i < pt->gpt_num_extended; i++) { + subtable = pt->gpt_entries[pt->gpt_num_simple + i].gle_subtable; + if (subtable == NULL) + continue; + for (j = 0; j < GASKET_SUBTABLE_ENTRIES; j++) { + if (subtable->gst_entries[j].gm_mapped && + subtable->gst_entries[j].gm_kind == + GASKET_MAP_COHERENT) + gasket_unmap_extended_locked(pt, + i * GASKET_SUBTABLE_ENTRIES + j, 1); + } + } + sx_xunlock(&pt->gpt_lock); +} + +static void +gasket_garbage_collect_locked(struct gasket_page_table *pt) +{ + struct gasket_subtable *subtable; + u_int lvl0; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + for (lvl0 = 0; lvl0 < pt->gpt_num_extended; lvl0++) { + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + if (subtable == NULL) + continue; + if (gasket_mapping_range_free_locked(pt, subtable->gst_entries, + GASKET_SUBTABLE_ENTRIES)) + gasket_free_subtable_locked(pt, lvl0); + } +} + +int +gasket_page_table_partition(struct gasket_page_table *pt, u_int num_simple) +{ + u_int i, start; + + if (pt == NULL) + return (EINVAL); + + sx_xlock(&pt->gpt_lock); + if (num_simple > pt->gpt_cfg.gpt_total_entries) { + sx_xunlock(&pt->gpt_lock); + return (EINVAL); + } + + gasket_garbage_collect_locked(pt); + + start = MIN(pt->gpt_num_simple, num_simple); + for (i = start; i < pt->gpt_cfg.gpt_total_entries; i++) { + if (pt->gpt_entries[i].gle_simple.gm_mapped || + pt->gpt_entries[i].gle_subtable != NULL) { + device_printf(pt->gpt_dev, + "page table entry %u is still in use\n", i); + sx_xunlock(&pt->gpt_lock); + return (EBUSY); + } + } + + pt->gpt_num_simple = num_simple; + pt->gpt_num_extended = pt->gpt_cfg.gpt_total_entries - num_simple; + gasket_wr8(pt->gpt_csr, pt->gpt_cfg.gpt_extended_reg, num_simple); + sx_xunlock(&pt->gpt_lock); + + return (0); +} + +u_int +gasket_page_table_num_entries(struct gasket_page_table *pt) +{ + + if (pt == NULL) + return (0); + return (pt->gpt_num_simple + pt->gpt_num_extended); +} + +u_int +gasket_page_table_num_simple_entries(struct gasket_page_table *pt) +{ + u_int n; + + if (pt == NULL) + return (0); + sx_slock(&pt->gpt_lock); + n = pt->gpt_num_simple; + sx_sunlock(&pt->gpt_lock); + + return (n); +} + +u_int +gasket_page_table_max_pages(struct gasket_page_table *pt) +{ + u_int n; + + if (pt == NULL) + return (0); + sx_slock(&pt->gpt_lock); + n = pt->gpt_num_simple + + pt->gpt_num_extended * GASKET_SUBTABLE_ENTRIES; + sx_sunlock(&pt->gpt_lock); + + return (n); +} + +u_int +gasket_page_table_num_active_pages(struct gasket_page_table *pt) +{ + u_int nactive; + + if (pt == NULL) + return (0); + + sx_slock(&pt->gpt_lock); + nactive = pt->gpt_num_active; + sx_sunlock(&pt->gpt_lock); + return (nactive); +} diff --git a/sys/dev/gasket/gasket_priv.h b/sys/dev/gasket/gasket_priv.h new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_priv.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _DEV_GASKET_GASKET_PRIV_H_ +#define _DEV_GASKET_GASKET_PRIV_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct proc; + +MALLOC_DECLARE(M_GASKET); + +#define GASKET_EVENT_RING 256 + +/* Lock order: gs_lock -> gpt_lock, vm_map -> VM object; never together. */ + +struct gasket_maphandle { + struct gasket_softc *gm_sc; + bool gm_coherent; + vm_paddr_t gm_bar_pa; + bus_size_t gm_base; + const vm_paddr_t *gm_pages; + u_int gm_npages; + uint64_t gm_gen; +}; + +struct gasket_softc { + device_t gs_dev; + const struct gasket_desc *gs_desc; + struct cdev *gs_cdev; + struct sx gs_lock; + int gs_opens; + bool gs_dying; + + struct gasket_page_table **gs_page_tables; + + bus_dma_tag_t gs_coherent_tag; + bus_dmamap_t gs_coherent_map; + void *gs_coherent_vaddr; + vm_paddr_t *gs_coherent_pages; /* CPU side */ + bus_addr_t gs_coherent_paddr; /* device */ + bus_size_t gs_coherent_size; + u_int gs_coherent_npages; + uint64_t gs_coherent_gen; + + u_int gs_coherent_maps; + u_int gs_bar_maps; + u_int gs_coherent_pt; + bool gs_coherent_defer; + bool gs_suspended; + + struct proc *gs_owner; + u_int gs_writers; + /* Set when a quiesce failed and the mappings had to be retained. */ + bool gs_stuck; + + struct mtx gs_evlock; + struct knlist gs_note; + struct gasket_interrupt_event *gs_ring; + uint64_t *gs_counts; + u_int gs_ev_head; + u_int gs_ev_tail; + uint32_t gs_ev_dropped; + bool gs_ev_waiting; +}; + +/* cdevsw entry points, defined in gasket_chardev.c / gasket_mmap.c. */ +d_open_t gasket_cdev_open; +d_read_t gasket_cdev_read; +d_kqfilter_t gasket_cdev_kqfilter; +d_close_t gasket_cdev_close; +d_ioctl_t gasket_cdev_ioctl; +d_mmap_single_t gasket_cdev_mmap_single; + +/* gasket_coherent.c, shared with gasket_chardev.c and gasket_mmap.c. */ +int gasket_coherent_alloc(struct gasket_softc *sc, u_int index, + bus_size_t size); +int gasket_coherent_free(struct gasket_softc *sc); + +/* gasket_mmap.c, shared with gasket_chardev.c's ioctl handler. */ +int gasket_map_buffer(struct gasket_softc *sc, struct gasket_page_table *pt, + vm_offset_t host_addr, uint64_t dev_addr, size_t size, + vm_prot_t prot); + +#endif /* _DEV_GASKET_GASKET_PRIV_H_ */ diff --git a/sys/dev/gasket/gasket_pt_priv.h b/sys/dev/gasket/gasket_pt_priv.h new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_pt_priv.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _DEV_GASKET_GASKET_PT_PRIV_H_ +#define _DEV_GASKET_GASKET_PT_PRIV_H_ + +#include +#include + +#include + +#define GASKET_PAGE_SHIFT 12 +#define GASKET_EXTENDED_LVL0_SHIFT 21 +#define GASKET_EXTENDED_LVL0_WIDTH 13 +#define GASKET_SUBTABLE_ENTRIES 512 +#define GASKET_PTE_VALID 0x1 + +MALLOC_DECLARE(M_GASKET_PT); + +struct gasket_mapping { + vm_page_t gm_page; + bus_dmamap_t gm_dma_map; + bus_addr_t gm_dma_addr; + int gm_prot; + enum gasket_map_kind gm_kind; + bool gm_mapped; +}; + +struct gasket_subtable { + uint64_t *gst_slots; + bus_dmamap_t gst_dma_map; + bus_addr_t gst_dma_addr; + struct gasket_mapping gst_entries[GASKET_SUBTABLE_ENTRIES]; +}; + +struct gasket_l0_entry { + struct gasket_mapping gle_simple; + struct gasket_subtable *gle_subtable; +}; + +struct gasket_page_table { + device_t gpt_dev; + struct resource *gpt_csr; + struct gasket_page_table_config gpt_cfg; + struct gasket_l0_entry *gpt_entries; + bus_dma_tag_t gpt_dma_tag; + struct sx gpt_lock; + uint64_t gpt_extended_flag; + u_int gpt_num_simple; + u_int gpt_num_extended; + u_int gpt_num_active; + + const vm_paddr_t *gpt_coherent_pages; + u_int gpt_coherent_npages; + bus_addr_t gpt_coherent_bus; +}; + +struct gasket_dma_load { + bus_addr_t gdl_addr; + int gdl_error; +}; + +/* gasket_dma.c, shared with gasket_ptbl_alloc.c and gasket_page_table.c. */ +void gasket_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, + int error); +int gasket_dma_load_page(struct gasket_page_table *pt, bus_dmamap_t map, + vm_page_t page, bus_addr_t *addrp); +int gasket_mapping_load_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping, vm_page_t page, bus_addr_t bus, + int prot, enum gasket_map_kind kind); +void gasket_mapping_discard_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping); +void gasket_mapping_unload_locked(struct gasket_page_table *pt, + struct gasket_mapping *mapping); + +/* gasket_ptbl_alloc.c, shared with gasket_page_table.c. */ +bus_size_t gasket_l0_slot_offset(struct gasket_page_table *pt, u_int slot); +bool gasket_mapping_range_free_locked(struct gasket_page_table *pt, + struct gasket_mapping *entries, u_int nentries); +bool gasket_simple_range_free_locked(struct gasket_page_table *pt, + u_int first, u_int nentries); +int gasket_alloc_subtable_locked(struct gasket_page_table *pt, u_int lvl0); +void gasket_free_subtable_locked(struct gasket_page_table *pt, u_int lvl0); +int gasket_validate_dev_range_locked(struct gasket_page_table *pt, + uint64_t dev_addr, u_int npages, bool *extendedp, u_int *indexp); +int gasket_prepare_extended_locked(struct gasket_page_table *pt, + u_int first, u_int npages); +int gasket_map_simple_locked(struct gasket_page_table *pt, u_int first, + vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot, + enum gasket_map_kind kind, u_int *nmappedp); +void gasket_discard_extended_locked(struct gasket_page_table *pt, + u_int first, u_int nloaded); +int gasket_map_extended_locked(struct gasket_page_table *pt, u_int first, + vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot, + enum gasket_map_kind kind, u_int *nmappedp); +void gasket_unmap_simple_locked(struct gasket_page_table *pt, u_int first, + u_int npages); +void gasket_unmap_extended_locked(struct gasket_page_table *pt, u_int first, + u_int npages); +void gasket_unmap_all_locked(struct gasket_page_table *pt); + +#endif /* _DEV_GASKET_GASKET_PT_PRIV_H_ */ diff --git a/sys/dev/gasket/gasket_ptbl_alloc.c b/sys/dev/gasket/gasket_ptbl_alloc.c new file mode 100644 --- /dev/null +++ b/sys/dev/gasket/gasket_ptbl_alloc.c @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include + +#include "gasket_pt_priv.h" + +bus_size_t +gasket_l0_slot_offset(struct gasket_page_table *pt, u_int slot) +{ + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + return (pt->gpt_cfg.gpt_base_reg + (bus_size_t)slot * sizeof(uint64_t)); +} + +bool +gasket_mapping_range_free_locked(struct gasket_page_table *pt, + struct gasket_mapping *entries, u_int nentries) +{ + u_int i; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + for (i = 0; i < nentries; i++) { + if (entries[i].gm_mapped) + return (false); + } + return (true); +} + +bool +gasket_simple_range_free_locked(struct gasket_page_table *pt, u_int first, + u_int nentries) +{ + u_int i; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + for (i = 0; i < nentries; i++) { + if (pt->gpt_entries[first + i].gle_simple.gm_mapped) + return (false); + } + return (true); +} + +int +gasket_alloc_subtable_locked(struct gasket_page_table *pt, u_int lvl0) +{ + struct gasket_dma_load load; + struct gasket_l0_entry *l0e; + struct gasket_subtable *subtable; + bus_size_t slot; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + KASSERT(lvl0 < pt->gpt_num_extended, + ("gasket: extended level 0 index out of range")); + + l0e = &pt->gpt_entries[pt->gpt_num_simple + lvl0]; + if (l0e->gle_subtable != NULL) + return (0); + + subtable = malloc(sizeof(*subtable), M_GASKET_PT, M_WAITOK | M_ZERO); + error = bus_dmamem_alloc(pt->gpt_dma_tag, + (void **)&subtable->gst_slots, + BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, + &subtable->gst_dma_map); + if (error != 0) { + free(subtable, M_GASKET_PT); + return (ENOMEM); + } + + load.gdl_addr = 0; + load.gdl_error = ENOMEM; + error = bus_dmamap_load(pt->gpt_dma_tag, subtable->gst_dma_map, + subtable->gst_slots, PAGE_SIZE, gasket_dma_load_cb, &load, + BUS_DMA_NOWAIT); + if (error != 0 || load.gdl_error != 0 || + (load.gdl_addr & PAGE_MASK) != 0) { + if (error == 0) + bus_dmamap_unload(pt->gpt_dma_tag, + subtable->gst_dma_map); + bus_dmamem_free(pt->gpt_dma_tag, subtable->gst_slots, + subtable->gst_dma_map); + free(subtable, M_GASKET_PT); + return (ENOMEM); + } + subtable->gst_dma_addr = load.gdl_addr; + + bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map, + BUS_DMASYNC_PREWRITE); + slot = gasket_l0_slot_offset(pt, pt->gpt_num_simple + lvl0); + gasket_wr8(pt->gpt_csr, slot, + subtable->gst_dma_addr | GASKET_PTE_VALID); + l0e->gle_subtable = subtable; + return (0); +} + +void +gasket_free_subtable_locked(struct gasket_page_table *pt, u_int lvl0) +{ + struct gasket_l0_entry *l0e; + struct gasket_subtable *subtable; + bus_size_t slot; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + KASSERT(lvl0 < pt->gpt_num_extended, + ("gasket: extended level 0 index out of range")); + + l0e = &pt->gpt_entries[pt->gpt_num_simple + lvl0]; + subtable = l0e->gle_subtable; + if (subtable == NULL) + return; + KASSERT(gasket_mapping_range_free_locked(pt, subtable->gst_entries, + GASKET_SUBTABLE_ENTRIES), + ("gasket: freeing active subtable")); + + /* Posted write: read back before the subtable page is freed. */ + slot = gasket_l0_slot_offset(pt, pt->gpt_num_simple + lvl0); + gasket_wr8(pt->gpt_csr, slot, 0); + (void)gasket_rd8(pt->gpt_csr, slot); + bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(pt->gpt_dma_tag, subtable->gst_dma_map); + bus_dmamem_free(pt->gpt_dma_tag, subtable->gst_slots, + subtable->gst_dma_map); + free(subtable, M_GASKET_PT); + l0e->gle_subtable = NULL; +} + +int +gasket_validate_dev_range_locked(struct gasket_page_table *pt, + uint64_t dev_addr, u_int npages, bool *extendedp, u_int *indexp) +{ + uint64_t global, limit, raw; + u_int lvl0; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + if ((dev_addr & PAGE_MASK) != 0) + return (EINVAL); + + if ((dev_addr & pt->gpt_extended_flag) == 0) { + global = dev_addr >> GASKET_PAGE_SHIFT; + if (global >= pt->gpt_num_simple || + npages > pt->gpt_num_simple - global) + return (EINVAL); + *extendedp = false; + *indexp = (u_int)global; + return (0); + } + + raw = dev_addr & ~pt->gpt_extended_flag; + global = raw >> GASKET_PAGE_SHIFT; + limit = (uint64_t)pt->gpt_num_extended * + GASKET_SUBTABLE_ENTRIES; + lvl0 = (u_int)(global / GASKET_SUBTABLE_ENTRIES); + if ((raw & PAGE_MASK) != 0 || + raw >> (GASKET_EXTENDED_LVL0_WIDTH + + GASKET_EXTENDED_LVL0_SHIFT) != 0 || + lvl0 >= pt->gpt_num_extended || global >= limit || + npages > limit - global) + return (EINVAL); + + *extendedp = true; + *indexp = (u_int)global; + return (0); +} + +int +gasket_prepare_extended_locked(struct gasket_page_table *pt, u_int first, + u_int npages) +{ + struct gasket_subtable *subtable; + u_int len, lvl0, lvl1, remain; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + remain = npages; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + if (subtable != NULL && + !gasket_mapping_range_free_locked(pt, + &subtable->gst_entries[lvl1], len)) + return (EBUSY); + remain -= len; + lvl0++; + lvl1 = 0; + } + + remain = npages; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + error = gasket_alloc_subtable_locked(pt, lvl0); + if (error != 0) + return (error); + remain -= len; + lvl0++; + lvl1 = 0; + } + return (0); +} + +int +gasket_map_simple_locked(struct gasket_page_table *pt, u_int first, + vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot, + enum gasket_map_kind kind, u_int *nmappedp) +{ + struct gasket_mapping *mapping; + uint64_t pte; + u_int i; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + for (i = 0; i < npages; i++) { + mapping = &pt->gpt_entries[first + i].gle_simple; + error = gasket_mapping_load_locked(pt, mapping, + pages != NULL ? pages[i] : NULL, bus[i], prot, kind); + if (error != 0) { + while (i-- != 0) + gasket_mapping_discard_locked(pt, + &pt->gpt_entries[first + i].gle_simple); + *nmappedp = 0; + return (error); + } + } + + for (i = 0; i < npages; i++) { + mapping = &pt->gpt_entries[first + i].gle_simple; + pte = mapping->gm_dma_addr | GASKET_PTE_VALID; + gasket_wr8(pt->gpt_csr, + gasket_l0_slot_offset(pt, first + i), pte); + mapping->gm_mapped = true; + pt->gpt_num_active++; + if (pages != NULL) + pages[i] = NULL; + } + *nmappedp = npages; + return (0); +} + +void +gasket_discard_extended_locked(struct gasket_page_table *pt, u_int first, + u_int nloaded) +{ + struct gasket_subtable *subtable; + u_int i, len, lvl0, lvl1, remain; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + remain = nloaded; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + KASSERT(subtable != NULL, ("gasket: missing subtable")); + + for (i = 0; i < len; i++) + gasket_mapping_discard_locked(pt, + &subtable->gst_entries[lvl1 + i]); + remain -= len; + lvl0++; + lvl1 = 0; + } +} + +int +gasket_map_extended_locked(struct gasket_page_table *pt, u_int first, + vm_page_t *pages, bus_addr_t *bus, u_int npages, int prot, + enum gasket_map_kind kind, u_int *nmappedp) +{ + struct gasket_mapping *mapping; + struct gasket_subtable *subtable; + uint64_t pte; + u_int done, i, len, lvl0, lvl1, remain; + int error; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + done = 0; + remain = npages; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + KASSERT(subtable != NULL, ("gasket: missing subtable")); + + for (i = 0; i < len; i++) { + mapping = &subtable->gst_entries[lvl1 + i]; + error = gasket_mapping_load_locked(pt, mapping, + pages != NULL ? pages[done] : NULL, bus[done], prot, + kind); + if (error != 0) { + gasket_discard_extended_locked(pt, first, done); + *nmappedp = 0; + return (error); + } + done++; + } + remain -= len; + lvl0++; + lvl1 = 0; + } + + done = 0; + remain = npages; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + + for (i = 0; i < len; i++) { + mapping = &subtable->gst_entries[lvl1 + i]; + pte = mapping->gm_dma_addr | GASKET_PTE_VALID; + subtable->gst_slots[lvl1 + i] = pte; + mapping->gm_mapped = true; + pt->gpt_num_active++; + if (pages != NULL) + pages[done] = NULL; + done++; + } + bus_dmamap_sync(pt->gpt_dma_tag, subtable->gst_dma_map, + BUS_DMASYNC_PREWRITE); + + remain -= len; + lvl0++; + lvl1 = 0; + } + *nmappedp = npages; + return (0); +} + +void +gasket_unmap_simple_locked(struct gasket_page_table *pt, u_int first, + u_int npages) +{ + struct gasket_mapping *mapping; + u_int i; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + /* MMIO writes are posted: flush before releasing the pages. */ + for (i = 0; i < npages; i++) { + gasket_wr8(pt->gpt_csr, + gasket_l0_slot_offset(pt, first + i), 0); + } + if (npages != 0) + (void)gasket_rd8(pt->gpt_csr, gasket_l0_slot_offset(pt, first)); + + for (i = 0; i < npages; i++) { + mapping = &pt->gpt_entries[first + i].gle_simple; + gasket_mapping_unload_locked(pt, mapping); + } +} + +void +gasket_unmap_extended_locked(struct gasket_page_table *pt, u_int first, + u_int npages) +{ + struct gasket_subtable *subtable; + u_int i, len, lvl0, lvl1, remain; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + remain = npages; + lvl0 = first / GASKET_SUBTABLE_ENTRIES; + lvl1 = first % GASKET_SUBTABLE_ENTRIES; + while (remain != 0) { + len = MIN(remain, GASKET_SUBTABLE_ENTRIES - lvl1); + subtable = pt->gpt_entries[pt->gpt_num_simple + + lvl0].gle_subtable; + if (subtable != NULL) { + for (i = 0; i < len; i++) + subtable->gst_slots[lvl1 + i] = 0; + bus_dmamap_sync(pt->gpt_dma_tag, + subtable->gst_dma_map, BUS_DMASYNC_PREWRITE); + (void)gasket_rd8(pt->gpt_csr, gasket_l0_slot_offset(pt, + pt->gpt_num_simple + lvl0)); + + for (i = 0; i < len; i++) + gasket_mapping_unload_locked(pt, + &subtable->gst_entries[lvl1 + i]); + } + remain -= len; + lvl0++; + lvl1 = 0; + } +} + +void +gasket_unmap_all_locked(struct gasket_page_table *pt) +{ + u_int i; + + sx_assert(&pt->gpt_lock, SA_XLOCKED); + + gasket_unmap_simple_locked(pt, 0, pt->gpt_num_simple); + for (i = 0; i < pt->gpt_num_extended; i++) { + if (pt->gpt_entries[pt->gpt_num_simple + + i].gle_subtable != NULL) + gasket_unmap_extended_locked(pt, + i * GASKET_SUBTABLE_ENTRIES, + GASKET_SUBTABLE_ENTRIES); + } +} diff --git a/sys/modules/Makefile b/sys/modules/Makefile --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -132,6 +132,7 @@ ${_ftwd} \ fusefs \ ${_fxp} \ + ${_gasket} \ gem \ geom \ ${_glxiic} \ @@ -555,6 +556,9 @@ .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" SUBDIR+= linux .endif +.if ${MACHINE_CPUARCH} == "amd64" +_gasket= gasket +.endif .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" SUBDIR+= linux64 SUBDIR+= linux_common diff --git a/sys/modules/gasket/Makefile b/sys/modules/gasket/Makefile new file mode 100644 --- /dev/null +++ b/sys/modules/gasket/Makefile @@ -0,0 +1,14 @@ +.PATH: ${SRCTOP}/sys/dev/gasket + +KMOD= gasket +SRCS= gasket_core.c gasket_chardev.c gasket_mmap.c gasket_coherent.c +SRCS+= gasket_page_table.c gasket_ptbl_alloc.c gasket_dma.c +SRCS+= gasket_if.c gasket_if.h +SRCS+= device_if.h bus_if.h pci_if.h vnode_if.h + +# The device page table needs atomic 64 bit MMIO and 4 KiB pages. +.if ${MACHINE_CPUARCH} != "amd64" +.error gasket requires an architecture with bus_read_8()/bus_write_8() +.endif + +.include