diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist --- a/etc/mtree/BSD.include.dist +++ b/etc/mtree/BSD.include.dist @@ -205,6 +205,8 @@ .. agp .. + apex + .. ciss .. coreboot diff --git a/include/Makefile b/include/Makefile --- a/include/Makefile +++ b/include/Makefile @@ -189,6 +189,10 @@ GASKET= gasket_ioctl.h GASKETDIR= ${INCLUDEDIR}/dev/gasket +.PATH: ${SRCTOP}/sys/dev/apex +APEX= apex_ioctl.h +APEXDIR= ${INCLUDEDIR}/dev/apex + .PATH: ${SRCTOP}/sys/dev/nvme NVME= nvme.h NVMEDIR= ${INCLUDEDIR}/dev/nvme @@ -258,6 +262,7 @@ INCSGROUPS= INCS \ ACPICA \ AGP \ + APEX \ BSM \ CAM \ CAMATA \ diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -56,6 +56,7 @@ ${_bxe.4} \ ${_aout.4} \ ${_apic.4} \ + apex.4 \ aq.4 \ arcmsr.4 \ arswitch.4 \ diff --git a/share/man/man4/apex.4 b/share/man/man4/apex.4 new file mode 100644 --- /dev/null +++ b/share/man/man4/apex.4 @@ -0,0 +1,92 @@ +.\" +.\" Copyright (c) 2026 Abdelkader Boudih +.\" +.\" SPDX-License-Identifier: BSD-2-Clause +.\" +.Dd July 30, 2026 +.Dt APEX 4 +.Os +.Sh NAME +.Nm apex +.Nd Google Coral Edge TPU accelerator driver +.Sh SYNOPSIS +.Cd "device gasket" +.Cd "device apex" +.Sh DESCRIPTION +The +.Nm +driver supports the Google Edge TPU, the inference accelerator on Coral +PCI Express and M.2 modules. +It attaches through the +.Xr gasket 4 +framework and creates +.Pa /dev/apex0 , +which an Edge TPU runtime in user space opens to run a model. +.Pp +The core is held in reset while nothing has the device open, and a kernel +task divides its clock as the die warms. +.Sh HARDWARE +The +.Nm +driver supports the following devices: +.Pp +.Bl -bullet -compact +.It +Coral M.2 Accelerator A+E key +.It +Coral M.2 Accelerator B+M key +.It +Coral Mini PCIe Accelerator +.El +.Sh LOADER TUNABLES +Read once at attach, settable in +.Xr loader.conf 5 : +.Bl -tag -width indent +.It Va hw.apex.temp_poll_interval +Sampling interval in milliseconds, 5000 by default; zero disables throttling. +.It Va hw.apex.hw_temp_warn1 +.It Va hw.apex.hw_temp_warn2 +Hardware warning thresholds in milli-degrees C, both 100000 by default. +.It Va hw.apex.hw_temp_warn1_en +Drive INTR on the first threshold, off by default. +.It Va hw.apex.hw_temp_warn2_en +Drive SD_ALARM on the second, on by default. +.El +.Sh SYSCTL VARIABLES +.Bl -tag -width indent +.It Va hw.apex.allow_power_save +Enter the low power state on last close, on by default. +The core is held in reset either way. +.It Va hw.apex.allow_hw_clock_gating +Let the hardware gate the clock when idle, on by default. +.It Va hw.apex.allow_sw_clock_gating +Honor clock gating requests from user space, off by default. +.El +.Pp +Read-only per-device state is under +.Va dev.apex.%d . +.Sh FILES +.Bl -tag -width indent +.It Pa /dev/apex%d +Accelerator control device. +.El +.Sh SEE ALSO +.Xr gasket 4 , +.Xr pci 4 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 16.0 . +.Sh AUTHORS +.An Abdelkader Boudih Aq Mt seuros@FreeBSD.org +.Sh CAVEATS +Suspend and detach fail with +.Er EBUSY +while a consumer holds the device open or mapped, since both clear the on-chip +page table. +.Pp +The ioctl numbers use native +.Fx +encoding, so a library written against the Linux driver needs more than a +recompile. diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -70,6 +70,12 @@ # device gasket +# +# Google Apex Edge TPU, as found on Coral PCIe and M.2 modules. Requires +# the gasket framework above. +# +device apex + # # Microsemi smartpqi controllers. # These controllers have a SCSI-like interface, and require the diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -112,6 +112,7 @@ crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl crypto/openssl/ossl_aes_gcm.c optional ossl dev/amdgpio/amdgpio.c optional amdgpio +dev/apex/apex.c optional apex pci dev/gasket/gasket_core.c optional gasket dev/gasket/gasket_page_table.c optional gasket dev/gasket/gasket_if.m optional gasket diff --git a/sys/dev/apex/apex.c b/sys/dev/apex/apex.c new file mode 100644 --- /dev/null +++ b/sys/dev/apex/apex.c @@ -0,0 +1,1178 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * Register level behavior derived from Google's Linux gasket driver, + * github.com/google/gasket-driver. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include "gasket_if.h" + +#include +#include + +struct apex_softc; + +struct apex_intr_vec { + struct apex_softc *av_sc; + struct resource *av_irq; + void *av_cookie; + int av_rid; + u_int av_index; + counter_u64_t av_count; +}; + +struct apex_softc { + device_t sc_dev; + struct resource *sc_csr; + int sc_csr_rid; + struct gasket_softc *sc_gasket; + struct gasket_desc sc_desc; + + struct apex_intr_vec sc_vec[APEX_INTR_COUNT]; + int sc_nvecs; + + struct sx sc_lock; + struct timeout_task sc_temp_task; + bool sc_dying; + + /* ADC counts descend as heat rises. */ + int sc_adc_trip[3]; + int sc_temp_poll_interval; /* milliseconds */ + int sc_hw_warn1_adc; + int sc_hw_warn2_adc; + bool sc_hw_warn1_en; + bool sc_hw_warn2_en; +}; + +static const struct gasket_region apex_regions[] = { + { 0x40000, 0x1000 }, + { 0x44000, 0x1000 }, + { 0x48000, 0x1000 }, +}; + +static const struct gasket_page_table_config apex_page_tables[] = { + { + .gpt_id = 0, + .gpt_total_entries = APEX_PAGE_TABLE_ENTRIES, + .gpt_base_reg = APEX_KHIB_PAGE_TABLE, + .gpt_extended_reg = APEX_KHIB_EXTENDED_TABLE, + .gpt_extended_bit = APEX_EXTENDED_SHIFT, + }, +}; + +static const struct gasket_interrupt_desc apex_interrupts[] = { + { APEX_INTR_INSTR_QUEUE, 0x46018, GASKET_INTR_UNPACKED }, + { APEX_INTR_INPUT_ACTV_QUEUE, 0x46020, GASKET_INTR_UNPACKED }, + { APEX_INTR_PARAM_QUEUE, 0x46028, GASKET_INTR_UNPACKED }, + { APEX_INTR_OUTPUT_ACTV_QUEUE, 0x46030, GASKET_INTR_UNPACKED }, + { APEX_INTR_SC_HOST_0, 0x46038, GASKET_INTR_PACK_0 }, + { APEX_INTR_SC_HOST_1, 0x46038, GASKET_INTR_PACK_1 }, + { APEX_INTR_SC_HOST_2, 0x46038, GASKET_INTR_PACK_2 }, + { APEX_INTR_SC_HOST_3, 0x46038, GASKET_INTR_PACK_3 }, + { APEX_INTR_TOP_LEVEL_0, 0x46040, GASKET_INTR_PACK_0 }, + { APEX_INTR_TOP_LEVEL_1, 0x46040, GASKET_INTR_PACK_1 }, + { APEX_INTR_TOP_LEVEL_2, 0x46040, GASKET_INTR_PACK_2 }, + { APEX_INTR_TOP_LEVEL_3, 0x46040, GASKET_INTR_PACK_3 }, + { APEX_INTR_FATAL_ERR, 0x46048, GASKET_INTR_UNPACKED }, +}; + +static const struct pci_device_table apex_devs[] = { + { PCI_DEV(APEX_VENDOR_ID, APEX_DEVICE_ID), + PCI_DESCR("Google Apex Edge TPU") }, +}; + +static int apex_allow_power_save = 1; +static int apex_allow_sw_clock_gating = 0; +static int apex_allow_hw_clock_gating = 1; +static int apex_temp_poll_interval = 5000; + +static SYSCTL_NODE(_hw, OID_AUTO, apex, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Google Apex Edge TPU"); +SYSCTL_INT(_hw_apex, OID_AUTO, allow_power_save, CTLFLAG_RWTUN, + &apex_allow_power_save, 0, + "Enter the low power state when the last consumer closes the device"); +SYSCTL_INT(_hw_apex, OID_AUTO, allow_sw_clock_gating, CTLFLAG_RWTUN, + &apex_allow_sw_clock_gating, 0, "Honor software clock gating requests"); +SYSCTL_INT(_hw_apex, OID_AUTO, allow_hw_clock_gating, CTLFLAG_RWTUN, + &apex_allow_hw_clock_gating, 0, "Let the hardware gate the GCB clock"); +SYSCTL_INT(_hw_apex, OID_AUTO, temp_poll_interval, CTLFLAG_RDTUN, + &apex_temp_poll_interval, 0, + "Temperature poll interval in milliseconds, 0 disables throttling"); + +/* Trip points in milli-degrees C, ascending. */ +static int apex_trip_point_temp[3] = { 85000, 90000, 95000 }; +static int apex_hw_temp_warn1 = 100000; +static int apex_hw_temp_warn2 = 100000; +static int apex_hw_temp_warn1_en = 0; +static int apex_hw_temp_warn2_en = 1; + +SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn1, CTLFLAG_RDTUN, + &apex_hw_temp_warn1, 0, + "Hardware temperature warning 1 in milli-degrees C"); +SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn2, CTLFLAG_RDTUN, + &apex_hw_temp_warn2, 0, + "Hardware temperature warning 2 in milli-degrees C"); +SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn1_en, CTLFLAG_RDTUN, + &apex_hw_temp_warn1_en, 0, "Drive INTR on the warning 1 threshold"); +SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn2_en, CTLFLAG_RDTUN, + &apex_hw_temp_warn2_en, 0, "Drive SD_ALARM on the warning 2 threshold"); + +static int apex_enter_reset_locked(struct apex_softc *sc); +static int apex_enter_reset(struct apex_softc *sc); +static void apex_program_interrupts(struct apex_softc *sc); +static int apex_restore_msix(struct apex_softc *sc); + +static void +apex_sched_temp_task(struct apex_softc *sc) +{ + + if (sc->sc_temp_poll_interval <= 0) + return; + taskqueue_enqueue_timeout(taskqueue_thread, &sc->sc_temp_task, + (int)((int64_t)sc->sc_temp_poll_interval * hz / 1000)); +} + +static int +apex_enter_reset(struct apex_softc *sc) +{ + int error; + + sx_xlock(&sc->sc_lock); + error = apex_enter_reset_locked(sc); + sx_xunlock(&sc->sc_lock); + + return (error); +} + +/* The temperature ADC is a linear fit optimized for 25C to 100C. */ +static int +apex_adc_to_millic(int adc) +{ + return ((662 - adc) * 250 + 550); +} + +static int +apex_millic_to_adc(int millic) +{ + return ((550 - millic) / 250 + 662); +} + +static int +apex_read_temp_adc(struct apex_softc *sc) +{ + uint32_t val; + + val = gasket_rd4(sc->sc_csr, APEX_OMC0_DC); + return ((val >> APEX_TEMP_ADC_SHIFT) & + ((1 << APEX_TEMP_ADC_WIDTH) - 1)); +} + +static bool +apex_gcb_in_reset(struct apex_softc *sc) +{ + return ((gasket_rd4(sc->sc_csr, APEX_SCU_3) & + APEX_SCU3_CUR_RST_GCB) != 0); +} + +static int +apex_enter_reset_locked(struct apex_softc *sc) +{ + int error; + + sx_assert(&sc->sc_lock, SA_XLOCKED); + + gasket_rmw8(sc->sc_csr, APEX_IDLEGEN_IDLEREGISTER, APEX_IDLEGEN_DISABLE, + APEX_IDLEGEN_ENABLE_WIDTH, APEX_IDLEGEN_ENABLE_SHIFT); + gasket_wr8(sc->sc_csr, APEX_UHIB_DMA_PAUSE, APEX_DMA_PAUSE_REQUEST); + + error = gasket_wait8(sc->sc_csr, APEX_UHIB_DMA_PAUSED, 1, 1, + APEX_RESET_RETRY, APEX_RESET_DELAY_MS); + if (error != 0) { + device_printf(sc->sc_dev, "DMA did not quiesce within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + return (error); + } + + gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_ASSERT, + APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_GATED_GCB_ON, + APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_SCU_3, APEX_RAM_SD_FORCE, + APEX_FIELD_WIDTH, APEX_SCU3_FORCE_RAM_SD_SHIFT); + + error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_RAM_SD_DONE, + APEX_SCU3_RAM_SD_DONE, APEX_RESET_RETRY, APEX_RESET_DELAY_MS); + if (error != 0) { + device_printf(sc->sc_dev, + "RAM did not shut down within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + return (error); + } + + return (0); +} + +static int +apex_quit_reset_locked(struct apex_softc *sc) +{ + int error; + + sx_assert(&sc->sc_lock, SA_XLOCKED); + + gasket_rmw4(sc->sc_csr, APEX_SCU_3, APEX_RAM_SD_RELEASE, + APEX_FIELD_WIDTH, APEX_SCU3_FORCE_RAM_SD_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_GATED_GCB_OFF, + APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_RELEASE, + APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT); + + error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_RAM_SD_DONE, 0, + APEX_RESET_RETRY, APEX_RESET_DELAY_MS); + if (error != 0) { + device_printf(sc->sc_dev, "RAM did not enable within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + return (error); + } + + error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_CUR_RST_GCB, 0, + APEX_RESET_RETRY, APEX_RESET_DELAY_MS); + if (error != 0) { + device_printf(sc->sc_dev, + "GCB did not leave reset within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + return (error); + } + + gasket_rmw4(sc->sc_csr, APEX_SCU_3, + apex_allow_hw_clock_gating ? APEX_PWR_STATE_ALLOW_INACTIVE : + APEX_PWR_STATE_ALWAYS_ON, + APEX_SCU3_PWR_STATE_OVR_WIDTH, APEX_SCU3_PWR_STATE_OVR_SHIFT); + + error = apex_restore_msix(sc); + if (error != 0) + return (error); + apex_program_interrupts(sc); + + return (0); +} + +static void +apex_program_hw_temp_warnings(struct apex_softc *sc) +{ + gasket_rmw4(sc->sc_csr, APEX_OMC0_D4, sc->sc_hw_warn1_adc, + APEX_OMC_TEMP_WARN_WIDTH, APEX_OMC_TEMP_WARN_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, sc->sc_hw_warn2_adc, + APEX_OMC_TEMP_WARN_WIDTH, APEX_OMC_TEMP_WARN_SHIFT); + + /* Unconditional: firmware may have left a warning enabled. */ + gasket_rmw4(sc->sc_csr, APEX_OMC0_D4, sc->sc_hw_warn1_en ? 1 : 0, + APEX_BIT_WIDTH, APEX_OMC_TEMP_WARN_EN_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, sc->sc_hw_warn2_en ? 1 : 0, + APEX_BIT_WIDTH, APEX_OMC_TEMP_WARN_EN_SHIFT); +} + +static void +apex_enable_thermal_sensing(struct apex_softc *sc) +{ + gasket_rmw4(sc->sc_csr, APEX_OMC0_D0, APEX_OMC_TEMP_CTRL_ENABLE, + APEX_BIT_WIDTH, APEX_OMC_TEMP_ENABLE_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, APEX_OMC_TEMP_SENSOR_ENABLES, + APEX_OMC_TEMP_SENSOR_ENABLES_WIDTH, + APEX_OMC_TEMP_SENSOR_ENABLES_SHIFT); + /* The controller enable must follow ENAD/ENVR/ENBG by 100 us. */ + DELAY(100); + gasket_rmw4(sc->sc_csr, APEX_OMC0_DC, APEX_OMC_TEMP_CTRL_ENABLE, + APEX_BIT_WIDTH, APEX_OMC_TEMP_CTRL_ENABLE_SHIFT); +} + +static void +apex_temp_task_fn(void *arg, int pending __unused) +{ + struct apex_softc *sc = arg; + const uint32_t mask = 0x3u << APEX_SCU3_CLK_DIV_SHIFT; + uint32_t clk_div, tmp; + int adc, i, millic; + + sx_xlock(&sc->sc_lock); + if (sc->sc_dying) { + sx_xunlock(&sc->sc_lock); + return; + } + + adc = apex_read_temp_adc(sc); + for (i = nitems(sc->sc_adc_trip) - 1; i >= 0; i--) { + if (adc <= sc->sc_adc_trip[i]) + break; + } + clk_div = (uint32_t)(i + 1) << APEX_SCU3_CLK_DIV_SHIFT; + + tmp = gasket_rd4(sc->sc_csr, APEX_SCU_3); + if (clk_div != (tmp & mask)) { + gasket_wr4(sc->sc_csr, APEX_SCU_3, (tmp & ~mask) | clk_div); + millic = apex_adc_to_millic(adc); + device_printf(sc->sc_dev, "clock %sthrottled at %d.%03d C\n", + i == -1 ? "un" : "", millic / 1000, abs(millic) % 1000); + } + + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); +} + +static void +apex_intr(void *arg) +{ + struct apex_intr_vec *av = arg; + + counter_u64_add(av->av_count, 1); + gasket_interrupt_notify(av->av_sc->sc_gasket, av->av_index); +} + +/* The kernel HIB loses this across a GCB reset; called on every reset exit. */ +static void +apex_program_interrupts(struct apex_softc *sc) +{ + const struct gasket_interrupt_desc *gid; + u_int i, shift, width; + + sx_assert(&sc->sc_lock, SA_XLOCKED); + + if (sc->sc_nvecs == 0) + return; + + width = APEX_INTR_PACK_WIDTH; + for (i = 0; i < nitems(apex_interrupts); i++) { + gid = &apex_interrupts[i]; + if (gid->gid_index >= (u_int)sc->sc_nvecs) + continue; + + if (gid->gid_packing == GASKET_INTR_UNPACKED) { + gasket_wr8(sc->sc_csr, gid->gid_reg, gid->gid_index); + continue; + } + + shift = (gid->gid_packing - GASKET_INTR_PACK_0) * width; + gasket_rmw8(sc->sc_csr, gid->gid_reg, gid->gid_index, width, + shift); + } +} + +/* + * A GCB reset zeroes the MSI-X table, which shares device memory with the HIB. + */ +static int +apex_restore_msix(struct apex_softc *sc) +{ + struct apex_intr_vec *av; + int error, i; + + for (i = 0; i < sc->sc_nvecs; i++) { + av = &sc->sc_vec[i]; + if (av->av_irq == NULL) + continue; + + if (av->av_cookie != NULL) { + error = bus_teardown_intr(sc->sc_dev, av->av_irq, + av->av_cookie); + if (error != 0) { + device_printf(sc->sc_dev, + "cannot tear down IRQ %d: %d\n", i, error); + return (error); + } + av->av_cookie = NULL; + } + + error = bus_setup_intr(sc->sc_dev, av->av_irq, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, apex_intr, av, + &av->av_cookie); + if (error != 0) { + device_printf(sc->sc_dev, + "cannot re-establish IRQ %d: %d\n", i, error); + return (error); + } + } + + return (0); +} + +static int +apex_teardown_msix(struct apex_softc *sc) +{ + struct apex_intr_vec *av; + int error, i, j; + + for (i = 0; i < sc->sc_nvecs; i++) { + av = &sc->sc_vec[i]; + if (av->av_cookie == NULL) + continue; + + error = bus_teardown_intr(sc->sc_dev, av->av_irq, + av->av_cookie); + if (error != 0) { + device_printf(sc->sc_dev, + "cannot tear down IRQ %d: %d\n", i, error); + + for (j = 0; j < i; j++) { + av = &sc->sc_vec[j]; + if (av->av_irq == NULL || av->av_cookie != NULL) + continue; + if (bus_setup_intr(sc->sc_dev, av->av_irq, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, + apex_intr, av, &av->av_cookie) != 0) + device_printf(sc->sc_dev, + "cannot restore IRQ %d, leaving it " + "for the next reset\n", j); + } + return (error); + } + av->av_cookie = NULL; + } + + for (i = 0; i < sc->sc_nvecs; i++) { + av = &sc->sc_vec[i]; + if (av->av_irq == NULL) + continue; + error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, + av->av_rid, av->av_irq); + if (error != 0) + device_printf(sc->sc_dev, + "cannot release IRQ %d, leaking it: %d\n", i, + error); + av->av_irq = NULL; + } + + if (sc->sc_nvecs != 0) { + error = pci_release_msi(sc->sc_dev); + if (error != 0) + device_printf(sc->sc_dev, + "cannot release MSI-X, leaking it: %d\n", error); + sc->sc_nvecs = 0; + } + + for (i = 0; i < APEX_INTR_COUNT; i++) { + if (sc->sc_vec[i].av_count != NULL) { + counter_u64_free(sc->sc_vec[i].av_count); + sc->sc_vec[i].av_count = NULL; + } + } + + return (0); +} + +static int +apex_setup_msix(struct apex_softc *sc) +{ + device_t dev = sc->sc_dev; + struct apex_intr_vec *av; + int avail, count, error, i, terror; + + avail = pci_msix_count(dev); + if (avail < APEX_INTR_COUNT) { + device_printf(dev, "want %d MSI-X vectors, only %d available\n", + APEX_INTR_COUNT, avail); + return (ENXIO); + } + + count = APEX_INTR_COUNT; + error = pci_alloc_msix(dev, &count); + if (error != 0) { + device_printf(dev, "cannot allocate MSI-X: %d\n", error); + return (error); + } + if (count != APEX_INTR_COUNT) { + device_printf(dev, "got %d MSI-X vectors, need %d\n", count, + APEX_INTR_COUNT); + pci_release_msi(dev); + return (ENXIO); + } + sc->sc_nvecs = count; + + for (i = 0; i < count; i++) { + av = &sc->sc_vec[i]; + av->av_sc = sc; + av->av_index = i; + av->av_rid = i + 1; + av->av_count = counter_u64_alloc(M_WAITOK); + av->av_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &av->av_rid, RF_ACTIVE); + if (av->av_irq == NULL) { + device_printf(dev, "cannot allocate IRQ %d\n", i); + error = ENXIO; + goto fail; + } + error = bus_setup_intr(dev, av->av_irq, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, apex_intr, av, + &av->av_cookie); + if (error != 0) { + device_printf(dev, "cannot set up IRQ %d: %d\n", i, + error); + goto fail; + } + } + + return (0); + +fail: + terror = apex_teardown_msix(sc); + if (terror != 0) + device_printf(dev, + "setup failed (%d) and the interrupt could not be released " + "(%d), it is still registered\n", error, terror); + return (error); +} + +static int +apex_sysctl_intr_counts(SYSCTL_HANDLER_ARGS) +{ + struct apex_softc *sc = arg1; + struct sbuf sb; + int error, i; + + sbuf_new_for_sysctl(&sb, NULL, 128, req); + for (i = 0; i < sc->sc_nvecs; i++) + sbuf_printf(&sb, "%s%ju", i == 0 ? "" : " ", + (uintmax_t)counter_u64_fetch(sc->sc_vec[i].av_count)); + error = sbuf_finish(&sb); + sbuf_delete(&sb); + + return (error); +} + +static int +apex_gasket_reset(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + sx_xlock(&sc->sc_lock); + if (!apex_gcb_in_reset(sc)) { + error = apex_enter_reset_locked(sc); + if (error != 0) { + sx_xunlock(&sc->sc_lock); + return (error); + } + } + error = apex_quit_reset_locked(sc); + sx_xunlock(&sc->sc_lock); + + return (error); +} + +static int +apex_gasket_quiesce(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error, i; + + sx_xlock(&sc->sc_lock); + if (apex_gcb_in_reset(sc)) { + sx_xunlock(&sc->sc_lock); + return (0); + } + + gasket_rmw8(sc->sc_csr, APEX_IDLEGEN_IDLEREGISTER, APEX_IDLEGEN_DISABLE, + APEX_IDLEGEN_ENABLE_WIDTH, APEX_IDLEGEN_ENABLE_SHIFT); + gasket_wr8(sc->sc_csr, APEX_UHIB_DMA_PAUSE, APEX_DMA_PAUSE_REQUEST); + error = gasket_wait8(sc->sc_csr, APEX_UHIB_DMA_PAUSED, 1, 1, + APEX_RESET_RETRY, APEX_RESET_DELAY_MS); + if (error != 0) { + device_printf(dev, "DMA did not quiesce within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + sx_xunlock(&sc->sc_lock); + return (error); + } + + /* The status lags the control write, so poll rather than read once. */ + gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_ASSERT, + APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT); + for (i = 0; i < APEX_RESET_RETRY; i++) { + if (apex_gcb_in_reset(sc)) + break; + pause_sbt("apexqui", mstosbt(APEX_RESET_DELAY_MS), 0, + C_HARDCLOCK); + } + if (!apex_gcb_in_reset(sc)) { + device_printf(dev, "GCB did not enter reset within %d ms\n", + APEX_RESET_RETRY * APEX_RESET_DELAY_MS); + sx_xunlock(&sc->sc_lock); + return (EIO); + } + sx_xunlock(&sc->sc_lock); + + return (0); +} + +static int +apex_gasket_enter_reset(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + if (!apex_allow_power_save) + return (0); + + sx_xlock(&sc->sc_lock); + error = apex_enter_reset_locked(sc); + sx_xunlock(&sc->sc_lock); + + return (error); +} + +static int +apex_gasket_quit_reset(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + sx_xlock(&sc->sc_lock); + error = apex_quit_reset_locked(sc); + sx_xunlock(&sc->sc_lock); + + return (error); +} + +static int +apex_gasket_status(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + uint32_t val; + + /* + * A dead or unpowered device reads all ones on the config-space + * shadow registers in BAR2. + */ + sx_xlock(&sc->sc_lock); + val = gasket_rd4(sc->sc_csr, APEX_SCU_3); + sx_xunlock(&sc->sc_lock); + + return (val == 0xffffffff ? GASKET_STATUS_DEAD : GASKET_STATUS_ALIVE); +} + +static int +apex_gate_clock(struct apex_softc *sc, const struct apex_gate_clock_ioctl *gc) +{ + if (gc->force_idle != 0) { + return (EOPNOTSUPP); + } + if (!apex_allow_sw_clock_gating) + return (0); + + sx_xlock(&sc->sc_lock); + if (gc->enable != 0) { + gasket_rmw4(sc->sc_csr, APEX_AXI_QUIESCE, + APEX_AXI_QUIESCE_ENABLE, APEX_BIT_WIDTH, + APEX_AXI_QUIESCE_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_GCB_CLOCK_GATE, APEX_GATED_GCB_ON, + APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT); + } else { + gasket_rmw4(sc->sc_csr, APEX_GCB_CLOCK_GATE, APEX_GATED_GCB_OFF, + APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT); + gasket_rmw4(sc->sc_csr, APEX_AXI_QUIESCE, + APEX_AXI_QUIESCE_DISABLE, APEX_BIT_WIDTH, + APEX_AXI_QUIESCE_SHIFT); + } + sx_xunlock(&sc->sc_lock); + + return (0); +} + +static int +apex_set_performance(struct apex_softc *sc, uint32_t performance) +{ + /* The 8051 clock is unused on PCIe parts and stays fixed. */ + const uint32_t mcu_clk_fixed = 1; + const int axi_clk_fixed_shift = 2; + const int mcu_clk_fixed_shift = 3; + uint32_t gcb_clk_div; + + switch (performance) { + case APEX_PERFORMANCE_LOW: + gcb_clk_div = 3; + break; + case APEX_PERFORMANCE_MED: + gcb_clk_div = 2; + break; + case APEX_PERFORMANCE_HIGH: + gcb_clk_div = 1; + break; + case APEX_PERFORMANCE_MAX: + gcb_clk_div = 0; + break; + default: + return (EINVAL); + } + + sx_xlock(&sc->sc_lock); + gasket_rmw4(sc->sc_csr, APEX_SCU_3, + gcb_clk_div | (0u << axi_clk_fixed_shift) | + (mcu_clk_fixed << mcu_clk_fixed_shift), + APEX_SCU3_CLK_DIV_WIDTH, APEX_SCU3_CLK_DIV_SHIFT); + sx_xunlock(&sc->sc_lock); + + return (0); +} + +static int +apex_gasket_ioctl(device_t dev, u_long cmd, caddr_t data, int fflag __unused) +{ + struct apex_softc *sc = device_get_softc(dev); + + switch (cmd) { + case APEX_IOCTL_GATE_CLOCK: + return (apex_gate_clock(sc, + (const struct apex_gate_clock_ioctl *)data)); + case APEX_IOCTL_PERFORMANCE_EXPECTATION: + return (apex_set_performance(sc, + ((const struct apex_performance_expectation_ioctl *)data)-> + performance)); + default: + return (ENOTTY); + } +} + +static int +apex_sysctl_temp(SYSCTL_HANDLER_ARGS) +{ + struct apex_softc *sc = arg1; + int millic, val; + + sx_xlock(&sc->sc_lock); + millic = apex_adc_to_millic(apex_read_temp_adc(sc)); + sx_xunlock(&sc->sc_lock); + + val = millic / 100 + 2731; + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +apex_sysctl_unique_id(SYSCTL_HANDLER_ARGS) +{ + struct apex_softc *sc = arg1; + char buf[33]; + uint32_t w0, w1, w2, w3; + + sx_xlock(&sc->sc_lock); + w0 = gasket_rd4(sc->sc_csr, APEX_EFUSE_DC); + w1 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E0); + w2 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E4); + w3 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E8); + sx_xunlock(&sc->sc_lock); + + snprintf(buf, sizeof(buf), "%08x%08x%08x%08x", w3, w2, w1, w0); + return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); +} + +static int +apex_sysctl_in_reset(SYSCTL_HANDLER_ARGS) +{ + struct apex_softc *sc = arg1; + int val; + + sx_xlock(&sc->sc_lock); + val = apex_gcb_in_reset(sc) ? 1 : 0; + sx_xunlock(&sc->sc_lock); + + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +apex_sysctl_reg64(SYSCTL_HANDLER_ARGS) +{ + struct apex_softc *sc = arg1; + uint64_t val; + + sx_xlock(&sc->sc_lock); + val = gasket_rd8(sc->sc_csr, (bus_size_t)arg2); + sx_xunlock(&sc->sc_lock); + + return (sysctl_handle_64(oidp, &val, 0, req)); +} + +static void +apex_add_sysctls(struct apex_softc *sc) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *tree; + + ctx = device_get_sysctl_ctx(sc->sc_dev); + tree = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->sc_dev)); + + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "temperature", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, + apex_sysctl_temp, "IK", "Die temperature"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "unique_id", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, + apex_sysctl_unique_id, "A", "Chip unique identifier from efuse"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "gcb_in_reset", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, + apex_sysctl_in_reset, "I", "GCB is held in reset"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "page_table_size", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, + APEX_KHIB_PAGE_TABLE_SIZE, apex_sysctl_reg64, "QU", + "Page table entry count reported by the hardware"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dma_pause", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, + APEX_UHIB_DMA_PAUSE, apex_sysctl_reg64, "QU", + "User HIB DMA pause request"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dma_paused", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, + APEX_UHIB_DMA_PAUSED, apex_sysctl_reg64, "QU", + "User HIB DMA paused status"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "hib_error_status", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, + APEX_UHIB_ERROR_STATUS, apex_sysctl_reg64, "QU", + "User HIB error status"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "interrupt_counts", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, + apex_sysctl_intr_counts, "A", + "Per-vector interrupt counts, in interrupt index order"); + SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "scalar_core_error_status", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, + APEX_SCALAR_CORE_ERROR_STATUS, apex_sysctl_reg64, "QU", + "Scalar core error status"); +} + +static int +apex_probe(device_t dev) +{ + const struct pci_device_table *ent; + + ent = PCI_MATCH(dev, apex_devs); + if (ent == NULL) + return (ENXIO); + + device_set_desc(dev, ent->descr); + return (BUS_PROBE_DEFAULT); +} + +static int +apex_wait_tables_ready(struct apex_softc *sc) +{ + uint64_t page_table, msix_table; + int i; + + page_table = msix_table = 0; + for (i = 0; i < APEX_RESET_RETRY; i++) { + page_table = gasket_rd8(sc->sc_csr, APEX_KHIB_PAGE_TABLE_INIT); + msix_table = gasket_rd8(sc->sc_csr, APEX_KHIB_MSIX_TABLE_INIT); + if (page_table != 0 && msix_table != 0) + return (0); + pause_sbt("apextb", mstosbt(APEX_RESET_DELAY_MS), 0, + C_HARDCLOCK); + } + + if (page_table == 0) + device_printf(sc->sc_dev, "page table init timed out\n"); + if (msix_table == 0) + device_printf(sc->sc_dev, "MSI-X table init timed out\n"); + return (ETIMEDOUT); +} + +static void +apex_apply_tunables(struct apex_softc *sc) +{ + int i; + + if (apex_trip_point_temp[0] > apex_trip_point_temp[1] || + apex_trip_point_temp[1] > apex_trip_point_temp[2]) { + device_printf(sc->sc_dev, + "trip points are not ascending, using defaults\n"); + apex_trip_point_temp[0] = 85000; + apex_trip_point_temp[1] = 90000; + apex_trip_point_temp[2] = 95000; + } + + for (i = 0; i < nitems(sc->sc_adc_trip); i++) { + sc->sc_adc_trip[i] = + apex_millic_to_adc(apex_trip_point_temp[i]); + } + + sc->sc_temp_poll_interval = apex_temp_poll_interval; + sc->sc_hw_warn1_adc = apex_millic_to_adc(apex_hw_temp_warn1); + sc->sc_hw_warn2_adc = apex_millic_to_adc(apex_hw_temp_warn2); + sc->sc_hw_warn1_en = apex_hw_temp_warn1_en != 0; + sc->sc_hw_warn2_en = apex_hw_temp_warn2_en != 0; +} + +static int +apex_attach(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + uint64_t entries; + int error, terror; + + sc->sc_dev = dev; + sx_init(&sc->sc_lock, "apex"); + TIMEOUT_TASK_INIT(taskqueue_thread, &sc->sc_temp_task, 0, + apex_temp_task_fn, sc); + + pci_enable_busmaster(dev); + + sc->sc_csr_rid = APEX_CSR_BAR; + sc->sc_csr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->sc_csr_rid, RF_ACTIVE); + if (sc->sc_csr == NULL) { + device_printf(dev, "cannot map CSR BAR\n"); + error = ENXIO; + goto fail_lock; + } + if (rman_get_size(sc->sc_csr) < APEX_CSR_SIZE) { + device_printf(dev, "CSR BAR is %ju bytes, expected %d\n", + (uintmax_t)rman_get_size(sc->sc_csr), APEX_CSR_SIZE); + error = ENXIO; + goto fail_bar; + } + + apex_apply_tunables(sc); + + sx_xlock(&sc->sc_lock); + apex_program_hw_temp_warnings(sc); + + if (!apex_gcb_in_reset(sc)) { + error = apex_enter_reset_locked(sc); + if (error != 0) { + sx_xunlock(&sc->sc_lock); + goto fail_bar; + } + } + error = apex_quit_reset_locked(sc); + if (error != 0) { + sx_xunlock(&sc->sc_lock); + device_printf(dev, "reset failed\n"); + goto fail_bar; + } + + error = apex_wait_tables_ready(sc); + if (error != 0) { + sx_xunlock(&sc->sc_lock); + goto fail_bar; + } + + entries = gasket_rd8(sc->sc_csr, APEX_KHIB_PAGE_TABLE_SIZE); + if (entries != APEX_PAGE_TABLE_ENTRIES) { + sx_xunlock(&sc->sc_lock); + device_printf(dev, + "page table reports %ju entries, expected %d\n", + (uintmax_t)entries, APEX_PAGE_TABLE_ENTRIES); + error = ENXIO; + goto fail_bar; + } + + apex_enable_thermal_sensing(sc); + sx_xunlock(&sc->sc_lock); + + error = apex_setup_msix(sc); + if (error != 0) { + if (sc->sc_nvecs != 0) { + device_printf(dev, + "interrupt could not be released after failed " + "setup (%d), device is attached but unusable\n", + error); + return (0); + } + goto fail_bar; + } + + sc->sc_desc.gd_name = "apex"; + sc->sc_desc.gd_csr = sc->sc_csr; + sc->sc_desc.gd_regions = apex_regions; + sc->sc_desc.gd_nregions = nitems(apex_regions); + sc->sc_desc.gd_page_tables = apex_page_tables; + sc->sc_desc.gd_npage_tables = nitems(apex_page_tables); + sc->sc_desc.gd_interrupts = apex_interrupts; + sc->sc_desc.gd_ninterrupts = nitems(apex_interrupts); + sc->sc_desc.gd_interrupt_pack_width = APEX_INTR_PACK_WIDTH; + sc->sc_desc.gd_coherent_base = APEX_COHERENT_OFFSET; + sc->sc_desc.gd_coherent_size = APEX_COHERENT_PAGES * PAGE_SIZE; + + error = gasket_attach(dev, &sc->sc_desc, &sc->sc_gasket); + if (error != 0) + goto fail_msix; + + apex_add_sysctls(sc); + + if (apex_allow_power_save) + apex_enter_reset(sc); + + apex_sched_temp_task(sc); + + if (bootverbose) { + sx_xlock(&sc->sc_lock); + device_printf(dev, "die temperature %d milli-degrees C\n", + apex_adc_to_millic(apex_read_temp_adc(sc))); + sx_xunlock(&sc->sc_lock); + } + + return (0); + +fail_msix: + terror = apex_teardown_msix(sc); + if (terror != 0) { + device_printf(dev, + "attach failed (%d) and the interrupt could not be " + "released " + "(%d), device is attached but unusable\n", error, terror); + return (0); + } + +fail_bar: + bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_csr_rid, sc->sc_csr); +fail_lock: + pci_disable_busmaster(dev); + sx_destroy(&sc->sc_lock); + return (error); +} + +static int +apex_detach(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + error = gasket_detach_begin(sc->sc_gasket); + if (error != 0) + return (error); + + sx_xlock(&sc->sc_lock); + sc->sc_dying = true; + sx_xunlock(&sc->sc_lock); + taskqueue_drain_timeout(taskqueue_thread, &sc->sc_temp_task); + + error = apex_teardown_msix(sc); + if (error != 0) { + device_printf(dev, "cannot release interrupts: %d\n", error); + sx_xlock(&sc->sc_lock); + sc->sc_dying = false; + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); + gasket_detach_abort(sc->sc_gasket); + return (error); + } + + gasket_detach_finish(sc->sc_gasket); + sc->sc_gasket = NULL; + + sx_xlock(&sc->sc_lock); + (void)apex_enter_reset_locked(sc); + sx_xunlock(&sc->sc_lock); + + bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_csr_rid, sc->sc_csr); + pci_disable_busmaster(dev); + sx_destroy(&sc->sc_lock); + + return (0); +} + +static int +apex_suspend(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + error = gasket_suspend_begin(sc->sc_gasket); + if (error != 0) { + device_printf(dev, "cannot suspend while in use\n"); + return (error); + } + + taskqueue_cancel_timeout(taskqueue_thread, &sc->sc_temp_task, NULL); + taskqueue_drain_timeout(taskqueue_thread, &sc->sc_temp_task); + + error = apex_enter_reset(sc); + if (error != 0) { + gasket_resume(sc->sc_gasket); + sx_xlock(&sc->sc_lock); + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); + } + + return (error); +} + +static int +apex_resume(device_t dev) +{ + struct apex_softc *sc = device_get_softc(dev); + int error; + + error = apex_gasket_reset(dev); + if (error != 0) { + sx_xlock(&sc->sc_lock); + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); + gasket_resume(sc->sc_gasket); + return (error); + } + + sx_xlock(&sc->sc_lock); + apex_program_hw_temp_warnings(sc); + apex_enable_thermal_sensing(sc); + sx_xunlock(&sc->sc_lock); + + if (apex_allow_power_save) { + error = apex_enter_reset(sc); + if (error != 0) { + sx_xlock(&sc->sc_lock); + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); + gasket_resume(sc->sc_gasket); + return (error); + } + } + + sx_xlock(&sc->sc_lock); + apex_sched_temp_task(sc); + sx_xunlock(&sc->sc_lock); + + gasket_resume(sc->sc_gasket); + + return (0); +} + +static device_method_t apex_methods[] = { + DEVMETHOD(device_probe, apex_probe), + DEVMETHOD(device_attach, apex_attach), + DEVMETHOD(device_detach, apex_detach), + DEVMETHOD(device_suspend, apex_suspend), + DEVMETHOD(device_resume, apex_resume), + + DEVMETHOD(gasket_reset, apex_gasket_reset), + DEVMETHOD(gasket_quiesce, apex_gasket_quiesce), + DEVMETHOD(gasket_enter_reset, apex_gasket_enter_reset), + DEVMETHOD(gasket_quit_reset, apex_gasket_quit_reset), + DEVMETHOD(gasket_status, apex_gasket_status), + DEVMETHOD(gasket_ioctl, apex_gasket_ioctl), + + DEVMETHOD_END +}; + +static driver_t apex_driver = { + "apex", + apex_methods, + sizeof(struct apex_softc) +}; + +DRIVER_MODULE(apex, pci, apex_driver, NULL, NULL); +MODULE_DEPEND(apex, pci, 1, 1, 1); +MODULE_DEPEND(apex, gasket, 1, 1, 1); +PCI_PNP_INFO(apex_devs); +MODULE_VERSION(apex, 1); diff --git a/sys/dev/apex/apex_ioctl.h b/sys/dev/apex/apex_ioctl.h new file mode 100644 --- /dev/null +++ b/sys/dev/apex/apex_ioctl.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef _DEV_APEX_APEX_IOCTL_H_ +#define _DEV_APEX_APEX_IOCTL_H_ + +#include +#include + +#include + +struct apex_gate_clock_ioctl { + uint64_t enable; + uint64_t force_idle; +}; + +enum apex_performance_expectation { + APEX_PERFORMANCE_LOW = 0, + APEX_PERFORMANCE_MED = 1, + APEX_PERFORMANCE_HIGH = 2, + APEX_PERFORMANCE_MAX = 3, +}; + +struct apex_performance_expectation_ioctl { + uint32_t performance; +}; + +/* Matches APEX_IOCTL_BASE in Google's apex ioctl header. */ +#define APEX_IOC_GROUP 0x7F + +#define APEX_IOCTL_GATE_CLOCK \ + _IOW(APEX_IOC_GROUP, 0, struct apex_gate_clock_ioctl) +#define APEX_IOCTL_PERFORMANCE_EXPECTATION \ + _IOW(APEX_IOC_GROUP, 1, struct apex_performance_expectation_ioctl) + +#endif /* _DEV_APEX_APEX_IOCTL_H_ */ diff --git a/sys/dev/apex/apexreg.h b/sys/dev/apex/apexreg.h new file mode 100644 --- /dev/null +++ b/sys/dev/apex/apexreg.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * Register definitions for the Google Apex Edge TPU, derived from Google's + * gasket/apex Linux driver (github.com/google/gasket-driver). + */ + +#ifndef _DEV_APEX_APEXREG_H_ +#define _DEV_APEX_APEXREG_H_ + +#define APEX_VENDOR_ID 0x1ac1 /* Global Unichip Corp. */ +#define APEX_DEVICE_ID 0x089a + +/* All CSRs live in BAR2, a 1 MB prefetchable window. */ +#define APEX_CSR_BAR PCIR_BAR(2) +#define APEX_CSR_SIZE 0x100000 + +#define APEX_SCU_BASE 0x1a300 +#define APEX_SCU_0 (APEX_SCU_BASE + 0x0c) +#define APEX_SCU_1 (APEX_SCU_BASE + 0x10) +#define APEX_SCU_2 (APEX_SCU_BASE + 0x14) +#define APEX_SCU_3 (APEX_SCU_BASE + 0x18) +#define APEX_SCU_4 (APEX_SCU_BASE + 0x1c) +#define APEX_SCU_5 (APEX_SCU_BASE + 0x20) +#define APEX_GCB_CLOCK_GATE (APEX_SCU_BASE + 0x14) +#define APEX_AXI_QUIESCE (APEX_SCU_BASE + 0x2c) + +#define APEX_SCU2_RST_GCB_SHIFT 2 +#define APEX_SCU2_GATED_GCB_SHIFT 18 +#define APEX_SCU3_FORCE_RAM_SD_SHIFT 14 +#define APEX_SCU3_RAM_SD_DONE (1u << 6) +#define APEX_SCU3_CUR_RST_GCB 0x10 +#define APEX_SCU3_PWR_STATE_OVR_SHIFT 26 +#define APEX_SCU3_PWR_STATE_OVR_WIDTH 2 +#define APEX_SCU3_CLK_DIV_SHIFT 28 +#define APEX_SCU3_CLK_DIV_WIDTH 4 +#define APEX_AXI_QUIESCE_SHIFT 16 +#define APEX_AXI_QUIESCE_ENABLE 0x1 +#define APEX_AXI_QUIESCE_DISABLE 0x0 +#define APEX_BIT_WIDTH 1 + +/* Two bit fields written through the shifts above. */ +#define APEX_FIELD_WIDTH 2 +#define APEX_RST_GCB_ASSERT 0x1 +#define APEX_RST_GCB_RELEASE 0x2 +#define APEX_GATED_GCB_ON 0x1 +#define APEX_GATED_GCB_OFF 0x0 +#define APEX_RAM_SD_FORCE 0x3 +#define APEX_RAM_SD_RELEASE 0x0 +#define APEX_PWR_STATE_ALLOW_INACTIVE 0x2 +#define APEX_PWR_STATE_ALWAYS_ON 0x3 + +#define APEX_OMC_TEMP_ENABLE_SHIFT 7 +#define APEX_OMC_TEMP_SENSOR_ENABLES 0x7 /* ENAD, ENVR, ENBG */ +#define APEX_OMC_TEMP_SENSOR_ENABLES_SHIFT 0 +#define APEX_OMC_TEMP_SENSOR_ENABLES_WIDTH 3 +#define APEX_OMC_TEMP_CTRL_ENABLE 0x1 +#define APEX_OMC_TEMP_CTRL_ENABLE_SHIFT 0 +#define APEX_OMC_TEMP_WARN_SHIFT 16 +#define APEX_OMC_TEMP_WARN_WIDTH 10 +#define APEX_OMC_TEMP_WARN_EN_SHIFT 31 + +#define APEX_OMC0_D0 0x01a0d0 +#define APEX_OMC0_D4 0x01a0d4 /* hw temp warn 1 */ +#define APEX_OMC0_D8 0x01a0d8 /* hw temp warn 2 */ +#define APEX_OMC0_DC 0x01a0dc /* temp ADC readout */ +#define APEX_EFUSE_DC 0x01a2dc +#define APEX_EFUSE_E0 0x01a2e0 +#define APEX_EFUSE_E4 0x01a2e4 +#define APEX_EFUSE_E8 0x01a2e8 + +#define APEX_TEMP_ADC_SHIFT 16 +#define APEX_TEMP_ADC_WIDTH 10 + +#define APEX_KHIB_PAGE_TABLE_SIZE 0x46000 +#define APEX_KHIB_EXTENDED_TABLE 0x46008 +#define APEX_KHIB_TRANSLATION_ENABLE 0x46010 +#define APEX_KHIB_DMA_PAUSE 0x46050 +#define APEX_KHIB_DMA_PAUSE_MASK 0x46058 +#define APEX_KHIB_PAGE_TABLE_INIT 0x46078 +#define APEX_KHIB_MSIX_TABLE_INIT 0x46080 +#define APEX_KHIB_PAGE_TABLE 0x50000 + +#define APEX_UHIB_DMA_PAUSE 0x486d8 +#define APEX_UHIB_DMA_PAUSED 0x486e0 +#define APEX_UHIB_ERROR_STATUS 0x486f0 +#define APEX_DMA_PAUSE_REQUEST 1 + +#define APEX_SCALAR_CORE_ERROR_STATUS 0x041a0 +#define APEX_IDLEGEN_IDLEREGISTER 0x4a000 +#define APEX_IDLEGEN_ENABLE_SHIFT 32 +#define APEX_IDLEGEN_ENABLE_WIDTH 1 +#define APEX_IDLEGEN_DISABLE 0 + +#define APEX_PAGE_TABLE_ENTRIES 8192 +#define APEX_EXTENDED_SHIFT 63 + +/* + * Interrupt indices. These must stay dense and must match the order of the + * descriptor table in apex.c. Up to four vector control fields of + * APEX_INTR_PACK_WIDTH bits share one register. + */ +#define APEX_INTR_PACK_WIDTH 7 + +enum apex_interrupt { + APEX_INTR_INSTR_QUEUE = 0, + APEX_INTR_INPUT_ACTV_QUEUE, + APEX_INTR_PARAM_QUEUE, + APEX_INTR_OUTPUT_ACTV_QUEUE, + APEX_INTR_SC_HOST_0, + APEX_INTR_SC_HOST_1, + APEX_INTR_SC_HOST_2, + APEX_INTR_SC_HOST_3, + APEX_INTR_TOP_LEVEL_0, + APEX_INTR_TOP_LEVEL_1, + APEX_INTR_TOP_LEVEL_2, + APEX_INTR_TOP_LEVEL_3, + APEX_INTR_FATAL_ERR, + APEX_INTR_COUNT +}; + +/* + * MSI-X table, as reported by the capability: BAR2 offset 0x46800, 16 bytes + * per entry with the vector control word last. Bit 0 of that word masks the + * vector. + */ +#define APEX_MSIX_TABLE 0x46800 +#define APEX_MSIX_ENTRY_SIZE 16 +#define APEX_MSIX_VECTOR_CTRL 12 + +/* + * Coherent buffer. User space maps it at APEX_COHERENT_OFFSET, which names + * the region rather than any device address. + */ +#define APEX_COHERENT_OFFSET 0x1000000 +#define APEX_COHERENT_PAGES 16 + +#define APEX_RESET_RETRY 120 +#define APEX_RESET_DELAY_MS 100 + +#endif /* _DEV_APEX_APEXREG_H_ */ diff --git a/sys/modules/Makefile b/sys/modules/Makefile --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -43,6 +43,7 @@ ${_allwinner} \ ${_armv8crypto} \ ${_armv8_rng} \ + ${_apex} \ ${_apple_bce} \ ${_asmc} \ ata \ @@ -555,6 +556,7 @@ SUBDIR+= linux .endif .if ${MACHINE_CPUARCH} == "amd64" +_apex= apex _gasket= gasket .endif .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" diff --git a/sys/modules/apex/Makefile b/sys/modules/apex/Makefile new file mode 100644 --- /dev/null +++ b/sys/modules/apex/Makefile @@ -0,0 +1,8 @@ +.PATH: ${SRCTOP}/sys/dev/apex ${SRCTOP}/sys/dev/gasket + +KMOD= apex +SRCS= apex.c +SRCS+= gasket_if.h +SRCS+= device_if.h bus_if.h pci_if.h + +.include