Page MenuHomeFreeBSD
Files F164622911

D58563.id.diff
No OneTemporary
Actions

D58563.id.diff
View Options

diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist
--- a/etc/mtree/BSD.include.dist
+++ b/etc/mtree/BSD.include.dist
@@ -205,6 +205,8 @@
..
agp
..
+ apex
+ ..
ciss
..
coreboot
diff --git a/include/Makefile b/include/Makefile
--- a/include/Makefile
+++ b/include/Makefile
@@ -187,6 +187,10 @@
GASKET= gasket_ioctl.h
GASKETDIR= ${INCLUDEDIR}/dev/gasket
+.PATH: ${SRCTOP}/sys/dev/apex
+APEX= apex_ioctl.h
+APEXDIR= ${INCLUDEDIR}/dev/apex
+
.PATH: ${SRCTOP}/sys/dev/nvme
NVME= nvme.h
NVMEDIR= ${INCLUDEDIR}/dev/nvme
@@ -256,6 +260,7 @@
INCSGROUPS= INCS \
ACPICA \
AGP \
+ APEX \
BSM \
CAM \
CAMATA \
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -56,6 +56,7 @@
${_bxe.4} \
${_aout.4} \
${_apic.4} \
+ apex.4 \
aq.4 \
arcmsr.4 \
arswitch.4 \
diff --git a/share/man/man4/apex.4 b/share/man/man4/apex.4
new file mode 100644
--- /dev/null
+++ b/share/man/man4/apex.4
@@ -0,0 +1,104 @@
+.\"
+.\" Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+.\"
+.\" SPDX-License-Identifier: BSD-2-Clause
+.\"
+.Dd July 30, 2026
+.Dt APEX 4
+.Os
+.Sh NAME
+.Nm apex
+.Nd Google Coral Edge TPU accelerator driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device gasket"
+.Cd "device apex"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+apex_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver supports the Google Edge TPU, the inference accelerator on Coral
+PCI Express and M.2 modules.
+It attaches through the
+.Xr gasket 4
+framework and creates
+.Pa /dev/apex0 ,
+which an Edge TPU runtime in user space opens to run a model.
+.Pp
+The core is held in reset while nothing has the device open, and a kernel
+task divides its clock as the die warms.
+.Sh HARDWARE
+The
+.Nm
+driver supports the following devices:
+.Pp
+.Bl -bullet -compact
+.It
+Coral M.2 Accelerator A+E key
+.It
+Coral M.2 Accelerator B+M key
+.It
+Coral Mini PCIe Accelerator
+.El
+.Sh LOADER TUNABLES
+Read once at attach, settable in
+.Xr loader.conf 5 :
+.Bl -tag -width indent
+.It Va hw.apex.temp_poll_interval
+Sampling interval in milliseconds, 5000 by default; zero disables throttling.
+.It Va hw.apex.hw_temp_warn1
+.It Va hw.apex.hw_temp_warn2
+Hardware warning thresholds in milli-degrees C, both 100000 by default.
+.It Va hw.apex.hw_temp_warn1_en
+Drive INTR on the first threshold, off by default.
+.It Va hw.apex.hw_temp_warn2_en
+Drive SD_ALARM on the second, on by default.
+.El
+.Sh SYSCTL VARIABLES
+.Bl -tag -width indent
+.It Va hw.apex.allow_power_save
+Enter the low power state on last close, on by default.
+The core is held in reset either way.
+.It Va hw.apex.allow_hw_clock_gating
+Let the hardware gate the clock when idle, on by default.
+.It Va hw.apex.allow_sw_clock_gating
+Honor clock gating requests from user space, off by default.
+.El
+.Pp
+Read-only per-device state is under
+.Va dev.apex.%d .
+.Sh FILES
+.Bl -tag -width indent
+.It Pa /dev/apex%d
+Accelerator control device.
+.El
+.Sh SEE ALSO
+.Xr gasket 4 ,
+.Xr pci 4
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Fx 16.0 .
+.Sh AUTHORS
+.An Abdelkader Boudih Aq Mt freebsd@seuros.com
+.Sh CAVEATS
+Suspend and detach fail with
+.Er EBUSY
+while a consumer holds the device open or mapped, since both clear the on-chip
+page table.
+.Pp
+The ioctl numbers use native
+.Fx
+encoding, so a library written against the Linux driver needs more than a
+recompile.
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -70,6 +70,12 @@
#
device gasket
+#
+# Google Apex Edge TPU, as found on Coral PCIe and M.2 modules. Requires
+# the gasket framework above.
+#
+device apex
+
#
# Microsemi smartpqi controllers.
# These controllers have a SCSI-like interface, and require the
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -112,6 +112,7 @@
crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl
crypto/openssl/ossl_aes_gcm.c optional ossl
dev/amdgpio/amdgpio.c optional amdgpio
+dev/apex/apex.c optional apex pci
dev/gasket/gasket_core.c optional gasket
dev/gasket/gasket_page_table.c optional gasket
dev/gasket/gasket_if.m optional gasket
diff --git a/sys/dev/apex/apex.c b/sys/dev/apex/apex.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/apex/apex.c
@@ -0,0 +1,1178 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Register level behavior derived from Google's Linux gasket driver,
+ * github.com/google/gasket-driver.
+ */
+
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/counter.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/sbuf.h>
+#include <sys/sx.h>
+#include <sys/sysctl.h>
+#include <sys/taskqueue.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <dev/gasket/gasket.h>
+#include "gasket_if.h"
+
+#include <dev/apex/apexreg.h>
+#include <dev/apex/apex_ioctl.h>
+
+struct apex_softc;
+
+struct apex_intr_vec {
+ struct apex_softc *av_sc;
+ struct resource *av_irq;
+ void *av_cookie;
+ int av_rid;
+ u_int av_index;
+ counter_u64_t av_count;
+};
+
+struct apex_softc {
+ device_t sc_dev;
+ struct resource *sc_csr;
+ int sc_csr_rid;
+ struct gasket_softc *sc_gasket;
+ struct gasket_desc sc_desc;
+
+ struct apex_intr_vec sc_vec[APEX_INTR_COUNT];
+ int sc_nvecs;
+
+ struct sx sc_lock;
+ struct timeout_task sc_temp_task;
+ bool sc_dying;
+
+ /* ADC counts descend as heat rises. */
+ int sc_adc_trip[3];
+ int sc_temp_poll_interval; /* milliseconds */
+ int sc_hw_warn1_adc;
+ int sc_hw_warn2_adc;
+ bool sc_hw_warn1_en;
+ bool sc_hw_warn2_en;
+};
+
+static const struct gasket_region apex_regions[] = {
+ { 0x40000, 0x1000 },
+ { 0x44000, 0x1000 },
+ { 0x48000, 0x1000 },
+};
+
+static const struct gasket_page_table_config apex_page_tables[] = {
+ {
+ .gpt_id = 0,
+ .gpt_total_entries = APEX_PAGE_TABLE_ENTRIES,
+ .gpt_base_reg = APEX_KHIB_PAGE_TABLE,
+ .gpt_extended_reg = APEX_KHIB_EXTENDED_TABLE,
+ .gpt_extended_bit = APEX_EXTENDED_SHIFT,
+ },
+};
+
+static const struct gasket_interrupt_desc apex_interrupts[] = {
+ { APEX_INTR_INSTR_QUEUE, 0x46018, GASKET_INTR_UNPACKED },
+ { APEX_INTR_INPUT_ACTV_QUEUE, 0x46020, GASKET_INTR_UNPACKED },
+ { APEX_INTR_PARAM_QUEUE, 0x46028, GASKET_INTR_UNPACKED },
+ { APEX_INTR_OUTPUT_ACTV_QUEUE, 0x46030, GASKET_INTR_UNPACKED },
+ { APEX_INTR_SC_HOST_0, 0x46038, GASKET_INTR_PACK_0 },
+ { APEX_INTR_SC_HOST_1, 0x46038, GASKET_INTR_PACK_1 },
+ { APEX_INTR_SC_HOST_2, 0x46038, GASKET_INTR_PACK_2 },
+ { APEX_INTR_SC_HOST_3, 0x46038, GASKET_INTR_PACK_3 },
+ { APEX_INTR_TOP_LEVEL_0, 0x46040, GASKET_INTR_PACK_0 },
+ { APEX_INTR_TOP_LEVEL_1, 0x46040, GASKET_INTR_PACK_1 },
+ { APEX_INTR_TOP_LEVEL_2, 0x46040, GASKET_INTR_PACK_2 },
+ { APEX_INTR_TOP_LEVEL_3, 0x46040, GASKET_INTR_PACK_3 },
+ { APEX_INTR_FATAL_ERR, 0x46048, GASKET_INTR_UNPACKED },
+};
+
+static const struct pci_device_table apex_devs[] = {
+ { PCI_DEV(APEX_VENDOR_ID, APEX_DEVICE_ID),
+ PCI_DESCR("Google Apex Edge TPU") },
+};
+
+static int apex_allow_power_save = 1;
+static int apex_allow_sw_clock_gating = 0;
+static int apex_allow_hw_clock_gating = 1;
+static int apex_temp_poll_interval = 5000;
+
+static SYSCTL_NODE(_hw, OID_AUTO, apex, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Google Apex Edge TPU");
+SYSCTL_INT(_hw_apex, OID_AUTO, allow_power_save, CTLFLAG_RWTUN,
+ &apex_allow_power_save, 0,
+ "Enter the low power state when the last consumer closes the device");
+SYSCTL_INT(_hw_apex, OID_AUTO, allow_sw_clock_gating, CTLFLAG_RWTUN,
+ &apex_allow_sw_clock_gating, 0, "Honor software clock gating requests");
+SYSCTL_INT(_hw_apex, OID_AUTO, allow_hw_clock_gating, CTLFLAG_RWTUN,
+ &apex_allow_hw_clock_gating, 0, "Let the hardware gate the GCB clock");
+SYSCTL_INT(_hw_apex, OID_AUTO, temp_poll_interval, CTLFLAG_RDTUN,
+ &apex_temp_poll_interval, 0,
+ "Temperature poll interval in milliseconds, 0 disables throttling");
+
+/* Trip points in milli-degrees C, ascending. */
+static int apex_trip_point_temp[3] = { 85000, 90000, 95000 };
+static int apex_hw_temp_warn1 = 100000;
+static int apex_hw_temp_warn2 = 100000;
+static int apex_hw_temp_warn1_en = 0;
+static int apex_hw_temp_warn2_en = 1;
+
+SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn1, CTLFLAG_RDTUN,
+ &apex_hw_temp_warn1, 0,
+ "Hardware temperature warning 1 in milli-degrees C");
+SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn2, CTLFLAG_RDTUN,
+ &apex_hw_temp_warn2, 0,
+ "Hardware temperature warning 2 in milli-degrees C");
+SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn1_en, CTLFLAG_RDTUN,
+ &apex_hw_temp_warn1_en, 0, "Drive INTR on the warning 1 threshold");
+SYSCTL_INT(_hw_apex, OID_AUTO, hw_temp_warn2_en, CTLFLAG_RDTUN,
+ &apex_hw_temp_warn2_en, 0, "Drive SD_ALARM on the warning 2 threshold");
+
+static int apex_enter_reset_locked(struct apex_softc *sc);
+static int apex_enter_reset(struct apex_softc *sc);
+static void apex_program_interrupts(struct apex_softc *sc);
+static int apex_restore_msix(struct apex_softc *sc);
+
+static void
+apex_sched_temp_task(struct apex_softc *sc)
+{
+
+ if (sc->sc_temp_poll_interval <= 0)
+ return;
+ taskqueue_enqueue_timeout(taskqueue_thread, &sc->sc_temp_task,
+ (int)((int64_t)sc->sc_temp_poll_interval * hz / 1000));
+}
+
+static int
+apex_enter_reset(struct apex_softc *sc)
+{
+ int error;
+
+ sx_xlock(&sc->sc_lock);
+ error = apex_enter_reset_locked(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ return (error);
+}
+
+/* The temperature ADC is a linear fit optimized for 25C to 100C. */
+static int
+apex_adc_to_millic(int adc)
+{
+ return ((662 - adc) * 250 + 550);
+}
+
+static int
+apex_millic_to_adc(int millic)
+{
+ return ((550 - millic) / 250 + 662);
+}
+
+static int
+apex_read_temp_adc(struct apex_softc *sc)
+{
+ uint32_t val;
+
+ val = gasket_rd4(sc->sc_csr, APEX_OMC0_DC);
+ return ((val >> APEX_TEMP_ADC_SHIFT) &
+ ((1 << APEX_TEMP_ADC_WIDTH) - 1));
+}
+
+static bool
+apex_gcb_in_reset(struct apex_softc *sc)
+{
+ return ((gasket_rd4(sc->sc_csr, APEX_SCU_3) &
+ APEX_SCU3_CUR_RST_GCB) != 0);
+}
+
+static int
+apex_enter_reset_locked(struct apex_softc *sc)
+{
+ int error;
+
+ sx_assert(&sc->sc_lock, SA_XLOCKED);
+
+ gasket_rmw8(sc->sc_csr, APEX_IDLEGEN_IDLEREGISTER, APEX_IDLEGEN_DISABLE,
+ APEX_IDLEGEN_ENABLE_WIDTH, APEX_IDLEGEN_ENABLE_SHIFT);
+ gasket_wr8(sc->sc_csr, APEX_UHIB_DMA_PAUSE, APEX_DMA_PAUSE_REQUEST);
+
+ error = gasket_wait8(sc->sc_csr, APEX_UHIB_DMA_PAUSED, 1, 1,
+ APEX_RESET_RETRY, APEX_RESET_DELAY_MS);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "DMA did not quiesce within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ return (error);
+ }
+
+ gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_ASSERT,
+ APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_GATED_GCB_ON,
+ APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_SCU_3, APEX_RAM_SD_FORCE,
+ APEX_FIELD_WIDTH, APEX_SCU3_FORCE_RAM_SD_SHIFT);
+
+ error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_RAM_SD_DONE,
+ APEX_SCU3_RAM_SD_DONE, APEX_RESET_RETRY, APEX_RESET_DELAY_MS);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "RAM did not shut down within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ return (error);
+ }
+
+ return (0);
+}
+
+static int
+apex_quit_reset_locked(struct apex_softc *sc)
+{
+ int error;
+
+ sx_assert(&sc->sc_lock, SA_XLOCKED);
+
+ gasket_rmw4(sc->sc_csr, APEX_SCU_3, APEX_RAM_SD_RELEASE,
+ APEX_FIELD_WIDTH, APEX_SCU3_FORCE_RAM_SD_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_GATED_GCB_OFF,
+ APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_RELEASE,
+ APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT);
+
+ error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_RAM_SD_DONE, 0,
+ APEX_RESET_RETRY, APEX_RESET_DELAY_MS);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "RAM did not enable within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ return (error);
+ }
+
+ error = gasket_wait8(sc->sc_csr, APEX_SCU_3, APEX_SCU3_CUR_RST_GCB, 0,
+ APEX_RESET_RETRY, APEX_RESET_DELAY_MS);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "GCB did not leave reset within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ return (error);
+ }
+
+ gasket_rmw4(sc->sc_csr, APEX_SCU_3,
+ apex_allow_hw_clock_gating ? APEX_PWR_STATE_ALLOW_INACTIVE :
+ APEX_PWR_STATE_ALWAYS_ON,
+ APEX_SCU3_PWR_STATE_OVR_WIDTH, APEX_SCU3_PWR_STATE_OVR_SHIFT);
+
+ error = apex_restore_msix(sc);
+ if (error != 0)
+ return (error);
+ apex_program_interrupts(sc);
+
+ return (0);
+}
+
+static void
+apex_program_hw_temp_warnings(struct apex_softc *sc)
+{
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D4, sc->sc_hw_warn1_adc,
+ APEX_OMC_TEMP_WARN_WIDTH, APEX_OMC_TEMP_WARN_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, sc->sc_hw_warn2_adc,
+ APEX_OMC_TEMP_WARN_WIDTH, APEX_OMC_TEMP_WARN_SHIFT);
+
+ /* Unconditional: firmware may have left a warning enabled. */
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D4, sc->sc_hw_warn1_en ? 1 : 0,
+ APEX_BIT_WIDTH, APEX_OMC_TEMP_WARN_EN_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, sc->sc_hw_warn2_en ? 1 : 0,
+ APEX_BIT_WIDTH, APEX_OMC_TEMP_WARN_EN_SHIFT);
+}
+
+static void
+apex_enable_thermal_sensing(struct apex_softc *sc)
+{
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D0, APEX_OMC_TEMP_CTRL_ENABLE,
+ APEX_BIT_WIDTH, APEX_OMC_TEMP_ENABLE_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_D8, APEX_OMC_TEMP_SENSOR_ENABLES,
+ APEX_OMC_TEMP_SENSOR_ENABLES_WIDTH,
+ APEX_OMC_TEMP_SENSOR_ENABLES_SHIFT);
+ /* The controller enable must follow ENAD/ENVR/ENBG by 100 us. */
+ DELAY(100);
+ gasket_rmw4(sc->sc_csr, APEX_OMC0_DC, APEX_OMC_TEMP_CTRL_ENABLE,
+ APEX_BIT_WIDTH, APEX_OMC_TEMP_CTRL_ENABLE_SHIFT);
+}
+
+static void
+apex_temp_task_fn(void *arg, int pending __unused)
+{
+ struct apex_softc *sc = arg;
+ const uint32_t mask = 0x3u << APEX_SCU3_CLK_DIV_SHIFT;
+ uint32_t clk_div, tmp;
+ int adc, i, millic;
+
+ sx_xlock(&sc->sc_lock);
+ if (sc->sc_dying) {
+ sx_xunlock(&sc->sc_lock);
+ return;
+ }
+
+ adc = apex_read_temp_adc(sc);
+ for (i = nitems(sc->sc_adc_trip) - 1; i >= 0; i--) {
+ if (adc <= sc->sc_adc_trip[i])
+ break;
+ }
+ clk_div = (uint32_t)(i + 1) << APEX_SCU3_CLK_DIV_SHIFT;
+
+ tmp = gasket_rd4(sc->sc_csr, APEX_SCU_3);
+ if (clk_div != (tmp & mask)) {
+ gasket_wr4(sc->sc_csr, APEX_SCU_3, (tmp & ~mask) | clk_div);
+ millic = apex_adc_to_millic(adc);
+ device_printf(sc->sc_dev, "clock %sthrottled at %d.%03d C\n",
+ i == -1 ? "un" : "", millic / 1000, abs(millic) % 1000);
+ }
+
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+}
+
+static void
+apex_intr(void *arg)
+{
+ struct apex_intr_vec *av = arg;
+
+ counter_u64_add(av->av_count, 1);
+ gasket_interrupt_notify(av->av_sc->sc_gasket, av->av_index);
+}
+
+/* The kernel HIB loses this across a GCB reset; called on every reset exit. */
+static void
+apex_program_interrupts(struct apex_softc *sc)
+{
+ const struct gasket_interrupt_desc *gid;
+ u_int i, shift, width;
+
+ sx_assert(&sc->sc_lock, SA_XLOCKED);
+
+ if (sc->sc_nvecs == 0)
+ return;
+
+ width = APEX_INTR_PACK_WIDTH;
+ for (i = 0; i < nitems(apex_interrupts); i++) {
+ gid = &apex_interrupts[i];
+ if (gid->gid_index >= (u_int)sc->sc_nvecs)
+ continue;
+
+ if (gid->gid_packing == GASKET_INTR_UNPACKED) {
+ gasket_wr8(sc->sc_csr, gid->gid_reg, gid->gid_index);
+ continue;
+ }
+
+ shift = (gid->gid_packing - GASKET_INTR_PACK_0) * width;
+ gasket_rmw8(sc->sc_csr, gid->gid_reg, gid->gid_index, width,
+ shift);
+ }
+}
+
+/*
+ * A GCB reset zeroes the MSI-X table, which shares device memory with the HIB.
+ */
+static int
+apex_restore_msix(struct apex_softc *sc)
+{
+ struct apex_intr_vec *av;
+ int error, i;
+
+ for (i = 0; i < sc->sc_nvecs; i++) {
+ av = &sc->sc_vec[i];
+ if (av->av_irq == NULL)
+ continue;
+
+ if (av->av_cookie != NULL) {
+ error = bus_teardown_intr(sc->sc_dev, av->av_irq,
+ av->av_cookie);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "cannot tear down IRQ %d: %d\n", i, error);
+ return (error);
+ }
+ av->av_cookie = NULL;
+ }
+
+ error = bus_setup_intr(sc->sc_dev, av->av_irq,
+ INTR_TYPE_MISC | INTR_MPSAFE, NULL, apex_intr, av,
+ &av->av_cookie);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "cannot re-establish IRQ %d: %d\n", i, error);
+ return (error);
+ }
+ }
+
+ return (0);
+}
+
+static int
+apex_teardown_msix(struct apex_softc *sc)
+{
+ struct apex_intr_vec *av;
+ int error, i, j;
+
+ for (i = 0; i < sc->sc_nvecs; i++) {
+ av = &sc->sc_vec[i];
+ if (av->av_cookie == NULL)
+ continue;
+
+ error = bus_teardown_intr(sc->sc_dev, av->av_irq,
+ av->av_cookie);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "cannot tear down IRQ %d: %d\n", i, error);
+
+ for (j = 0; j < i; j++) {
+ av = &sc->sc_vec[j];
+ if (av->av_irq == NULL || av->av_cookie != NULL)
+ continue;
+ if (bus_setup_intr(sc->sc_dev, av->av_irq,
+ INTR_TYPE_MISC | INTR_MPSAFE, NULL,
+ apex_intr, av, &av->av_cookie) != 0)
+ device_printf(sc->sc_dev,
+ "cannot restore IRQ %d, leaving it "
+ "for the next reset\n", j);
+ }
+ return (error);
+ }
+ av->av_cookie = NULL;
+ }
+
+ for (i = 0; i < sc->sc_nvecs; i++) {
+ av = &sc->sc_vec[i];
+ if (av->av_irq == NULL)
+ continue;
+ error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ,
+ av->av_rid, av->av_irq);
+ if (error != 0)
+ device_printf(sc->sc_dev,
+ "cannot release IRQ %d, leaking it: %d\n", i,
+ error);
+ av->av_irq = NULL;
+ }
+
+ if (sc->sc_nvecs != 0) {
+ error = pci_release_msi(sc->sc_dev);
+ if (error != 0)
+ device_printf(sc->sc_dev,
+ "cannot release MSI-X, leaking it: %d\n", error);
+ sc->sc_nvecs = 0;
+ }
+
+ for (i = 0; i < APEX_INTR_COUNT; i++) {
+ if (sc->sc_vec[i].av_count != NULL) {
+ counter_u64_free(sc->sc_vec[i].av_count);
+ sc->sc_vec[i].av_count = NULL;
+ }
+ }
+
+ return (0);
+}
+
+static int
+apex_setup_msix(struct apex_softc *sc)
+{
+ device_t dev = sc->sc_dev;
+ struct apex_intr_vec *av;
+ int avail, count, error, i, terror;
+
+ avail = pci_msix_count(dev);
+ if (avail < APEX_INTR_COUNT) {
+ device_printf(dev, "want %d MSI-X vectors, only %d available\n",
+ APEX_INTR_COUNT, avail);
+ return (ENXIO);
+ }
+
+ count = APEX_INTR_COUNT;
+ error = pci_alloc_msix(dev, &count);
+ if (error != 0) {
+ device_printf(dev, "cannot allocate MSI-X: %d\n", error);
+ return (error);
+ }
+ if (count != APEX_INTR_COUNT) {
+ device_printf(dev, "got %d MSI-X vectors, need %d\n", count,
+ APEX_INTR_COUNT);
+ pci_release_msi(dev);
+ return (ENXIO);
+ }
+ sc->sc_nvecs = count;
+
+ for (i = 0; i < count; i++) {
+ av = &sc->sc_vec[i];
+ av->av_sc = sc;
+ av->av_index = i;
+ av->av_rid = i + 1;
+ av->av_count = counter_u64_alloc(M_WAITOK);
+ av->av_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+ &av->av_rid, RF_ACTIVE);
+ if (av->av_irq == NULL) {
+ device_printf(dev, "cannot allocate IRQ %d\n", i);
+ error = ENXIO;
+ goto fail;
+ }
+ error = bus_setup_intr(dev, av->av_irq,
+ INTR_TYPE_MISC | INTR_MPSAFE, NULL, apex_intr, av,
+ &av->av_cookie);
+ if (error != 0) {
+ device_printf(dev, "cannot set up IRQ %d: %d\n", i,
+ error);
+ goto fail;
+ }
+ }
+
+ return (0);
+
+fail:
+ terror = apex_teardown_msix(sc);
+ if (terror != 0)
+ device_printf(dev,
+ "setup failed (%d) and the interrupt could not be released "
+ "(%d), it is still registered\n", error, terror);
+ return (error);
+}
+
+static int
+apex_sysctl_intr_counts(SYSCTL_HANDLER_ARGS)
+{
+ struct apex_softc *sc = arg1;
+ struct sbuf sb;
+ int error, i;
+
+ sbuf_new_for_sysctl(&sb, NULL, 128, req);
+ for (i = 0; i < sc->sc_nvecs; i++)
+ sbuf_printf(&sb, "%s%ju", i == 0 ? "" : " ",
+ (uintmax_t)counter_u64_fetch(sc->sc_vec[i].av_count));
+ error = sbuf_finish(&sb);
+ sbuf_delete(&sb);
+
+ return (error);
+}
+
+static int
+apex_gasket_reset(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ sx_xlock(&sc->sc_lock);
+ if (!apex_gcb_in_reset(sc)) {
+ error = apex_enter_reset_locked(sc);
+ if (error != 0) {
+ sx_xunlock(&sc->sc_lock);
+ return (error);
+ }
+ }
+ error = apex_quit_reset_locked(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ return (error);
+}
+
+static int
+apex_gasket_quiesce(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error, i;
+
+ sx_xlock(&sc->sc_lock);
+ if (apex_gcb_in_reset(sc)) {
+ sx_xunlock(&sc->sc_lock);
+ return (0);
+ }
+
+ gasket_rmw8(sc->sc_csr, APEX_IDLEGEN_IDLEREGISTER, APEX_IDLEGEN_DISABLE,
+ APEX_IDLEGEN_ENABLE_WIDTH, APEX_IDLEGEN_ENABLE_SHIFT);
+ gasket_wr8(sc->sc_csr, APEX_UHIB_DMA_PAUSE, APEX_DMA_PAUSE_REQUEST);
+ error = gasket_wait8(sc->sc_csr, APEX_UHIB_DMA_PAUSED, 1, 1,
+ APEX_RESET_RETRY, APEX_RESET_DELAY_MS);
+ if (error != 0) {
+ device_printf(dev, "DMA did not quiesce within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ sx_xunlock(&sc->sc_lock);
+ return (error);
+ }
+
+ /* The status lags the control write, so poll rather than read once. */
+ gasket_rmw4(sc->sc_csr, APEX_SCU_2, APEX_RST_GCB_ASSERT,
+ APEX_FIELD_WIDTH, APEX_SCU2_RST_GCB_SHIFT);
+ for (i = 0; i < APEX_RESET_RETRY; i++) {
+ if (apex_gcb_in_reset(sc))
+ break;
+ pause_sbt("apexqui", mstosbt(APEX_RESET_DELAY_MS), 0,
+ C_HARDCLOCK);
+ }
+ if (!apex_gcb_in_reset(sc)) {
+ device_printf(dev, "GCB did not enter reset within %d ms\n",
+ APEX_RESET_RETRY * APEX_RESET_DELAY_MS);
+ sx_xunlock(&sc->sc_lock);
+ return (EIO);
+ }
+ sx_xunlock(&sc->sc_lock);
+
+ return (0);
+}
+
+static int
+apex_gasket_enter_reset(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ if (!apex_allow_power_save)
+ return (0);
+
+ sx_xlock(&sc->sc_lock);
+ error = apex_enter_reset_locked(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ return (error);
+}
+
+static int
+apex_gasket_quit_reset(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ sx_xlock(&sc->sc_lock);
+ error = apex_quit_reset_locked(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ return (error);
+}
+
+static int
+apex_gasket_status(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ uint32_t val;
+
+ /*
+ * A dead or unpowered device reads all ones on the config-space
+ * shadow registers in BAR2.
+ */
+ sx_xlock(&sc->sc_lock);
+ val = gasket_rd4(sc->sc_csr, APEX_SCU_3);
+ sx_xunlock(&sc->sc_lock);
+
+ return (val == 0xffffffff ? GASKET_STATUS_DEAD : GASKET_STATUS_ALIVE);
+}
+
+static int
+apex_gate_clock(struct apex_softc *sc, const struct apex_gate_clock_ioctl *gc)
+{
+ if (gc->force_idle != 0) {
+ return (EOPNOTSUPP);
+ }
+ if (!apex_allow_sw_clock_gating)
+ return (0);
+
+ sx_xlock(&sc->sc_lock);
+ if (gc->enable != 0) {
+ gasket_rmw4(sc->sc_csr, APEX_AXI_QUIESCE,
+ APEX_AXI_QUIESCE_ENABLE, APEX_BIT_WIDTH,
+ APEX_AXI_QUIESCE_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_GCB_CLOCK_GATE, APEX_GATED_GCB_ON,
+ APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT);
+ } else {
+ gasket_rmw4(sc->sc_csr, APEX_GCB_CLOCK_GATE, APEX_GATED_GCB_OFF,
+ APEX_FIELD_WIDTH, APEX_SCU2_GATED_GCB_SHIFT);
+ gasket_rmw4(sc->sc_csr, APEX_AXI_QUIESCE,
+ APEX_AXI_QUIESCE_DISABLE, APEX_BIT_WIDTH,
+ APEX_AXI_QUIESCE_SHIFT);
+ }
+ sx_xunlock(&sc->sc_lock);
+
+ return (0);
+}
+
+static int
+apex_set_performance(struct apex_softc *sc, uint32_t performance)
+{
+ /* The 8051 clock is unused on PCIe parts and stays fixed. */
+ const uint32_t mcu_clk_fixed = 1;
+ const int axi_clk_fixed_shift = 2;
+ const int mcu_clk_fixed_shift = 3;
+ uint32_t gcb_clk_div;
+
+ switch (performance) {
+ case APEX_PERFORMANCE_LOW:
+ gcb_clk_div = 3;
+ break;
+ case APEX_PERFORMANCE_MED:
+ gcb_clk_div = 2;
+ break;
+ case APEX_PERFORMANCE_HIGH:
+ gcb_clk_div = 1;
+ break;
+ case APEX_PERFORMANCE_MAX:
+ gcb_clk_div = 0;
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ sx_xlock(&sc->sc_lock);
+ gasket_rmw4(sc->sc_csr, APEX_SCU_3,
+ gcb_clk_div | (0u << axi_clk_fixed_shift) |
+ (mcu_clk_fixed << mcu_clk_fixed_shift),
+ APEX_SCU3_CLK_DIV_WIDTH, APEX_SCU3_CLK_DIV_SHIFT);
+ sx_xunlock(&sc->sc_lock);
+
+ return (0);
+}
+
+static int
+apex_gasket_ioctl(device_t dev, u_long cmd, caddr_t data, int fflag __unused)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+
+ switch (cmd) {
+ case APEX_IOCTL_GATE_CLOCK:
+ return (apex_gate_clock(sc,
+ (const struct apex_gate_clock_ioctl *)data));
+ case APEX_IOCTL_PERFORMANCE_EXPECTATION:
+ return (apex_set_performance(sc,
+ ((const struct apex_performance_expectation_ioctl *)data)->
+ performance));
+ default:
+ return (ENOTTY);
+ }
+}
+
+static int
+apex_sysctl_temp(SYSCTL_HANDLER_ARGS)
+{
+ struct apex_softc *sc = arg1;
+ int millic, val;
+
+ sx_xlock(&sc->sc_lock);
+ millic = apex_adc_to_millic(apex_read_temp_adc(sc));
+ sx_xunlock(&sc->sc_lock);
+
+ val = millic / 100 + 2731;
+ return (sysctl_handle_int(oidp, &val, 0, req));
+}
+
+static int
+apex_sysctl_unique_id(SYSCTL_HANDLER_ARGS)
+{
+ struct apex_softc *sc = arg1;
+ char buf[33];
+ uint32_t w0, w1, w2, w3;
+
+ sx_xlock(&sc->sc_lock);
+ w0 = gasket_rd4(sc->sc_csr, APEX_EFUSE_DC);
+ w1 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E0);
+ w2 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E4);
+ w3 = gasket_rd4(sc->sc_csr, APEX_EFUSE_E8);
+ sx_xunlock(&sc->sc_lock);
+
+ snprintf(buf, sizeof(buf), "%08x%08x%08x%08x", w3, w2, w1, w0);
+ return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
+}
+
+static int
+apex_sysctl_in_reset(SYSCTL_HANDLER_ARGS)
+{
+ struct apex_softc *sc = arg1;
+ int val;
+
+ sx_xlock(&sc->sc_lock);
+ val = apex_gcb_in_reset(sc) ? 1 : 0;
+ sx_xunlock(&sc->sc_lock);
+
+ return (sysctl_handle_int(oidp, &val, 0, req));
+}
+
+static int
+apex_sysctl_reg64(SYSCTL_HANDLER_ARGS)
+{
+ struct apex_softc *sc = arg1;
+ uint64_t val;
+
+ sx_xlock(&sc->sc_lock);
+ val = gasket_rd8(sc->sc_csr, (bus_size_t)arg2);
+ sx_xunlock(&sc->sc_lock);
+
+ return (sysctl_handle_64(oidp, &val, 0, req));
+}
+
+static void
+apex_add_sysctls(struct apex_softc *sc)
+{
+ struct sysctl_ctx_list *ctx;
+ struct sysctl_oid_list *tree;
+
+ ctx = device_get_sysctl_ctx(sc->sc_dev);
+ tree = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->sc_dev));
+
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "temperature",
+ CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+ apex_sysctl_temp, "IK", "Die temperature");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "unique_id",
+ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+ apex_sysctl_unique_id, "A", "Chip unique identifier from efuse");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "gcb_in_reset",
+ CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+ apex_sysctl_in_reset, "I", "GCB is held in reset");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "page_table_size",
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
+ APEX_KHIB_PAGE_TABLE_SIZE, apex_sysctl_reg64, "QU",
+ "Page table entry count reported by the hardware");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dma_pause",
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
+ APEX_UHIB_DMA_PAUSE, apex_sysctl_reg64, "QU",
+ "User HIB DMA pause request");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dma_paused",
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
+ APEX_UHIB_DMA_PAUSED, apex_sysctl_reg64, "QU",
+ "User HIB DMA paused status");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "hib_error_status",
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
+ APEX_UHIB_ERROR_STATUS, apex_sysctl_reg64, "QU",
+ "User HIB error status");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "interrupt_counts",
+ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+ apex_sysctl_intr_counts, "A",
+ "Per-vector interrupt counts, in interrupt index order");
+ SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "scalar_core_error_status",
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
+ APEX_SCALAR_CORE_ERROR_STATUS, apex_sysctl_reg64, "QU",
+ "Scalar core error status");
+}
+
+static int
+apex_probe(device_t dev)
+{
+ const struct pci_device_table *ent;
+
+ ent = PCI_MATCH(dev, apex_devs);
+ if (ent == NULL)
+ return (ENXIO);
+
+ device_set_desc(dev, ent->descr);
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+apex_wait_tables_ready(struct apex_softc *sc)
+{
+ uint64_t page_table, msix_table;
+ int i;
+
+ page_table = msix_table = 0;
+ for (i = 0; i < APEX_RESET_RETRY; i++) {
+ page_table = gasket_rd8(sc->sc_csr, APEX_KHIB_PAGE_TABLE_INIT);
+ msix_table = gasket_rd8(sc->sc_csr, APEX_KHIB_MSIX_TABLE_INIT);
+ if (page_table != 0 && msix_table != 0)
+ return (0);
+ pause_sbt("apextb", mstosbt(APEX_RESET_DELAY_MS), 0,
+ C_HARDCLOCK);
+ }
+
+ if (page_table == 0)
+ device_printf(sc->sc_dev, "page table init timed out\n");
+ if (msix_table == 0)
+ device_printf(sc->sc_dev, "MSI-X table init timed out\n");
+ return (ETIMEDOUT);
+}
+
+static void
+apex_apply_tunables(struct apex_softc *sc)
+{
+ int i;
+
+ if (apex_trip_point_temp[0] > apex_trip_point_temp[1] ||
+ apex_trip_point_temp[1] > apex_trip_point_temp[2]) {
+ device_printf(sc->sc_dev,
+ "trip points are not ascending, using defaults\n");
+ apex_trip_point_temp[0] = 85000;
+ apex_trip_point_temp[1] = 90000;
+ apex_trip_point_temp[2] = 95000;
+ }
+
+ for (i = 0; i < nitems(sc->sc_adc_trip); i++) {
+ sc->sc_adc_trip[i] =
+ apex_millic_to_adc(apex_trip_point_temp[i]);
+ }
+
+ sc->sc_temp_poll_interval = apex_temp_poll_interval;
+ sc->sc_hw_warn1_adc = apex_millic_to_adc(apex_hw_temp_warn1);
+ sc->sc_hw_warn2_adc = apex_millic_to_adc(apex_hw_temp_warn2);
+ sc->sc_hw_warn1_en = apex_hw_temp_warn1_en != 0;
+ sc->sc_hw_warn2_en = apex_hw_temp_warn2_en != 0;
+}
+
+static int
+apex_attach(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ uint64_t entries;
+ int error, terror;
+
+ sc->sc_dev = dev;
+ sx_init(&sc->sc_lock, "apex");
+ TIMEOUT_TASK_INIT(taskqueue_thread, &sc->sc_temp_task, 0,
+ apex_temp_task_fn, sc);
+
+ pci_enable_busmaster(dev);
+
+ sc->sc_csr_rid = APEX_CSR_BAR;
+ sc->sc_csr = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+ &sc->sc_csr_rid, RF_ACTIVE);
+ if (sc->sc_csr == NULL) {
+ device_printf(dev, "cannot map CSR BAR\n");
+ error = ENXIO;
+ goto fail_lock;
+ }
+ if (rman_get_size(sc->sc_csr) < APEX_CSR_SIZE) {
+ device_printf(dev, "CSR BAR is %ju bytes, expected %d\n",
+ (uintmax_t)rman_get_size(sc->sc_csr), APEX_CSR_SIZE);
+ error = ENXIO;
+ goto fail_bar;
+ }
+
+ apex_apply_tunables(sc);
+
+ sx_xlock(&sc->sc_lock);
+ apex_program_hw_temp_warnings(sc);
+
+ if (!apex_gcb_in_reset(sc)) {
+ error = apex_enter_reset_locked(sc);
+ if (error != 0) {
+ sx_xunlock(&sc->sc_lock);
+ goto fail_bar;
+ }
+ }
+ error = apex_quit_reset_locked(sc);
+ if (error != 0) {
+ sx_xunlock(&sc->sc_lock);
+ device_printf(dev, "reset failed\n");
+ goto fail_bar;
+ }
+
+ error = apex_wait_tables_ready(sc);
+ if (error != 0) {
+ sx_xunlock(&sc->sc_lock);
+ goto fail_bar;
+ }
+
+ entries = gasket_rd8(sc->sc_csr, APEX_KHIB_PAGE_TABLE_SIZE);
+ if (entries != APEX_PAGE_TABLE_ENTRIES) {
+ sx_xunlock(&sc->sc_lock);
+ device_printf(dev,
+ "page table reports %ju entries, expected %d\n",
+ (uintmax_t)entries, APEX_PAGE_TABLE_ENTRIES);
+ error = ENXIO;
+ goto fail_bar;
+ }
+
+ apex_enable_thermal_sensing(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ error = apex_setup_msix(sc);
+ if (error != 0) {
+ if (sc->sc_nvecs != 0) {
+ device_printf(dev,
+ "interrupt could not be released after failed "
+ "setup (%d), device is attached but unusable\n",
+ error);
+ return (0);
+ }
+ goto fail_bar;
+ }
+
+ sc->sc_desc.gd_name = "apex";
+ sc->sc_desc.gd_csr = sc->sc_csr;
+ sc->sc_desc.gd_regions = apex_regions;
+ sc->sc_desc.gd_nregions = nitems(apex_regions);
+ sc->sc_desc.gd_page_tables = apex_page_tables;
+ sc->sc_desc.gd_npage_tables = nitems(apex_page_tables);
+ sc->sc_desc.gd_interrupts = apex_interrupts;
+ sc->sc_desc.gd_ninterrupts = nitems(apex_interrupts);
+ sc->sc_desc.gd_interrupt_pack_width = APEX_INTR_PACK_WIDTH;
+ sc->sc_desc.gd_coherent_base = APEX_COHERENT_OFFSET;
+ sc->sc_desc.gd_coherent_size = APEX_COHERENT_PAGES * PAGE_SIZE;
+
+ error = gasket_attach(dev, &sc->sc_desc, &sc->sc_gasket);
+ if (error != 0)
+ goto fail_msix;
+
+ apex_add_sysctls(sc);
+
+ if (apex_allow_power_save)
+ apex_enter_reset(sc);
+
+ apex_sched_temp_task(sc);
+
+ if (bootverbose) {
+ sx_xlock(&sc->sc_lock);
+ device_printf(dev, "die temperature %d milli-degrees C\n",
+ apex_adc_to_millic(apex_read_temp_adc(sc)));
+ sx_xunlock(&sc->sc_lock);
+ }
+
+ return (0);
+
+fail_msix:
+ terror = apex_teardown_msix(sc);
+ if (terror != 0) {
+ device_printf(dev,
+ "attach failed (%d) and the interrupt could not be "
+ "released "
+ "(%d), device is attached but unusable\n", error, terror);
+ return (0);
+ }
+
+fail_bar:
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_csr_rid, sc->sc_csr);
+fail_lock:
+ pci_disable_busmaster(dev);
+ sx_destroy(&sc->sc_lock);
+ return (error);
+}
+
+static int
+apex_detach(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ error = gasket_detach_begin(sc->sc_gasket);
+ if (error != 0)
+ return (error);
+
+ sx_xlock(&sc->sc_lock);
+ sc->sc_dying = true;
+ sx_xunlock(&sc->sc_lock);
+ taskqueue_drain_timeout(taskqueue_thread, &sc->sc_temp_task);
+
+ error = apex_teardown_msix(sc);
+ if (error != 0) {
+ device_printf(dev, "cannot release interrupts: %d\n", error);
+ sx_xlock(&sc->sc_lock);
+ sc->sc_dying = false;
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+ gasket_detach_abort(sc->sc_gasket);
+ return (error);
+ }
+
+ gasket_detach_finish(sc->sc_gasket);
+ sc->sc_gasket = NULL;
+
+ sx_xlock(&sc->sc_lock);
+ (void)apex_enter_reset_locked(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_csr_rid, sc->sc_csr);
+ pci_disable_busmaster(dev);
+ sx_destroy(&sc->sc_lock);
+
+ return (0);
+}
+
+static int
+apex_suspend(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ error = gasket_suspend_begin(sc->sc_gasket);
+ if (error != 0) {
+ device_printf(dev, "cannot suspend while in use\n");
+ return (error);
+ }
+
+ taskqueue_cancel_timeout(taskqueue_thread, &sc->sc_temp_task, NULL);
+ taskqueue_drain_timeout(taskqueue_thread, &sc->sc_temp_task);
+
+ error = apex_enter_reset(sc);
+ if (error != 0) {
+ gasket_resume(sc->sc_gasket);
+ sx_xlock(&sc->sc_lock);
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+ }
+
+ return (error);
+}
+
+static int
+apex_resume(device_t dev)
+{
+ struct apex_softc *sc = device_get_softc(dev);
+ int error;
+
+ error = apex_gasket_reset(dev);
+ if (error != 0) {
+ sx_xlock(&sc->sc_lock);
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+ gasket_resume(sc->sc_gasket);
+ return (error);
+ }
+
+ sx_xlock(&sc->sc_lock);
+ apex_program_hw_temp_warnings(sc);
+ apex_enable_thermal_sensing(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ if (apex_allow_power_save) {
+ error = apex_enter_reset(sc);
+ if (error != 0) {
+ sx_xlock(&sc->sc_lock);
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+ gasket_resume(sc->sc_gasket);
+ return (error);
+ }
+ }
+
+ sx_xlock(&sc->sc_lock);
+ apex_sched_temp_task(sc);
+ sx_xunlock(&sc->sc_lock);
+
+ gasket_resume(sc->sc_gasket);
+
+ return (0);
+}
+
+static device_method_t apex_methods[] = {
+ DEVMETHOD(device_probe, apex_probe),
+ DEVMETHOD(device_attach, apex_attach),
+ DEVMETHOD(device_detach, apex_detach),
+ DEVMETHOD(device_suspend, apex_suspend),
+ DEVMETHOD(device_resume, apex_resume),
+
+ DEVMETHOD(gasket_reset, apex_gasket_reset),
+ DEVMETHOD(gasket_quiesce, apex_gasket_quiesce),
+ DEVMETHOD(gasket_enter_reset, apex_gasket_enter_reset),
+ DEVMETHOD(gasket_quit_reset, apex_gasket_quit_reset),
+ DEVMETHOD(gasket_status, apex_gasket_status),
+ DEVMETHOD(gasket_ioctl, apex_gasket_ioctl),
+
+ DEVMETHOD_END
+};
+
+static driver_t apex_driver = {
+ "apex",
+ apex_methods,
+ sizeof(struct apex_softc)
+};
+
+DRIVER_MODULE(apex, pci, apex_driver, NULL, NULL);
+MODULE_DEPEND(apex, pci, 1, 1, 1);
+MODULE_DEPEND(apex, gasket, 1, 1, 1);
+PCI_PNP_INFO(apex_devs);
+MODULE_VERSION(apex, 1);
diff --git a/sys/dev/apex/apex_ioctl.h b/sys/dev/apex/apex_ioctl.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/apex/apex_ioctl.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _DEV_APEX_APEX_IOCTL_H_
+#define _DEV_APEX_APEX_IOCTL_H_
+
+#include <sys/ioccom.h>
+#include <sys/types.h>
+
+#include <dev/gasket/gasket_ioctl.h>
+
+struct apex_gate_clock_ioctl {
+ uint64_t enable;
+ uint64_t force_idle;
+};
+
+enum apex_performance_expectation {
+ APEX_PERFORMANCE_LOW = 0,
+ APEX_PERFORMANCE_MED = 1,
+ APEX_PERFORMANCE_HIGH = 2,
+ APEX_PERFORMANCE_MAX = 3,
+};
+
+struct apex_performance_expectation_ioctl {
+ uint32_t performance;
+};
+
+/* Matches APEX_IOCTL_BASE in Google's apex ioctl header. */
+#define APEX_IOC_GROUP 0x7F
+
+#define APEX_IOCTL_GATE_CLOCK \
+ _IOW(APEX_IOC_GROUP, 0, struct apex_gate_clock_ioctl)
+#define APEX_IOCTL_PERFORMANCE_EXPECTATION \
+ _IOW(APEX_IOC_GROUP, 1, struct apex_performance_expectation_ioctl)
+
+#endif /* _DEV_APEX_APEX_IOCTL_H_ */
diff --git a/sys/dev/apex/apexreg.h b/sys/dev/apex/apexreg.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/apex/apexreg.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Register definitions for the Google Apex Edge TPU, derived from Google's
+ * gasket/apex Linux driver (github.com/google/gasket-driver).
+ */
+
+#ifndef _DEV_APEX_APEXREG_H_
+#define _DEV_APEX_APEXREG_H_
+
+#define APEX_VENDOR_ID 0x1ac1 /* Global Unichip Corp. */
+#define APEX_DEVICE_ID 0x089a
+
+/* All CSRs live in BAR2, a 1 MB prefetchable window. */
+#define APEX_CSR_BAR PCIR_BAR(2)
+#define APEX_CSR_SIZE 0x100000
+
+#define APEX_SCU_BASE 0x1a300
+#define APEX_SCU_0 (APEX_SCU_BASE + 0x0c)
+#define APEX_SCU_1 (APEX_SCU_BASE + 0x10)
+#define APEX_SCU_2 (APEX_SCU_BASE + 0x14)
+#define APEX_SCU_3 (APEX_SCU_BASE + 0x18)
+#define APEX_SCU_4 (APEX_SCU_BASE + 0x1c)
+#define APEX_SCU_5 (APEX_SCU_BASE + 0x20)
+#define APEX_GCB_CLOCK_GATE (APEX_SCU_BASE + 0x14)
+#define APEX_AXI_QUIESCE (APEX_SCU_BASE + 0x2c)
+
+#define APEX_SCU2_RST_GCB_SHIFT 2
+#define APEX_SCU2_GATED_GCB_SHIFT 18
+#define APEX_SCU3_FORCE_RAM_SD_SHIFT 14
+#define APEX_SCU3_RAM_SD_DONE (1u << 6)
+#define APEX_SCU3_CUR_RST_GCB 0x10
+#define APEX_SCU3_PWR_STATE_OVR_SHIFT 26
+#define APEX_SCU3_PWR_STATE_OVR_WIDTH 2
+#define APEX_SCU3_CLK_DIV_SHIFT 28
+#define APEX_SCU3_CLK_DIV_WIDTH 4
+#define APEX_AXI_QUIESCE_SHIFT 16
+#define APEX_AXI_QUIESCE_ENABLE 0x1
+#define APEX_AXI_QUIESCE_DISABLE 0x0
+#define APEX_BIT_WIDTH 1
+
+/* Two bit fields written through the shifts above. */
+#define APEX_FIELD_WIDTH 2
+#define APEX_RST_GCB_ASSERT 0x1
+#define APEX_RST_GCB_RELEASE 0x2
+#define APEX_GATED_GCB_ON 0x1
+#define APEX_GATED_GCB_OFF 0x0
+#define APEX_RAM_SD_FORCE 0x3
+#define APEX_RAM_SD_RELEASE 0x0
+#define APEX_PWR_STATE_ALLOW_INACTIVE 0x2
+#define APEX_PWR_STATE_ALWAYS_ON 0x3
+
+#define APEX_OMC_TEMP_ENABLE_SHIFT 7
+#define APEX_OMC_TEMP_SENSOR_ENABLES 0x7 /* ENAD, ENVR, ENBG */
+#define APEX_OMC_TEMP_SENSOR_ENABLES_SHIFT 0
+#define APEX_OMC_TEMP_SENSOR_ENABLES_WIDTH 3
+#define APEX_OMC_TEMP_CTRL_ENABLE 0x1
+#define APEX_OMC_TEMP_CTRL_ENABLE_SHIFT 0
+#define APEX_OMC_TEMP_WARN_SHIFT 16
+#define APEX_OMC_TEMP_WARN_WIDTH 10
+#define APEX_OMC_TEMP_WARN_EN_SHIFT 31
+
+#define APEX_OMC0_D0 0x01a0d0
+#define APEX_OMC0_D4 0x01a0d4 /* hw temp warn 1 */
+#define APEX_OMC0_D8 0x01a0d8 /* hw temp warn 2 */
+#define APEX_OMC0_DC 0x01a0dc /* temp ADC readout */
+#define APEX_EFUSE_DC 0x01a2dc
+#define APEX_EFUSE_E0 0x01a2e0
+#define APEX_EFUSE_E4 0x01a2e4
+#define APEX_EFUSE_E8 0x01a2e8
+
+#define APEX_TEMP_ADC_SHIFT 16
+#define APEX_TEMP_ADC_WIDTH 10
+
+#define APEX_KHIB_PAGE_TABLE_SIZE 0x46000
+#define APEX_KHIB_EXTENDED_TABLE 0x46008
+#define APEX_KHIB_TRANSLATION_ENABLE 0x46010
+#define APEX_KHIB_DMA_PAUSE 0x46050
+#define APEX_KHIB_DMA_PAUSE_MASK 0x46058
+#define APEX_KHIB_PAGE_TABLE_INIT 0x46078
+#define APEX_KHIB_MSIX_TABLE_INIT 0x46080
+#define APEX_KHIB_PAGE_TABLE 0x50000
+
+#define APEX_UHIB_DMA_PAUSE 0x486d8
+#define APEX_UHIB_DMA_PAUSED 0x486e0
+#define APEX_UHIB_ERROR_STATUS 0x486f0
+#define APEX_DMA_PAUSE_REQUEST 1
+
+#define APEX_SCALAR_CORE_ERROR_STATUS 0x041a0
+#define APEX_IDLEGEN_IDLEREGISTER 0x4a000
+#define APEX_IDLEGEN_ENABLE_SHIFT 32
+#define APEX_IDLEGEN_ENABLE_WIDTH 1
+#define APEX_IDLEGEN_DISABLE 0
+
+#define APEX_PAGE_TABLE_ENTRIES 8192
+#define APEX_EXTENDED_SHIFT 63
+
+/*
+ * Interrupt indices. These must stay dense and must match the order of the
+ * descriptor table in apex.c. Up to four vector control fields of
+ * APEX_INTR_PACK_WIDTH bits share one register.
+ */
+#define APEX_INTR_PACK_WIDTH 7
+
+enum apex_interrupt {
+ APEX_INTR_INSTR_QUEUE = 0,
+ APEX_INTR_INPUT_ACTV_QUEUE,
+ APEX_INTR_PARAM_QUEUE,
+ APEX_INTR_OUTPUT_ACTV_QUEUE,
+ APEX_INTR_SC_HOST_0,
+ APEX_INTR_SC_HOST_1,
+ APEX_INTR_SC_HOST_2,
+ APEX_INTR_SC_HOST_3,
+ APEX_INTR_TOP_LEVEL_0,
+ APEX_INTR_TOP_LEVEL_1,
+ APEX_INTR_TOP_LEVEL_2,
+ APEX_INTR_TOP_LEVEL_3,
+ APEX_INTR_FATAL_ERR,
+ APEX_INTR_COUNT
+};
+
+/*
+ * MSI-X table, as reported by the capability: BAR2 offset 0x46800, 16 bytes
+ * per entry with the vector control word last. Bit 0 of that word masks the
+ * vector.
+ */
+#define APEX_MSIX_TABLE 0x46800
+#define APEX_MSIX_ENTRY_SIZE 16
+#define APEX_MSIX_VECTOR_CTRL 12
+
+/*
+ * Coherent buffer. User space maps it at APEX_COHERENT_OFFSET, which names
+ * the region rather than any device address.
+ */
+#define APEX_COHERENT_OFFSET 0x1000000
+#define APEX_COHERENT_PAGES 16
+
+#define APEX_RESET_RETRY 120
+#define APEX_RESET_DELAY_MS 100
+
+#endif /* _DEV_APEX_APEXREG_H_ */
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -43,6 +43,7 @@
${_allwinner} \
${_armv8crypto} \
${_armv8_rng} \
+ ${_apex} \
${_apple_bce} \
${_asmc} \
ata \
@@ -552,6 +553,7 @@
SUBDIR+= linux
.endif
.if ${MACHINE_CPUARCH} == "amd64"
+_apex= apex
_gasket= gasket
.endif
.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64"
diff --git a/sys/modules/apex/Makefile b/sys/modules/apex/Makefile
new file mode 100644
--- /dev/null
+++ b/sys/modules/apex/Makefile
@@ -0,0 +1,8 @@
+.PATH: ${SRCTOP}/sys/dev/apex ${SRCTOP}/sys/dev/gasket
+
+KMOD= apex
+SRCS= apex.c
+SRCS+= gasket_if.h
+SRCS+= device_if.h bus_if.h pci_if.h
+
+.include <bsd.kmod.mk>

File Metadata

Mime Type
text/plain
Expires
Mon, Aug 3, 2:45 PM (4 h, 37 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35871579
Default Alt Text
D58563.id.diff (42 KB)

Event Timeline