Page MenuHomeFreeBSD
Files F164467809

D57088.id179754.diff
No OneTemporary
Actions

D57088.id179754.diff
View Options

diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -112,6 +112,9 @@
crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl
crypto/openssl/ossl_aes_gcm.c optional ossl
dev/amdgpio/amdgpio.c optional amdgpio
+dev/apple_bce/apple_bce.c optional apple_bce pci
+dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci
+dev/apple_bce/apple_bce_queue.c optional apple_bce pci
dev/asmc/asmc.c optional asmc isa
dev/asmc/asmcmmio.c optional asmc isa
dev/axgbe/if_axgbe_pci.c optional axp
diff --git a/sys/dev/apple_bce/apple_bce.h b/sys/dev/apple_bce/apple_bce.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce.h
@@ -0,0 +1,304 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple T2 Buffer Copy Engine (BCE) driver.
+ * PCI transport layer for T2 coprocessor communication.
+ */
+
+#ifndef _APPLE_BCE_H_
+#define _APPLE_BCE_H_
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/sema.h>
+#include <sys/callout.h>
+#include <sys/rman.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#define BCE_PCI_VENDOR_APPLE 0x106b
+#define BCE_PCI_DEVICE_T2 0x1801
+
+#define BCE_MAX_QUEUE_COUNT 0x100
+#define BCE_MAX_CQ_COUNT 16 /* Max completion queues tracked */
+#define BCE_QUEUE_USER_MIN 2
+#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1)
+#define BCE_CMD_SIZE 0x40
+
+#define BCE_FW_PROTOCOL_VER 0x20001
+#define BCE_MBOX_TIMEOUT_MS 1000
+#define BCE_TIMESTAMP_MS 150 /* Firmware keepalive interval */
+
+/*
+ * Mailbox register offsets (BAR4).
+ */
+#define BCE_REG_MBOX_OUT 0x820 /* 4x u32 to-device */
+#define BCE_REG_MBOX_REPLY 0x810 /* 4x u32 from-device */
+#define BCE_REG_MBOX_REPLY_CTR 0x108 /* Reply counter (bits 23:20 = count) */
+#define BCE_REG_TIMESTAMP 0xC000 /* Timestamp sync */
+
+/*
+ * DMA register offsets (BAR2).
+ */
+#define BCE_REG_DOORBELL_BASE 0x44000
+
+/*
+ * Mailbox message format: 6-bit type (63:58) + 58-bit value.
+ */
+#define BCE_MB_MSG(type, val) (((uint64_t)(type) << 58) | \
+ ((val) & 0x3FFFFFFFFFFFFFFULL))
+#define BCE_MB_TYPE(v) ((uint32_t)((v) >> 58))
+#define BCE_MB_VALUE(v) ((v) & 0x3FFFFFFFFFFFFFFULL)
+
+/*
+ * Mailbox message types.
+ */
+enum bce_mb_type {
+ BCE_MB_REGISTER_CMD_SQ = 0x07,
+ BCE_MB_REGISTER_CMD_CQ = 0x08,
+ BCE_MB_REGISTER_QUEUE_REPLY = 0x0B,
+ BCE_MB_SET_FW_PROTOCOL_VER = 0x0C,
+ BCE_MB_SLEEP_NO_STATE = 0x14,
+ BCE_MB_RESTORE_NO_STATE = 0x15,
+ BCE_MB_SAVE_AND_SLEEP = 0x17,
+ BCE_MB_RESTORE_AND_WAKE = 0x18,
+ BCE_MB_SAVE_FAILURE = 0x19,
+ BCE_MB_SAVE_RESTORE_COMPLETE = 0x1A,
+};
+
+/*
+ * Queue types.
+ */
+enum bce_queue_type {
+ BCE_QUEUE_CQ = 0,
+ BCE_QUEUE_SQ = 1,
+};
+
+/*
+ * Completion queue entry.
+ */
+struct bce_qe_completion {
+ uint64_t result;
+ uint64_t data_size;
+ uint16_t qid; /* Source SQ */
+ uint16_t completion_index;
+ uint16_t status;
+ uint16_t flags;
+#define BCE_CQ_FLAG_PENDING 0x8000
+} __packed;
+
+/*
+ * Completion status codes.
+ */
+enum bce_completion_status {
+ BCE_COMP_SUCCESS = 0,
+ BCE_COMP_ERROR = 1,
+ BCE_COMP_ABORTED = 2,
+ BCE_COMP_NO_SPACE = 3,
+ BCE_COMP_OVERRUN = 4,
+};
+
+/*
+ * Submission queue entry.
+ */
+struct bce_qe_submission {
+ uint64_t length;
+ uint64_t addr;
+ uint64_t segl_addr; /* Scatter-gather list */
+ uint64_t segl_length;
+} __packed;
+
+/*
+ * Queue memory configuration (for mailbox registration).
+ */
+struct bce_queue_memcfg {
+ uint16_t qid;
+ uint16_t el_count;
+ uint16_t vector_or_cq;
+ uint16_t _pad;
+ uint64_t addr;
+ uint64_t length;
+} __packed;
+
+/*
+ * Completion data cached per SQ slot.
+ */
+struct bce_sq_completion_data {
+ uint32_t status;
+ uint64_t data_size;
+ uint64_t result;
+};
+
+/*
+ * Completion callback.
+ */
+struct bce_queue_sq;
+typedef void (*bce_sq_completion_fn)(struct bce_queue_sq *sq);
+
+/*
+ * Completion queue.
+ */
+struct bce_queue_cq {
+ int qid;
+ uint32_t el_count;
+ bus_dma_tag_t dma_tag;
+ bus_dmamap_t dma_map;
+ bus_addr_t dma_addr;
+ struct bce_qe_completion *data;
+ uint32_t index; /* Consumer index */
+};
+
+/*
+ * Submission queue.
+ */
+struct bce_queue_sq {
+ int qid;
+ uint32_t el_size;
+ uint32_t el_count;
+ bus_dma_tag_t dma_tag;
+ bus_dmamap_t dma_map;
+ bus_addr_t dma_addr;
+ void *data;
+ void *userdata;
+
+ /* Submission tracking */
+ volatile int available_commands;
+ uint32_t head;
+ uint32_t tail;
+
+ /* Completion tracking */
+ uint32_t completion_cidx;
+ uint32_t completion_tail;
+ struct bce_sq_completion_data *completion_data;
+ int has_pending;
+ bce_sq_completion_fn completion;
+};
+
+/*
+ * Command queue (wraps SQ for synchronous ops).
+ */
+struct bce_queue_cmdq_result {
+ struct sema cmpl;
+ uint32_t status;
+ uint64_t result;
+};
+
+struct bce_queue_cmdq {
+ struct bce_queue_sq *sq;
+ struct mtx lck;
+ struct bce_queue_cmdq_result **tres; /* Pending results per slot */
+};
+
+/*
+ * Command queue command IDs.
+ */
+enum bce_cmdq_cmd {
+ BCE_CMD_REGISTER_QUEUE = 0x20,
+ BCE_CMD_UNREGISTER_QUEUE = 0x30,
+ BCE_CMD_FLUSH_QUEUE = 0x40,
+ BCE_CMD_SET_QUEUE_PROPERTY = 0x50,
+};
+
+/*
+ * Register queue command payload (fits in BCE_CMD_SIZE).
+ */
+struct bce_cmdq_reg_cmd {
+ uint16_t cmd;
+ uint16_t flags;
+#define BCE_CMDQ_FLAG_NAMED 0x02
+#define BCE_CMDQ_FLAG_OUT 0x01
+ uint16_t qid;
+ uint16_t _pad;
+ uint16_t el_count;
+ uint16_t vector_or_cq;
+ uint16_t _pad2;
+ uint16_t name_len;
+ char name[0x20];
+ uint64_t addr;
+ uint64_t length;
+} __packed;
+
+/*
+ * Simple queue command (unregister/flush).
+ */
+struct bce_cmdq_simple_cmd {
+ uint16_t cmd;
+ uint16_t flags;
+ uint16_t qid;
+} __packed;
+
+/*
+ * Mailbox state.
+ */
+struct bce_mailbox {
+ struct resource *reg; /* BAR4 resource */
+ volatile int status; /* 0=idle, 1=pending, 2=ready */
+ int initialized;
+ struct sema mb_cmpl;
+ uint64_t mb_result;
+};
+
+/*
+ * Main device softc.
+ */
+struct apple_bce_softc {
+ device_t sc_dev;
+
+ /* PCI resources */
+ int sc_bar2_rid;
+ struct resource *sc_bar2; /* DMA registers */
+ int sc_bar4_rid;
+ struct resource *sc_bar4; /* Mailbox registers */
+
+ /* MSI */
+ int sc_msi_count;
+ int sc_irq_rid_mbox;
+ struct resource *sc_irq_mbox;
+ void *sc_irq_mbox_cookie;
+ int sc_irq_rid_dma;
+ struct resource *sc_irq_dma;
+ void *sc_irq_dma_cookie;
+
+ /* DMA */
+ bus_dma_tag_t sc_dma_tag;
+
+ /* Mailbox */
+ struct bce_mailbox sc_mbox;
+
+ /* Timestamp keepalive */
+ struct callout sc_timestamp_co;
+ struct mtx sc_timestamp_lock;
+ int sc_timestamp_stopped;
+
+ /* Command path */
+ struct bce_queue_cq *sc_cmd_cq;
+ struct bce_queue_cmdq *sc_cmd_cmdq;
+
+ /* Queue registry */
+ void *sc_queues[BCE_MAX_QUEUE_COUNT];
+ struct mtx sc_queues_lock;
+ struct bce_queue_cq *sc_cq_list[BCE_MAX_CQ_COUNT];
+ struct bce_queue_sq *sc_int_sq_list[BCE_MAX_QUEUE_COUNT];
+};
+
+/* Inline helpers */
+static __inline void *
+bce_sq_element(struct bce_queue_sq *sq, uint32_t idx)
+{
+ return ((char *)sq->data + (idx * sq->el_size));
+}
+
+static __inline struct bce_qe_completion *
+bce_cq_element(struct bce_queue_cq *cq, uint32_t idx)
+{
+ return (&cq->data[idx]);
+}
+
+#endif /* _APPLE_BCE_H_ */
diff --git a/sys/dev/apple_bce/apple_bce.c b/sys/dev/apple_bce/apple_bce.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce.c
@@ -0,0 +1,642 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple T2 Buffer Copy Engine (BCE) PCI driver.
+ * Provides the transport layer for T2 coprocessor communication:
+ * mailbox handshake, DMA queue setup, and firmware keepalive.
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/sema.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/callout.h>
+#include <sys/rman.h>
+#include <sys/time.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/atomic.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include "apple_bce.h"
+#include "apple_bce_mailbox.h"
+#include "apple_bce_queue.h"
+
+static int apple_bce_probe(device_t dev);
+static int apple_bce_attach(device_t dev);
+static int apple_bce_detach(device_t dev);
+static void apple_bce_timestamp_cb(void *arg);
+static void apple_bce_timestamp_init(struct apple_bce_softc *sc);
+static void apple_bce_timestamp_start(struct apple_bce_softc *sc,
+ int is_initial);
+static void apple_bce_timestamp_stop(struct apple_bce_softc *sc);
+
+/*
+ * Timestamp protocol -- T2 firmware expects periodic boottime updates.
+ *
+ * Register layout at BAR4 + 0xC000:
+ * regb + 0 (0xC000): high 32 bits / control
+ * regb + 8 (0xC008): low 32 bits / data
+ *
+ * Control opcodes (written to regb+8 with 0xFFFFFFFF to regb+0):
+ * -4 (0xFFFFFFFC): initial start
+ * -3 (0xFFFFFFFD): restart
+ * -2 (0xFFFFFFFE): stop
+ */
+static void
+apple_bce_timestamp_init(struct apple_bce_softc *sc)
+{
+ /* Read control register and barrier to sync with firmware */
+ bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP);
+ mb();
+}
+
+static void
+apple_bce_timestamp_start(struct apple_bce_softc *sc, int is_initial)
+{
+ /* Send start opcode: -4 for initial, -3 for restart */
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8,
+ is_initial ? 0xFFFFFFFC : 0xFFFFFFFD);
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF);
+
+ mtx_lock_spin(&sc->sc_timestamp_lock);
+ sc->sc_timestamp_stopped = 0;
+ mtx_unlock_spin(&sc->sc_timestamp_lock);
+
+ callout_reset(&sc->sc_timestamp_co, hz * BCE_TIMESTAMP_MS / 1000,
+ apple_bce_timestamp_cb, sc);
+}
+
+static void
+apple_bce_timestamp_stop(struct apple_bce_softc *sc)
+{
+ mtx_lock_spin(&sc->sc_timestamp_lock);
+ sc->sc_timestamp_stopped = 1;
+ mtx_unlock_spin(&sc->sc_timestamp_lock);
+
+ callout_drain(&sc->sc_timestamp_co);
+
+ /* Send stop opcode */
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, 0xFFFFFFFE);
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF);
+}
+
+static void
+apple_bce_timestamp_cb(void *arg)
+{
+ struct apple_bce_softc *sc = arg;
+ struct bintime bt;
+ uint64_t ns;
+
+ /* Read to sync, then barrier */
+ bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8);
+ mb();
+
+ /* Get boot time in nanoseconds */
+ binuptime(&bt);
+ ns = (uint64_t)bt.sec * 1000000000ULL +
+ (((uint64_t)(bt.frac >> 32) * 1000000000ULL) >> 32);
+
+ /* Write: low 32 bits to regb+8, high 32 bits to regb+0 */
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, (uint32_t)ns);
+ bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, (uint32_t)(ns >> 32));
+
+ mtx_lock_spin(&sc->sc_timestamp_lock);
+ if (!sc->sc_timestamp_stopped)
+ callout_schedule(&sc->sc_timestamp_co,
+ hz * BCE_TIMESTAMP_MS / 1000);
+ mtx_unlock_spin(&sc->sc_timestamp_lock);
+}
+
+/*
+ * IRQ handlers.
+ */
+static void
+apple_bce_mbox_intr(void *arg)
+{
+ struct apple_bce_softc *sc = arg;
+
+ bce_mailbox_handle_interrupt(&sc->sc_mbox);
+}
+
+static void
+apple_bce_dma_intr(void *arg)
+{
+ struct apple_bce_softc *sc = arg;
+ int i;
+
+ /*
+ * Only process registered CQs. sc_cq_list[] contains only CQ
+ * pointers -- separate from sc_queues[] which mixes CQ and SQ.
+ */
+ mtx_lock(&sc->sc_queues_lock);
+ for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
+ if (sc->sc_cq_list[i] != NULL)
+ bce_handle_cq_completions(sc, sc->sc_cq_list[i]);
+ }
+ mtx_unlock(&sc->sc_queues_lock);
+}
+
+/*
+ * Enable bus master on PCI function 0 (NVMe).
+ * T2 requires function 0 to be bus master for DMA on function 1.
+ */
+static int
+apple_bce_enable_pci0_busmaster(device_t dev)
+{
+ device_t pci0;
+ device_t bus;
+
+ bus = device_get_parent(dev);
+ if (bus == NULL)
+ return (ENXIO);
+
+ /*
+ * Find function 0 on the same bus/slot.
+ * Our device is function 1; function 0 is NVMe.
+ */
+ pci0 = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
+ pci_get_slot(dev), 0);
+ if (pci0 == NULL) {
+ device_printf(dev, "cannot find PCI function 0\n");
+ return (ENXIO);
+ }
+
+ pci_enable_busmaster(pci0);
+ device_printf(dev, "enabled bus master on function 0 (%s)\n",
+ device_get_nameunit(pci0));
+ return (0);
+}
+
+/*
+ * Firmware handshake via mailbox.
+ */
+static int
+apple_bce_fw_handshake(struct apple_bce_softc *sc)
+{
+ uint64_t reply;
+ int error;
+
+ error = bce_mailbox_send(&sc->sc_mbox,
+ BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VER, BCE_FW_PROTOCOL_VER),
+ &reply, BCE_MBOX_TIMEOUT_MS);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "firmware handshake timeout\n");
+ return (error);
+ }
+
+ if (BCE_MB_TYPE(reply) != BCE_MB_SET_FW_PROTOCOL_VER ||
+ BCE_MB_VALUE(reply) != BCE_FW_PROTOCOL_VER) {
+ device_printf(sc->sc_dev,
+ "firmware version mismatch: got type=%u val=0x%llx\n",
+ BCE_MB_TYPE(reply),
+ (unsigned long long)BCE_MB_VALUE(reply));
+ return (ENODEV);
+ }
+
+ device_printf(sc->sc_dev, "firmware handshake OK (protocol 0x%x)\n",
+ BCE_FW_PROTOCOL_VER);
+ return (0);
+}
+
+/*
+ * DMA callback for command queue registration.
+ */
+static void
+bce_reg_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+ struct { bus_addr_t addr; int error; } *cb = arg;
+
+ cb->error = error;
+ if (error == 0)
+ cb->addr = segs[0].ds_addr;
+}
+
+/*
+ * Register a command queue (CQ or SQ) with firmware via mailbox.
+ * The memcfg struct is DMA-mapped and its physical address sent
+ * in the mailbox message -- firmware reads the config from DMA.
+ */
+static int
+apple_bce_register_cmd_queue(struct apple_bce_softc *sc,
+ struct bce_queue_memcfg *cfg, int is_sq)
+{
+ bus_dma_tag_t tag;
+ bus_dmamap_t map;
+ bus_addr_t paddr;
+ struct bce_queue_memcfg *dma_cfg;
+ struct {
+ bus_addr_t addr;
+ int error;
+ } cb;
+ uint64_t reply;
+ int error, cmd_type;
+
+ /* Allocate DMA-coherent buffer for memcfg (8-byte aligned) */
+ error = bus_dma_tag_create(sc->sc_dma_tag,
+ 8, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+ NULL, NULL, sizeof(*cfg), 1, sizeof(*cfg),
+ BUS_DMA_WAITOK, NULL, NULL, &tag);
+ if (error != 0)
+ return (error);
+
+ error = bus_dmamem_alloc(tag, (void **)&dma_cfg,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &map);
+ if (error != 0) {
+ bus_dma_tag_destroy(tag);
+ return (error);
+ }
+
+ /* Copy config and load DMA address */
+ memcpy(dma_cfg, cfg, sizeof(*cfg));
+
+ cb.error = 0;
+ error = bus_dmamap_load(tag, map, dma_cfg, sizeof(*cfg),
+ bce_reg_dma_cb, &cb, BUS_DMA_WAITOK);
+ if (error != 0 || cb.error != 0) {
+ bus_dmamem_free(tag, dma_cfg, map);
+ bus_dma_tag_destroy(tag);
+ return (error != 0 ? error : cb.error);
+ }
+ paddr = cb.addr;
+
+ /* Sync DMA buffer before device access */
+ bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
+
+ /* Send DMA address of memcfg to firmware */
+ cmd_type = is_sq ? BCE_MB_REGISTER_CMD_SQ : BCE_MB_REGISTER_CMD_CQ;
+ error = bce_mailbox_send(&sc->sc_mbox,
+ BCE_MB_MSG(cmd_type, paddr), &reply, BCE_MBOX_TIMEOUT_MS);
+
+ bus_dmamap_sync(tag, map, BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(tag, map);
+ bus_dmamem_free(tag, dma_cfg, map);
+ bus_dma_tag_destroy(tag);
+
+ if (error != 0)
+ return (error);
+
+ if (BCE_MB_TYPE(reply) != BCE_MB_REGISTER_QUEUE_REPLY) {
+ device_printf(sc->sc_dev,
+ "unexpected queue registration reply: type=%u\n",
+ BCE_MB_TYPE(reply));
+ return (EINVAL);
+ }
+
+ return (0);
+}
+
+/*
+ * Setup command queues (CQ qid=0, SQ qid=1).
+ */
+static int
+apple_bce_setup_cmd_queues(struct apple_bce_softc *sc)
+{
+ struct bce_queue_memcfg cfg;
+ struct bce_queue_sq *sq;
+ int error;
+
+ /* Allocate command CQ (qid 0, 32 entries) */
+ sc->sc_cmd_cq = bce_alloc_cq(sc, 0, 32);
+ if (sc->sc_cmd_cq == NULL) {
+ device_printf(sc->sc_dev, "failed to allocate command CQ\n");
+ return (ENOMEM);
+ }
+
+ /* Register CQ with firmware via DMA-mapped memcfg */
+ bce_get_cq_memcfg(sc->sc_cmd_cq, &cfg);
+ error = apple_bce_register_cmd_queue(sc, &cfg, 0);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "failed to register command CQ\n");
+ goto fail_cq;
+ }
+
+ /* Store CQ in queue registries */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[0] = sc->sc_cmd_cq;
+ sc->sc_cq_list[0] = sc->sc_cmd_cq;
+ mtx_unlock(&sc->sc_queues_lock);
+
+ /* Allocate command SQ (qid 1, 64-byte elements, 32 entries) */
+ sq = bce_alloc_sq(sc, 1, BCE_CMD_SIZE, 32, NULL, NULL);
+ if (sq == NULL) {
+ device_printf(sc->sc_dev, "failed to allocate command SQ\n");
+ goto fail_cq;
+ }
+
+ /* Wrap SQ in command queue (sets completion callback internally) */
+ sc->sc_cmd_cmdq = bce_alloc_cmdq(sc, sq);
+ if (sc->sc_cmd_cmdq == NULL) {
+ bce_free_sq(sc, sq);
+ goto fail_cq;
+ }
+
+ /* Register SQ with firmware via DMA-mapped memcfg */
+ bce_get_sq_memcfg(sq, sc->sc_cmd_cq, &cfg);
+ error = apple_bce_register_cmd_queue(sc, &cfg, 1);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "failed to register command SQ\n");
+ goto fail_sq;
+ }
+
+ /* Store SQ in queue registry */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[1] = sq;
+ sc->sc_int_sq_list[0] = sq;
+ mtx_unlock(&sc->sc_queues_lock);
+
+ device_printf(sc->sc_dev, "command queues created (CQ=0, SQ=1)\n");
+ return (0);
+
+fail_sq:
+ bce_free_cmdq(sc->sc_cmd_cmdq);
+ sc->sc_cmd_cmdq = NULL;
+ bce_free_sq(sc, sq);
+fail_cq:
+ /* Clear CQ from registries before freeing */
+ mtx_lock(&sc->sc_queues_lock);
+ sc->sc_queues[0] = NULL;
+ sc->sc_cq_list[0] = NULL;
+ mtx_unlock(&sc->sc_queues_lock);
+ bce_free_cq(sc, sc->sc_cmd_cq);
+ sc->sc_cmd_cq = NULL;
+ return (error != 0 ? error : ENOMEM);
+}
+
+/*
+ * PCI probe.
+ */
+static int
+apple_bce_probe(device_t dev)
+{
+ if (pci_get_vendor(dev) != BCE_PCI_VENDOR_APPLE ||
+ pci_get_device(dev) != BCE_PCI_DEVICE_T2)
+ return (ENXIO);
+
+ /* Only attach to function 1 (BCE), not function 0 (NVMe) */
+ if (pci_get_function(dev) != 1)
+ return (ENXIO);
+
+ device_set_desc(dev, "Apple T2 Buffer Copy Engine");
+ return (BUS_PROBE_DEFAULT);
+}
+
+/*
+ * PCI attach.
+ */
+static int
+apple_bce_attach(device_t dev)
+{
+ struct apple_bce_softc *sc = device_get_softc(dev);
+ int error;
+
+ sc->sc_dev = dev;
+ pci_enable_busmaster(dev);
+ mtx_init(&sc->sc_queues_lock, "bce_queues", NULL, MTX_DEF);
+ mtx_init(&sc->sc_timestamp_lock, "bce_ts", NULL, MTX_SPIN);
+ callout_init(&sc->sc_timestamp_co, 1);
+
+ /* Map BAR2 (DMA) and BAR4 (mailbox) */
+ sc->sc_bar2_rid = PCIR_BAR(2);
+ sc->sc_bar2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+ &sc->sc_bar2_rid, RF_ACTIVE);
+ if (sc->sc_bar2 == NULL) {
+ device_printf(dev, "cannot map BAR2 (DMA)\n");
+ error = ENXIO;
+ goto fail;
+ }
+
+ sc->sc_bar4_rid = PCIR_BAR(4);
+ sc->sc_bar4 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+ &sc->sc_bar4_rid, RF_ACTIVE);
+ if (sc->sc_bar4 == NULL) {
+ device_printf(dev, "cannot map BAR4 (mailbox)\n");
+ error = ENXIO;
+ goto fail;
+ }
+
+ /* Allocate MSI vectors (need at least 5: mbox=0, dma=4) */
+ sc->sc_msi_count = 8; /* Must be power of 2 for FreeBSD */
+ error = pci_alloc_msi(dev, &sc->sc_msi_count);
+ if (error != 0 || sc->sc_msi_count < 8) {
+ device_printf(dev, "cannot allocate MSI vectors: "
+ "error=%d got=%d\n", error, sc->sc_msi_count);
+ if (error == 0 && sc->sc_msi_count > 0)
+ pci_release_msi(dev);
+ sc->sc_msi_count = 0;
+ error = ENXIO;
+ goto fail;
+ }
+
+ /* Initialize mailbox before installing handlers */
+ bce_mailbox_init(&sc->sc_mbox, sc->sc_bar4);
+
+ /* Setup IRQ: vector 0 = mailbox, vector 4 = DMA */
+ sc->sc_irq_rid_mbox = 1; /* MSI vectors start at rid 1 */
+ sc->sc_irq_mbox = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+ &sc->sc_irq_rid_mbox, RF_ACTIVE);
+ if (sc->sc_irq_mbox == NULL) {
+ device_printf(dev, "cannot allocate mailbox IRQ\n");
+ error = ENXIO;
+ goto fail;
+ }
+
+ error = bus_setup_intr(dev, sc->sc_irq_mbox,
+ INTR_TYPE_MISC | INTR_MPSAFE,
+ NULL, apple_bce_mbox_intr, sc, &sc->sc_irq_mbox_cookie);
+ if (error != 0) {
+ device_printf(dev, "cannot setup mailbox IRQ\n");
+ goto fail;
+ }
+
+ sc->sc_irq_rid_dma = 5; /* Vector 4 = rid 5 */
+ sc->sc_irq_dma = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+ &sc->sc_irq_rid_dma, RF_ACTIVE);
+ if (sc->sc_irq_dma == NULL) {
+ device_printf(dev, "cannot allocate DMA IRQ\n");
+ error = ENXIO;
+ goto fail;
+ }
+
+ error = bus_setup_intr(dev, sc->sc_irq_dma,
+ INTR_TYPE_MISC | INTR_MPSAFE,
+ NULL, apple_bce_dma_intr, sc, &sc->sc_irq_dma_cookie);
+ if (error != 0) {
+ device_printf(dev, "cannot setup DMA IRQ\n");
+ goto fail;
+ }
+
+ /* Create parent DMA tag with 37-bit addressing limit */
+ error = bus_dma_tag_create(bus_get_dma_tag(dev),
+ 1, 0, /* alignment, boundary */
+ (1ULL << 37) - 1, /* lowaddr: 37-bit limit */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter */
+ BUS_SPACE_MAXSIZE, /* maxsize */
+ BUS_SPACE_UNRESTRICTED, /* nsegments */
+ BUS_SPACE_MAXSIZE, /* maxsegsize */
+ 0, /* flags */
+ NULL, NULL, /* lockfunc */
+ &sc->sc_dma_tag);
+ if (error != 0) {
+ device_printf(dev, "cannot create DMA tag\n");
+ goto fail;
+ }
+
+ /*
+ * Enable bus master on function 0 (NVMe) -- T2 quirk.
+ * Must be done before firmware handshake; T2 rejects DMA
+ * on function 1 unless function 0 is bus master.
+ */
+ apple_bce_enable_pci0_busmaster(dev);
+
+ /* Initialize and start timestamp keepalive */
+ apple_bce_timestamp_init(sc);
+ apple_bce_timestamp_start(sc, 1);
+
+ /* Firmware handshake */
+ error = apple_bce_fw_handshake(sc);
+ if (error != 0)
+ goto fail;
+
+ /* Setup command queues */
+ error = apple_bce_setup_cmd_queues(sc);
+ if (error != 0)
+ goto fail;
+
+ device_printf(dev, "Apple T2 BCE initialized\n");
+ return (0);
+
+fail:
+ apple_bce_detach(dev);
+ return (error);
+}
+
+/*
+ * PCI detach.
+ */
+static int
+apple_bce_detach(device_t dev)
+{
+ struct apple_bce_softc *sc = device_get_softc(dev);
+
+ /* 1. Stop timestamp */
+ if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock))
+ apple_bce_timestamp_stop(sc);
+ else
+ callout_drain(&sc->sc_timestamp_co);
+
+ /* 2. Tear down IRQs first -- no more interrupts after this */
+ if (sc->sc_irq_dma_cookie != NULL) {
+ bus_teardown_intr(dev, sc->sc_irq_dma, sc->sc_irq_dma_cookie);
+ sc->sc_irq_dma_cookie = NULL;
+ }
+ if (sc->sc_irq_mbox_cookie != NULL) {
+ bus_teardown_intr(dev, sc->sc_irq_mbox,
+ sc->sc_irq_mbox_cookie);
+ sc->sc_irq_mbox_cookie = NULL;
+ }
+
+ /* 3. Free command queues (safe now -- no IRQs can fire) */
+ if (sc->sc_cmd_cmdq != NULL) {
+ struct bce_queue_sq *cmd_sq;
+
+ cmd_sq = sc->sc_cmd_cmdq->sq;
+ bce_free_cmdq(sc->sc_cmd_cmdq);
+ sc->sc_cmd_cmdq = NULL;
+ bce_free_sq(sc, cmd_sq);
+ }
+ if (sc->sc_cmd_cq != NULL) {
+ bce_free_cq(sc, sc->sc_cmd_cq);
+ sc->sc_cmd_cq = NULL;
+ }
+
+ /* 4. Clear queue registries */
+ if (mtx_initialized(&sc->sc_queues_lock)) {
+ mtx_lock(&sc->sc_queues_lock);
+ memset(sc->sc_queues, 0, sizeof(sc->sc_queues));
+ memset(sc->sc_cq_list, 0, sizeof(sc->sc_cq_list));
+ memset(sc->sc_int_sq_list, 0, sizeof(sc->sc_int_sq_list));
+ mtx_unlock(&sc->sc_queues_lock);
+ }
+
+ /* 5. Destroy mailbox */
+ bce_mailbox_destroy(&sc->sc_mbox);
+
+ /* 6. Release DMA tag */
+ if (sc->sc_dma_tag != NULL) {
+ bus_dma_tag_destroy(sc->sc_dma_tag);
+ sc->sc_dma_tag = NULL;
+ }
+
+ /* 7. Release IRQ resources */
+ if (sc->sc_irq_dma != NULL) {
+ bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_dma,
+ sc->sc_irq_dma);
+ sc->sc_irq_dma = NULL;
+ }
+ if (sc->sc_irq_mbox != NULL) {
+ bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_mbox,
+ sc->sc_irq_mbox);
+ sc->sc_irq_mbox = NULL;
+ }
+ if (sc->sc_msi_count > 0) {
+ pci_release_msi(dev);
+ sc->sc_msi_count = 0;
+ }
+
+ /* 8. Release BARs */
+ if (sc->sc_bar4 != NULL) {
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar4_rid,
+ sc->sc_bar4);
+ sc->sc_bar4 = NULL;
+ }
+ if (sc->sc_bar2 != NULL) {
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar2_rid,
+ sc->sc_bar2);
+ sc->sc_bar2 = NULL;
+ }
+
+ if (mtx_initialized(&sc->sc_timestamp_lock))
+ mtx_destroy(&sc->sc_timestamp_lock);
+ if (mtx_initialized(&sc->sc_queues_lock))
+ mtx_destroy(&sc->sc_queues_lock);
+
+ return (0);
+}
+
+static device_method_t apple_bce_methods[] = {
+ DEVMETHOD(device_probe, apple_bce_probe),
+ DEVMETHOD(device_attach, apple_bce_attach),
+ DEVMETHOD(device_detach, apple_bce_detach),
+ DEVMETHOD_END
+};
+
+static driver_t apple_bce_driver = {
+ "apple_bce",
+ apple_bce_methods,
+ sizeof(struct apple_bce_softc)
+};
+
+DRIVER_MODULE(apple_bce, pci, apple_bce_driver, NULL, NULL);
+MODULE_DEPEND(apple_bce, pci, 1, 1, 1);
+static const struct {
+ uint16_t vendor;
+ uint16_t device;
+} apple_bce_pnp[] = {
+ { BCE_PCI_VENDOR_APPLE, BCE_PCI_DEVICE_T2 },
+};
+
+MODULE_PNP_INFO("U16:vendor;U16:device", pci, apple_bce, apple_bce_pnp,
+ nitems(apple_bce_pnp));
diff --git a/sys/dev/apple_bce/apple_bce_mailbox.h b/sys/dev/apple_bce/apple_bce_mailbox.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce_mailbox.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple BCE mailbox interface.
+ */
+
+#ifndef _APPLE_BCE_MAILBOX_H_
+#define _APPLE_BCE_MAILBOX_H_
+
+#include "apple_bce.h"
+
+int bce_mailbox_init(struct bce_mailbox *mb, struct resource *reg);
+void bce_mailbox_destroy(struct bce_mailbox *mb);
+int bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv,
+ unsigned int timeout_ms);
+int bce_mailbox_handle_interrupt(struct bce_mailbox *mb);
+
+#endif /* _APPLE_BCE_MAILBOX_H_ */
diff --git a/sys/dev/apple_bce/apple_bce_mailbox.c b/sys/dev/apple_bce/apple_bce_mailbox.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce_mailbox.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple BCE mailbox -- 64-bit request/reply over BAR4 registers.
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/sema.h>
+#include <machine/bus.h>
+#include <machine/atomic.h>
+
+#include "apple_bce.h"
+#include "apple_bce_mailbox.h"
+
+int
+bce_mailbox_init(struct bce_mailbox *mb, struct resource *reg)
+{
+ mb->reg = reg;
+ mb->status = 0;
+ sema_init(&mb->mb_cmpl, 0, "bce_mb");
+ mb->mb_result = 0;
+ mb->initialized = 1;
+ return (0);
+}
+
+void
+bce_mailbox_destroy(struct bce_mailbox *mb)
+{
+ if (mb->initialized) {
+ sema_destroy(&mb->mb_cmpl);
+ mb->initialized = 0;
+ }
+}
+
+/*
+ * Send a mailbox message and wait for reply.
+ * Only one message in flight at a time (enforced by cmpxchg).
+ */
+int
+bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv,
+ unsigned int timeout_ms)
+{
+ int old;
+
+ /* Acquire: idle (0) -> pending (1) */
+ old = atomic_cmpset_int(&mb->status, 0, 1);
+ if (old == 0)
+ return (EBUSY);
+
+ /* Write 64-bit message as 4x u32 to MBOX_OUT */
+ bus_write_4(mb->reg, BCE_REG_MBOX_OUT, (uint32_t)msg);
+ bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 4, (uint32_t)(msg >> 32));
+ bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 8, 0);
+ bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 12, 0);
+
+ /* Wait for interrupt-driven reply */
+ if (sema_timedwait(&mb->mb_cmpl, hz * timeout_ms / 1000) != 0) {
+ /* Timeout -- reset to idle */
+ atomic_store_int(&mb->status, 0);
+ return (ETIMEDOUT);
+ }
+
+ if (atomic_load_int(&mb->status) != 2) {
+ atomic_store_int(&mb->status, 0);
+ return (ETIMEDOUT);
+ }
+
+ if (recv != NULL)
+ *recv = mb->mb_result;
+
+ atomic_store_int(&mb->status, 0);
+ return (0);
+}
+
+/*
+ * Called from ISR to retrieve mailbox reply.
+ */
+int
+bce_mailbox_handle_interrupt(struct bce_mailbox *mb)
+{
+ uint32_t res, lo, hi;
+ int count;
+
+ res = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY_CTR);
+ count = (res >> 20) & 0xf;
+
+ if (count == 0)
+ return (ENOMSG);
+
+ while (count-- > 0) {
+ lo = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY);
+ hi = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 4);
+ bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 8);
+ bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 12);
+ mb->mb_result = ((uint64_t)hi << 32) | lo;
+ }
+
+ if (atomic_load_int(&mb->status) == 1) {
+ atomic_store_int(&mb->status, 2);
+ sema_post(&mb->mb_cmpl);
+ }
+
+ return (0);
+}
diff --git a/sys/dev/apple_bce/apple_bce_queue.h b/sys/dev/apple_bce/apple_bce_queue.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce_queue.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple BCE queue management.
+ */
+
+#ifndef _APPLE_BCE_QUEUE_H_
+#define _APPLE_BCE_QUEUE_H_
+
+#include "apple_bce.h"
+
+struct apple_bce_softc;
+
+/* CQ operations */
+struct bce_queue_cq *bce_alloc_cq(struct apple_bce_softc *sc, int qid,
+ uint32_t el_count);
+void bce_free_cq(struct apple_bce_softc *sc, struct bce_queue_cq *cq);
+void bce_get_cq_memcfg(struct bce_queue_cq *cq,
+ struct bce_queue_memcfg *cfg);
+void bce_handle_cq_completions(struct apple_bce_softc *sc,
+ struct bce_queue_cq *cq);
+
+/* SQ operations */
+struct bce_queue_sq *bce_alloc_sq(struct apple_bce_softc *sc, int qid,
+ uint32_t el_size, uint32_t el_count,
+ bce_sq_completion_fn compl, void *userdata);
+void bce_free_sq(struct apple_bce_softc *sc, struct bce_queue_sq *sq);
+void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq,
+ struct bce_queue_memcfg *cfg);
+
+/* Submission helpers */
+int bce_reserve_submission(struct bce_queue_sq *sq);
+void *bce_next_submission(struct bce_queue_sq *sq);
+void bce_submit_to_device(struct apple_bce_softc *sc,
+ struct bce_queue_sq *sq);
+void bce_notify_submission_complete(struct bce_queue_sq *sq);
+
+/* Command queue */
+struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_softc *sc,
+ struct bce_queue_sq *sq);
+void bce_free_cmdq(struct bce_queue_cmdq *cmdq);
+uint32_t bce_cmd_register_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, struct bce_queue_memcfg *cfg,
+ const char *name, int isdirout);
+uint32_t bce_cmd_unregister_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, int qid);
+uint32_t bce_cmd_flush_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, int qid);
+
+#endif /* _APPLE_BCE_QUEUE_H_ */
diff --git a/sys/dev/apple_bce/apple_bce_queue.c b/sys/dev/apple_bce/apple_bce_queue.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/apple_bce/apple_bce_queue.c
@@ -0,0 +1,492 @@
+/*
+ * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Apple BCE queue management -- CQ, SQ, and command queue operations.
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/sema.h>
+#include <sys/systm.h>
+#include <machine/bus.h>
+#include <machine/atomic.h>
+
+#include "apple_bce.h"
+#include "apple_bce_queue.h"
+
+static MALLOC_DEFINE(M_BCE, "apple_bce", "Apple BCE driver");
+
+#define BCE_CMDQ_TIMEOUT_MS 5000
+
+/*
+ * DMA callback for bus_dmamap_load.
+ */
+struct bce_dma_cb_arg {
+ bus_addr_t addr;
+ int error;
+};
+
+static int
+bce_cmdq_wait(struct bce_queue_cmdq *cmdq, uint32_t slot,
+ struct bce_queue_cmdq_result *res)
+{
+ int error;
+
+ error = sema_timedwait(&res->cmpl, hz * BCE_CMDQ_TIMEOUT_MS / 1000);
+ if (error == 0)
+ return (0);
+
+ mtx_lock(&cmdq->lck);
+ if (cmdq->tres[slot] == res)
+ cmdq->tres[slot] = NULL;
+ mtx_unlock(&cmdq->lck);
+
+ return (ETIMEDOUT);
+}
+
+static void
+bce_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+ struct bce_dma_cb_arg *cb = arg;
+
+ cb->error = error;
+ if (error == 0)
+ cb->addr = segs[0].ds_addr;
+}
+
+/*
+ * Allocate a completion queue with DMA-coherent memory.
+ */
+struct bce_queue_cq *
+bce_alloc_cq(struct apple_bce_softc *sc, int qid, uint32_t el_count)
+{
+ struct bce_queue_cq *cq;
+ struct bce_dma_cb_arg cb;
+ int error;
+
+ cq = malloc(sizeof(*cq), M_BCE, M_WAITOK | M_ZERO);
+ cq->qid = qid;
+ cq->el_count = el_count;
+ cq->index = 0;
+
+ error = bus_dma_tag_create(sc->sc_dma_tag,
+ 4, 0, /* alignment, boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter */
+ el_count * sizeof(struct bce_qe_completion),
+ 1, /* nsegments */
+ el_count * sizeof(struct bce_qe_completion),
+ BUS_DMA_WAITOK, /* flags */
+ NULL, NULL, /* lockfunc */
+ &cq->dma_tag);
+ if (error) {
+ free(cq, M_BCE);
+ return (NULL);
+ }
+
+ error = bus_dmamem_alloc(cq->dma_tag, (void **)&cq->data,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &cq->dma_map);
+ if (error) {
+ bus_dma_tag_destroy(cq->dma_tag);
+ free(cq, M_BCE);
+ return (NULL);
+ }
+
+ error = bus_dmamap_load(cq->dma_tag, cq->dma_map, cq->data,
+ el_count * sizeof(struct bce_qe_completion),
+ bce_dma_cb, &cb, BUS_DMA_WAITOK);
+ if (error || cb.error) {
+ bus_dmamem_free(cq->dma_tag, cq->data, cq->dma_map);
+ bus_dma_tag_destroy(cq->dma_tag);
+ free(cq, M_BCE);
+ return (NULL);
+ }
+ cq->dma_addr = cb.addr;
+
+ return (cq);
+}
+
+void
+bce_free_cq(struct apple_bce_softc *sc, struct bce_queue_cq *cq)
+{
+ if (cq == NULL)
+ return;
+ bus_dmamap_unload(cq->dma_tag, cq->dma_map);
+ bus_dmamem_free(cq->dma_tag, cq->data, cq->dma_map);
+ bus_dma_tag_destroy(cq->dma_tag);
+ free(cq, M_BCE);
+}
+
+void
+bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg)
+{
+ cfg->qid = (uint16_t)cq->qid;
+ cfg->el_count = (uint16_t)cq->el_count;
+ cfg->vector_or_cq = 0;
+ cfg->_pad = 0;
+ cfg->addr = cq->dma_addr;
+ cfg->length = cq->el_count * sizeof(struct bce_qe_completion);
+}
+
+/*
+ * Process completions from a CQ and dispatch to the target SQs.
+ */
+void
+bce_handle_cq_completions(struct apple_bce_softc *sc, struct bce_queue_cq *cq)
+{
+ struct bce_qe_completion *e;
+ struct bce_queue_sq *sq;
+ int i;
+ bus_dmamap_sync(cq->dma_tag, cq->dma_map, BUS_DMASYNC_POSTREAD);
+
+ e = bce_cq_element(cq, cq->index);
+ if (!(e->flags & BCE_CQ_FLAG_PENDING))
+ return;
+
+ while ((e = bce_cq_element(cq, cq->index))->flags &
+ BCE_CQ_FLAG_PENDING) {
+ /* Route completion to target SQ (skip qid 0 which is the CQ) */
+ if (e->qid > 0 && e->qid < BCE_MAX_QUEUE_COUNT) {
+ sq = (struct bce_queue_sq *)sc->sc_queues[e->qid];
+ if (sq != NULL &&
+ e->completion_index < sq->el_count) {
+ sq->completion_data[e->completion_index].status =
+ e->status;
+ sq->completion_data[e->completion_index].data_size =
+ e->data_size;
+ sq->completion_data[e->completion_index].result =
+ e->result;
+ /* Advance tail so completion callback sees it */
+ sq->completion_tail =
+ (e->completion_index + 1) % sq->el_count;
+ sq->has_pending = 1;
+ }
+ }
+
+ e->flags = 0;
+ cq->index = (cq->index + 1) % cq->el_count;
+ }
+
+ bus_dmamap_sync(cq->dma_tag, cq->dma_map, BUS_DMASYNC_PREWRITE);
+
+ /* Ring doorbell with updated consumer index */
+ bus_write_4(sc->sc_bar2, BCE_REG_DOORBELL_BASE + cq->qid * 4,
+ cq->index);
+
+ /* Fire callbacks on SQs that received completions */
+ for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) {
+ sq = sc->sc_int_sq_list[i];
+ if (sq != NULL && sq->has_pending) {
+ sq->has_pending = 0;
+ if (sq->completion != NULL)
+ sq->completion(sq);
+ }
+ }
+}
+
+/*
+ * Allocate a submission queue with DMA-coherent memory.
+ */
+struct bce_queue_sq *
+bce_alloc_sq(struct apple_bce_softc *sc, int qid, uint32_t el_size,
+ uint32_t el_count, bce_sq_completion_fn compl, void *userdata)
+{
+ struct bce_queue_sq *sq;
+ struct bce_dma_cb_arg cb;
+ int error;
+
+ sq = malloc(sizeof(*sq), M_BCE, M_WAITOK | M_ZERO);
+ sq->qid = qid;
+ sq->el_size = el_size;
+ sq->el_count = el_count;
+ sq->completion = compl;
+ sq->userdata = userdata;
+ sq->available_commands = el_count - 1;
+
+ error = bus_dma_tag_create(sc->sc_dma_tag,
+ 4, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+ NULL, NULL,
+ el_count * el_size, 1, el_count * el_size,
+ BUS_DMA_WAITOK,
+ NULL, NULL,
+ &sq->dma_tag);
+ if (error) {
+ free(sq, M_BCE);
+ return (NULL);
+ }
+
+ error = bus_dmamem_alloc(sq->dma_tag, &sq->data,
+ BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sq->dma_map);
+ if (error) {
+ bus_dma_tag_destroy(sq->dma_tag);
+ free(sq, M_BCE);
+ return (NULL);
+ }
+
+ error = bus_dmamap_load(sq->dma_tag, sq->dma_map, sq->data,
+ el_count * el_size, bce_dma_cb, &cb, BUS_DMA_WAITOK);
+ if (error || cb.error) {
+ bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map);
+ bus_dma_tag_destroy(sq->dma_tag);
+ free(sq, M_BCE);
+ return (NULL);
+ }
+ sq->dma_addr = cb.addr;
+
+ sq->completion_data = malloc(sizeof(*sq->completion_data) * el_count,
+ M_BCE, M_WAITOK | M_ZERO);
+
+ return (sq);
+}
+
+void
+bce_free_sq(struct apple_bce_softc *sc, struct bce_queue_sq *sq)
+{
+ if (sq == NULL)
+ return;
+ free(sq->completion_data, M_BCE);
+ bus_dmamap_unload(sq->dma_tag, sq->dma_map);
+ bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map);
+ bus_dma_tag_destroy(sq->dma_tag);
+ free(sq, M_BCE);
+}
+
+void
+bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq,
+ struct bce_queue_memcfg *cfg)
+{
+ cfg->qid = (uint16_t)sq->qid;
+ cfg->el_count = (uint16_t)sq->el_count;
+ cfg->vector_or_cq = (uint16_t)cq->qid;
+ cfg->_pad = 0;
+ cfg->addr = sq->dma_addr;
+ cfg->length = sq->el_count * sq->el_size;
+}
+
+int
+bce_reserve_submission(struct bce_queue_sq *sq)
+{
+ int old, new;
+
+ do {
+ old = atomic_load_int(&sq->available_commands);
+ if (old <= 0)
+ return (EAGAIN);
+ new = old - 1;
+ } while (!atomic_cmpset_int(&sq->available_commands, old, new));
+
+ return (0);
+}
+
+void *
+bce_next_submission(struct bce_queue_sq *sq)
+{
+ void *ret;
+
+ ret = bce_sq_element(sq, sq->tail);
+ sq->tail = (sq->tail + 1) % sq->el_count;
+ return (ret);
+}
+
+void
+bce_submit_to_device(struct apple_bce_softc *sc, struct bce_queue_sq *sq)
+{
+ bus_dmamap_sync(sq->dma_tag, sq->dma_map, BUS_DMASYNC_PREWRITE);
+ bus_write_4(sc->sc_bar2, BCE_REG_DOORBELL_BASE + sq->qid * 4,
+ sq->tail);
+}
+
+void
+bce_notify_submission_complete(struct bce_queue_sq *sq)
+{
+ sq->head = (sq->head + 1) % sq->el_count;
+ atomic_add_int(&sq->available_commands, 1);
+}
+
+/*
+ * Command queue -- wraps an SQ for synchronous control operations.
+ */
+static void bce_cmdq_completion(struct bce_queue_sq *sq);
+
+struct bce_queue_cmdq *
+bce_alloc_cmdq(struct apple_bce_softc *sc, struct bce_queue_sq *sq)
+{
+ struct bce_queue_cmdq *cmdq;
+
+ cmdq = malloc(sizeof(*cmdq), M_BCE, M_WAITOK | M_ZERO);
+ cmdq->sq = sq;
+ mtx_init(&cmdq->lck, "bce_cmdq", NULL, MTX_DEF);
+ cmdq->tres = malloc(sizeof(void *) * sq->el_count, M_BCE,
+ M_WAITOK | M_ZERO);
+
+ /* Wire up completion callback */
+ sq->completion = bce_cmdq_completion;
+ sq->userdata = cmdq;
+
+ return (cmdq);
+}
+
+void
+bce_free_cmdq(struct bce_queue_cmdq *cmdq)
+{
+ if (cmdq == NULL)
+ return;
+ mtx_destroy(&cmdq->lck);
+ free(cmdq->tres, M_BCE);
+ free(cmdq, M_BCE);
+}
+
+/*
+ * Command queue completion callback -- wake waiters.
+ */
+static void
+bce_cmdq_completion(struct bce_queue_sq *sq)
+{
+ struct bce_queue_cmdq *cmdq = sq->userdata;
+ struct bce_queue_cmdq_result *res;
+
+ mtx_lock(&cmdq->lck);
+ while (sq->completion_cidx != sq->completion_tail) {
+ struct bce_sq_completion_data *cd;
+
+ cd = &sq->completion_data[sq->completion_cidx];
+ res = cmdq->tres[sq->completion_cidx];
+ if (res != NULL) {
+ res->status = cd->status;
+ res->result = cd->result;
+ sema_post(&res->cmpl);
+ cmdq->tres[sq->completion_cidx] = NULL;
+ }
+ sq->completion_cidx = (sq->completion_cidx + 1) %
+ sq->el_count;
+ bce_notify_submission_complete(sq);
+ }
+ mtx_unlock(&cmdq->lck);
+}
+
+uint32_t
+bce_cmd_register_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, struct bce_queue_memcfg *cfg,
+ const char *name, int isdirout)
+{
+ struct bce_queue_cmdq_result res;
+ struct bce_cmdq_reg_cmd *cmd;
+ uint32_t slot;
+ int error;
+
+ sema_init(&res.cmpl, 0, "bce_cmd");
+
+ if (bce_reserve_submission(cmdq->sq) != 0) {
+ sema_destroy(&res.cmpl);
+ return (EAGAIN);
+ }
+
+ mtx_lock(&cmdq->lck);
+ slot = cmdq->sq->tail;
+ cmdq->tres[slot] = &res;
+ cmd = bce_next_submission(cmdq->sq);
+
+ memset(cmd, 0, BCE_CMD_SIZE);
+ cmd->cmd = BCE_CMD_REGISTER_QUEUE;
+ cmd->flags = (name ? BCE_CMDQ_FLAG_NAMED : 0) |
+ (isdirout ? BCE_CMDQ_FLAG_OUT : 0);
+ cmd->qid = cfg->qid;
+ cmd->el_count = cfg->el_count;
+ cmd->vector_or_cq = cfg->vector_or_cq;
+ if (name != NULL) {
+ cmd->name_len = (uint16_t)MIN(strlen(name), sizeof(cmd->name));
+ memcpy(cmd->name, name, cmd->name_len);
+ }
+ cmd->addr = cfg->addr;
+ cmd->length = cfg->length;
+
+ bce_submit_to_device(sc, cmdq->sq);
+ mtx_unlock(&cmdq->lck);
+
+ error = bce_cmdq_wait(cmdq, slot, &res);
+ sema_destroy(&res.cmpl);
+ if (error != 0)
+ return (error);
+
+ return (res.status);
+}
+
+uint32_t
+bce_cmd_unregister_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, int qid)
+{
+ struct bce_queue_cmdq_result res;
+ struct bce_cmdq_simple_cmd *cmd;
+ uint32_t slot;
+ int error;
+
+ sema_init(&res.cmpl, 0, "bce_cmd");
+
+ if (bce_reserve_submission(cmdq->sq) != 0) {
+ sema_destroy(&res.cmpl);
+ return (EAGAIN);
+ }
+
+ mtx_lock(&cmdq->lck);
+ slot = cmdq->sq->tail;
+ cmdq->tres[slot] = &res;
+ cmd = bce_next_submission(cmdq->sq);
+
+ memset(cmd, 0, BCE_CMD_SIZE);
+ cmd->cmd = BCE_CMD_UNREGISTER_QUEUE;
+ cmd->qid = (uint16_t)qid;
+
+ bce_submit_to_device(sc, cmdq->sq);
+ mtx_unlock(&cmdq->lck);
+
+ error = bce_cmdq_wait(cmdq, slot, &res);
+ sema_destroy(&res.cmpl);
+ if (error != 0)
+ return (error);
+
+ return (res.status);
+}
+
+uint32_t
+bce_cmd_flush_queue(struct bce_queue_cmdq *cmdq,
+ struct apple_bce_softc *sc, int qid)
+{
+ struct bce_queue_cmdq_result res;
+ struct bce_cmdq_simple_cmd *cmd;
+ uint32_t slot;
+ int error;
+
+ sema_init(&res.cmpl, 0, "bce_cmd");
+
+ if (bce_reserve_submission(cmdq->sq) != 0) {
+ sema_destroy(&res.cmpl);
+ return (EAGAIN);
+ }
+
+ mtx_lock(&cmdq->lck);
+ slot = cmdq->sq->tail;
+ cmdq->tres[slot] = &res;
+ cmd = bce_next_submission(cmdq->sq);
+
+ memset(cmd, 0, BCE_CMD_SIZE);
+ cmd->cmd = BCE_CMD_FLUSH_QUEUE;
+ cmd->qid = (uint16_t)qid;
+
+ bce_submit_to_device(sc, cmdq->sq);
+ mtx_unlock(&cmdq->lck);
+
+ error = bce_cmdq_wait(cmdq, slot, &res);
+ sema_destroy(&res.cmpl);
+ if (error != 0)
+ return (error);
+
+ return (res.status);
+}
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -43,6 +43,7 @@
${_allwinner} \
${_armv8crypto} \
${_armv8_rng} \
+ ${_apple_bce} \
${_asmc} \
ata \
${_aq} \
@@ -863,6 +864,7 @@
.if ${MACHINE_CPUARCH} == "amd64"
_amdgpio= amdgpio
+_apple_bce= apple_bce
_asmc= asmc
_ccp= ccp
_enic= enic
diff --git a/sys/modules/apple_bce/Makefile b/sys/modules/apple_bce/Makefile
new file mode 100644
--- /dev/null
+++ b/sys/modules/apple_bce/Makefile
@@ -0,0 +1,7 @@
+.PATH: ${SRCTOP}/sys/dev/apple_bce
+
+KMOD= apple_bce
+SRCS= apple_bce.c apple_bce_mailbox.c apple_bce_queue.c
+SRCS+= bus_if.h device_if.h pci_if.h
+
+.include <bsd.kmod.mk>

File Metadata

Mime Type
text/plain
Expires
Sun, Aug 2, 6:05 AM (8 h, 26 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35847949
Default Alt Text
D57088.id179754.diff (42 KB)

Event Timeline