diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -53,6 +53,7 @@ riscv/riscv/nexus.c standard riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/plic.c standard +riscv/riscv/aplic.c standard riscv/riscv/pmap.c standard riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons diff --git a/sys/riscv/riscv/aplic.c b/sys/riscv/riscv/aplic.c new file mode 100644 --- /dev/null +++ b/sys/riscv/riscv/aplic.c @@ -0,0 +1,484 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Himanshu Chauhan + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +#define APLIC_MAX_IRQS 1023 + +/* Smaller priority number means higher priority */ +#define APLIC_INTR_DEF_PRIO (1) + +static pic_disable_intr_t aplic_disable_intr; +static pic_enable_intr_t aplic_enable_intr; +static pic_map_intr_t aplic_map_intr; +static pic_setup_intr_t aplic_setup_intr; +static pic_post_ithread_t aplic_post_ithread; +static pic_pre_ithread_t aplic_pre_ithread; +static pic_bind_intr_t aplic_bind_intr; + +#define DOMAIN_CFG_OFFS (0x0000) +#define SOURCECFG_OFFS (0x0004) +#define MMSIADDRCFG_OFFS (0x1BC0) +#define MMSIADDRCFGH_OFFS (0x1BC4) +#define SMSIADDRCFG_OFFS (0x1BC8) +#define SMSIADDRCFGH_OFFS (0x1BCC) +#define SETIP_OFFS (0x1C00) +#define SETIPNUM_OFFS (0x1CDC) +#define IN_CLRIP_OFFS (0x1D00) +#define CLRIPNUM_OFFS (0x1DDC) +#define SETIE_OFFS (0x1E00) +#define SETIENUM_OFFS (0x1EDC) +#define CLRIE_OFFS (0x1F00) +#define CLRIENUM_OFFS (0x1FDC) +#define SETIPNUM_LE_OFFS (0x2000) +#define SETIPNUM_BE_OFFS (0x2004) +#define GENMSI_OFFS (0x3000) +#define TARGET_OFFS (0x3004) +#define APLIC_IDC_OFFS (0x4000) + +struct aplic_idc { + uint32_t idelivery; + uint32_t iforce; + uint32_t ithreshold; + uint8_t res[12]; + uint32_t topi; + uint32_t claimi; +} __packed; + +struct aplic_irqsrc { + struct intr_irqsrc isrc; + u_int irq; +}; + +struct aplic_target { + uint32_t hart_id; + uint32_t cpu; +}; + +struct aplic_softc { + device_t dev; + struct resource *intc_res; + struct aplic_irqsrc isrcs[APLIC_MAX_IRQS]; + int ndev; +}; + +#define APLIC_DOMAIN_CFG_IE (1UL << 8) +#define APLIC_DOMAIN_CFG_DM (1UL << 2) +#define APLIC_DOMAIN_CFG_BE (1UL << 0) + +#define APLIC_MODE_DIRECT 0 +#define APLIC_MODE_MSI 1 + +#define APLIC_SRC_CFG_DLGT (1UL << 10) /* Source delegation */ +#define APLIC_SRC_CFG_SM_SHIFT 0 +#define APLIC_SRC_CFG_SM_MASK ((0x7UL) << APLIC_SRC_CFG_SM_SHIFT) + +#define APLIC_SRC_CFG_SM_INACTIVE 0 +#define APLIC_SRC_CFG_SM_DETACHED 1 +#define APLIC_SRC_CFG_SM_EDGE_RSE 4 /* Rising edge */ +#define APLIC_SRC_CFG_SM_EDGE_FLL 5 /* Falling edge */ +#define APLIC_SRC_CFG_SM_LVL_HI 6 /* Asserted when high */ +#define APLIC_SRC_CFG_SM_LVL_LO 7 /* Asserted when low */ + +#define APLIC_DOMAIN_CFG (DOMAIN_CFG_OFFS) +#define APLIC_SRC_CFG(_idx) (SOURCECFG_OFFS + ((_idx) * 4)) +#define APLIC_SETIP(_idx) (SETIP_OFFS + ((_idx) * 4)) +#define APLIC_IN_CLRIP(_idx) (CLRIP_OFFS + ((_idx) * 4)) +#define APLIC_TARGET(_idx) (TARGET_OFFS + ((_idx) * 4)) +#define APLIC_CLRIENUM (CLRIENUM_OFFS) +#define APLIC_SETIENUM (SETIENUM_OFFS) + +#define APLIC_CLAIMI_PRIO_MASK (0xff) +#define APLIC_CLAIMI_IRQ_SHIFT (16) +#define APLIC_CLAIMI_IRQ_MASK (0x3ff) + +#define APLIC_CLAIMI_IRQ(_claimi) \ + (((_claimi) >> APLIC_CLAIMI_IRQ_SHIFT) \ + & APLIC_CLAIMI_IRQ_MASK) + +#define APLIC_CLAIMI_PRIO(_claimi) ((_claimi) & APLIC_CLAIMI_PRIO_MASK) + +#define APLIC_IDC_DELIVERY(_hart_id) \ + (APLIC_IDC_OFFS + (_hart_id * sizeof(struct aplic_idc) \ + + offsetof(struct aplic_idc, idelivery))) + +#define APLIC_IDC_FORCE(_hart_id) \ + (APLIC_IDC_OFFS + (_hart_id * sizeof(struct aplic_idc) \ + + offsetof(struct aplic_idc, iforce))) + +#define APLIC_IDC_THRESHOLD(_hart_id) \ + (APLIC_IDC_OFFS + (_hart_id * sizeof(struct aplic_idc) \ + + offsetof(struct aplic_idc, ithreshold))) + +#define APLIC_IDC_TOPI(_hart_id) \ + (APLIC_IDC_OFFS + (_hart_id * sizeof(struct aplic_idc) \ + + offsetof(struct aplic_idc, topi))) + +#define APLIC_IDC_CLAIMI(_hart_id) \ + (APLIC_IDC_OFFS + (_hart_id * sizeof(struct aplic_idc) \ + + offsetof(struct aplic_idc, claimi))) + +#define APLIC_MK_IRQ_TARGET(_cpu, _prio) (_cpu << 18 | (_prio & 0xff)) + +#define aplic_read(sc, reg) \ + bus_read_4(sc->intc_res, (reg)) +#define aplic_write(sc, reg, val) \ + bus_write_4(sc->intc_res, (reg), (val)) + +static u_int aplic_irq_cpu; + +static int +riscv_hartid_to_cpu(int hartid) +{ + int i; + + CPU_FOREACH(i) { + if (pcpu_find(i)->pc_hart == hartid) + return (i); + } + + return (-1); +} + +static inline void +aplic_irq_dispatch(struct aplic_softc *sc, u_int irq, u_int prio, + struct trapframe *tf) +{ + struct aplic_irqsrc *src; + + src = &sc->isrcs[irq]; + + if (intr_isrc_dispatch(&src->isrc, tf) != 0) + device_printf(sc->dev, "Stray irq %u detected\n", irq); +} + +static int +aplic_intr(void *arg) +{ + struct aplic_softc *sc; + struct trapframe *tf; + uint32_t pending, prio; + uint32_t cpu; + + sc = arg; + cpu = PCPU_GET(cpuid); + cpu = pcpu_find(cpu)->pc_hart; + + /* Claim any pending interrupt. */ + pending = aplic_read(sc, APLIC_IDC_CLAIMI(cpu)); + prio = APLIC_CLAIMI_PRIO(pending); + pending = APLIC_CLAIMI_IRQ(pending); + + if (pending) { + tf = curthread->td_intr_frame; + aplic_irq_dispatch(sc, pending, prio, tf); + } else { + return (FILTER_STRAY); + } + + return (FILTER_HANDLED); +} + +static void +aplic_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct aplic_softc *sc; + struct aplic_irqsrc *src; + + sc = device_get_softc(dev); + src = (struct aplic_irqsrc *)isrc; + + /* Disable the interrupt source */ + aplic_write(sc, APLIC_CLRIENUM, src->irq); +} + +static void +aplic_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct aplic_softc *sc; + struct aplic_irqsrc *src; + + sc = device_get_softc(dev); + src = (struct aplic_irqsrc *)isrc; + + /* Enable the interrupt source */ + aplic_write(sc, APLIC_SETIENUM, src->irq); +} + +static int +aplic_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct intr_map_data_fdt *daf; + struct aplic_softc *sc; + + sc = device_get_softc(dev); + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + + if (daf->ncells != 2 || daf->cells[0] > sc->ndev) { + device_printf(dev, "Invalid cell data\n"); + return (EINVAL); + } + + *isrcp = &sc->isrcs[daf->cells[0]].isrc; + + return (0); +} + +static int +aplic_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "riscv,aplic")) + return (ENXIO); + + /* This aplic domain doesn't belong to S-mode */ + if (ofw_bus_has_prop(dev, "riscv,delegate")) + return (ENXIO); + + /* APLIC with IMSIC on hart is not supported */ + if (ofw_bus_has_prop(dev, "msi-parent")) + return (ENXIO); + + device_set_desc(dev, "aplic-direct"); + + return (BUS_PROBE_DEFAULT); +} + +/* + * Setup APLIC in direct mode. + */ +static int +aplic_setup_direct_mode(device_t dev) +{ + struct aplic_irqsrc *isrcs; + struct aplic_softc *sc; + struct intr_pic *pic; + uint32_t irq; + uint32_t cpu; + const char *name; + phandle_t node; + phandle_t xref; + int error; + int rid; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + sc->dev = dev; + + if ((OF_getencprop(node, "riscv,num-sources", &sc->ndev, + sizeof(sc->ndev))) < 0) { + device_printf(dev, "Error: could not get number of devices\n"); + return (ENXIO); + } + + if (sc->ndev > APLIC_MAX_IRQS) { + device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); + return (ENXIO); + } + + /* Request memory resources */ + rid = 0; + sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->intc_res == NULL) { + device_printf(dev, + "Error: could not allocate memory resources\n"); + return (ENXIO); + } + + /* Set APLIC in direct mode and enable all interrupts */ + aplic_write(sc, APLIC_DOMAIN_CFG, + (APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE)); + + /* Register the interrupt sources */ + isrcs = sc->isrcs; + name = device_get_nameunit(sc->dev); + for (irq = 1; irq <= sc->ndev; irq++) { + isrcs[irq].irq = irq; + error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, + 0, "%s,%u", name, irq); + if (error != 0) + goto fail; + + aplic_write(sc, APLIC_SRC_CFG(irq-1), + APLIC_SRC_CFG_SM_DETACHED); + } + + CPU_FOREACH(cpu) { + aplic_write(sc, APLIC_IDC_DELIVERY(cpu), 0); + aplic_write(sc, APLIC_IDC_THRESHOLD(cpu), 0); + } + + xref = OF_xref_from_node(node); + pic = intr_pic_register(sc->dev, xref); + if (pic == NULL) { + error = ENXIO; + goto fail; + } + + intr_pic_claim_root(sc->dev, xref, aplic_intr, sc, 0); + + csr_set(sie, SIE_SEIE); + + device_printf(dev, "APLIC configured in direct mode.\n"); + + return (0); + + fail: + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->intc_res); + sc->intc_res = NULL; + + return (error); +} + +static int +aplic_attach(device_t dev) +{ + int rc; + + rc = aplic_setup_direct_mode(dev); + + return (rc); +} + +static void +aplic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + + aplic_disable_intr(dev, isrc); +} + +static void +aplic_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + + aplic_enable_intr(dev, isrc); +} + +static int +aplic_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + struct aplic_irqsrc *src; + struct aplic_softc *sc; + + CPU_ZERO(&isrc->isrc_cpu); + + sc = device_get_softc(dev); + src = (struct aplic_irqsrc *)isrc; + + aplic_write(sc, APLIC_SRC_CFG(src->irq-1), + APLIC_SRC_CFG_SM_EDGE_RSE); + + return (0); +} + +static int +aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct aplic_softc *sc; + struct aplic_irqsrc *src; + u_int cpu; + + sc = device_get_softc(dev); + src = (struct aplic_irqsrc *)isrc; + + /* Disable the interrupt source */ + aplic_write(sc, APLIC_CLRIENUM, src->irq); + + if (CPU_EMPTY(&isrc->isrc_cpu)) { + cpu = aplic_irq_cpu = intr_irq_next_cpu(aplic_irq_cpu, + &all_cpus); + CPU_SETOF(cpu, &isrc->isrc_cpu); + } else { + cpu = CPU_FFS(&isrc->isrc_cpu) - 1; + } + + device_printf(dev, "Binding IRQ %d to CPU %d\n", src->irq, cpu); + aplic_write(sc, APLIC_TARGET(src->irq-1), + APLIC_MK_IRQ_TARGET(cpu, APLIC_INTR_DEF_PRIO)); + aplic_write(sc, APLIC_IDC_DELIVERY(cpu), 1); + aplic_enable_intr(dev, isrc); + + return (0); +} + +static device_method_t aplic_methods[] = { + DEVMETHOD(device_probe, aplic_probe), + DEVMETHOD(device_attach, aplic_attach), + + DEVMETHOD(pic_disable_intr, aplic_disable_intr), + DEVMETHOD(pic_enable_intr, aplic_enable_intr), + DEVMETHOD(pic_map_intr, aplic_map_intr), + DEVMETHOD(pic_pre_ithread, aplic_pre_ithread), + DEVMETHOD(pic_post_ithread, aplic_post_ithread), + DEVMETHOD(pic_post_filter, aplic_post_ithread), + DEVMETHOD(pic_setup_intr, aplic_setup_intr), + DEVMETHOD(pic_bind_intr, aplic_bind_intr), + + DEVMETHOD_END +}; + +static driver_t aplic_driver = { + "aplic", + aplic_methods, + sizeof(struct aplic_softc), +}; + +EARLY_DRIVER_MODULE(aplic, simplebus, aplic_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);