diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -49,6 +49,7 @@ riscv/riscv/exec_machdep.c standard riscv/riscv/fpe.c standard riscv/riscv/gdb_machdep.c optional gdb +riscv/riscv/imsic.c standard riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h --- a/sys/riscv/include/cpufunc.h +++ b/sys/riscv/include/cpufunc.h @@ -48,6 +48,16 @@ #include +#define readb(va) (*(volatile uint8_t *) (va)) +#define readw(va) (*(volatile uint16_t *) (va)) +#define readl(va) (*(volatile uint32_t *) (va)) +#define readq(va) (*(volatile uint64_t *) (va)) + +#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) +#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) +#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) +#define writeq(va, d) (*(volatile uint64_t *) (va) = (d)) + static __inline register_t intr_disable(void) { diff --git a/sys/riscv/include/imsic.h b/sys/riscv/include/imsic.h new file mode 100644 --- /dev/null +++ b/sys/riscv/include/imsic.h @@ -0,0 +1,32 @@ +/*- + * Copyright (c) 2024 Himanshu Chauhan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _IMSIC_H_ +#define _IMSIC_H_ + +int imsic_alloc_aplic_msi(device_t dev, int cpu, int irq, int *msi); +int imsic_enable_irq(device_t dev, int cpu, int irq); + +#endif diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h --- a/sys/riscv/include/intr.h +++ b/sys/riscv/include/intr.h @@ -57,6 +57,6 @@ #endif /* !LOCORE */ #define INTR_ROOT_IRQ 0 -#define INTR_ROOT_COUNT 1 +#define INTR_ROOT_COUNT 2 #endif /* !_MACHINE_INTR_MACHDEP_H_ */ diff --git a/sys/riscv/riscv/aplic.c b/sys/riscv/riscv/aplic.c --- a/sys/riscv/riscv/aplic.c +++ b/sys/riscv/riscv/aplic.c @@ -38,6 +38,10 @@ #include #include #include +#include + +#include +#include #include #include @@ -61,6 +65,7 @@ struct aplic_irqsrc { struct intr_irqsrc isrc; u_int irq; + u_int msi; }; struct aplic_softc { @@ -70,7 +75,10 @@ void *ih; struct aplic_irqsrc isrcs[APLIC_MAX_IRQS + 1]; unsigned int hart_indices[MAXCPU]; + device_t parent; + struct intr_pic *pic; int ndev; + int has_imsic; }; #define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */ @@ -180,6 +188,32 @@ return (-1); } +static int +fdt_get_parent_imsic(device_t dev) +{ + struct aplic_softc *sc; + phandle_t parent_xref, node; + int rc; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + rc = OF_getencprop(node, "msi-parent", &parent_xref, + sizeof(parent_xref)); + if (rc <= 0) { + device_printf(dev, "can't read parent node property\n"); + return (ENXIO); + } + + sc->parent = OF_device_from_xref(parent_xref); + if (sc->parent == NULL) { + device_printf(dev, "can't find parent controller\n"); + return (ENXIO); + } + + return 0; +} + static int fdt_get_hartid(device_t dev, phandle_t aplic) { @@ -219,6 +253,19 @@ device_printf(sc->dev, "Stray irq %u detected\n", irq); } +static int +aplic_imsic_intr(void *arg, unsigned long irq) +{ + struct aplic_softc *sc; + struct trapframe *tf; + + sc = arg; + tf = curthread->td_intr_frame; + aplic_irq_dispatch(sc, irq, 0, tf); + + return (FILTER_HANDLED); +} + static int aplic_intr(void *arg) { @@ -308,67 +355,54 @@ return (BUS_PROBE_DEFAULT); } -/* - * Setup APLIC in direct mode. - */ static int -aplic_setup_direct_mode(device_t dev) +aplic_setup_imsic_mode(device_t dev) { - struct aplic_irqsrc *isrcs; struct aplic_softc *sc; - struct intr_pic *pic; - const char *name; - phandle_t node, xref, iparent; - pcell_t *cells, cell; int error = ENXIO; - u_int irq; - int cpu, hartid, rid, i, nintr, idc; - device_t rootdev; sc = device_get_softc(dev); - node = ofw_bus_get_node(dev); - sc->dev = dev; + device_printf(dev, "Configuring APLIC with IMSIC.\n"); - if ((OF_getencprop(node, "riscv,num-sources", &sc->ndev, - sizeof(sc->ndev))) < 0) { - device_printf(dev, "Error: could not get number of devices\n"); - return (error); - } + aplic_write(sc, APLIC_DOMAIN_CFG, APLIC_MODE_MSI | APLIC_DOMAIN_CFG_IE); - if (sc->ndev > APLIC_MAX_IRQS) { - device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); - return (error); - } + if ((error = fdt_get_parent_imsic(dev)) != 0) + goto fail; - /* Request memory resources */ - rid = 0; - sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, - RF_ACTIVE); - if (sc->mem_res == NULL) { - device_printf(dev, - "Error: could not allocate memory resources\n"); - return (error); + error = intr_pic_add_handler(sc->parent, sc->pic, + aplic_imsic_intr, sc, 1, APLIC_MAX_IRQS); + + if (error != 0) { + device_printf(dev, "Failed to install child handler\n"); + goto fail; } - /* Set APLIC in direct mode and enable all interrupts */ - aplic_write(sc, APLIC_DOMAIN_CFG, - APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE); + return (0); - /* Register the interrupt sources */ - isrcs = sc->isrcs; - name = device_get_nameunit(sc->dev); - for (irq = 1; irq <= sc->ndev; irq++) { - isrcs[irq].irq = irq; + fail: + return (error); +} - error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, - 0, "%s,%u", name, irq); - if (error != 0) - goto fail; +/* + * Setup APLIC in direct mode. + */ +static int +aplic_setup_direct_mode(device_t dev) +{ + struct aplic_softc *sc; + phandle_t node; + pcell_t *cells; + int error = ENXIO; + int cpu, hartid, i, nintr, idc; - aplic_write(sc, APLIC_SRC_CFG(irq), - APLIC_SRC_CFG_SM_DETACHED); - } + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + device_printf(dev, "Configuring APLIC in Direct mode.\n"); + /* Set APLIC in direct mode and enable all interrupts */ + aplic_write(sc, APLIC_DOMAIN_CFG, + APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE); nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(uint32_t), (void **)&cells); @@ -400,7 +434,6 @@ sc->hart_indices[cpu] = idc; } - OF_prop_free(cells); /* Turn off the interrupt delivery for all CPUs within or out domain */ CPU_FOREACH(cpu) { @@ -410,6 +443,71 @@ APLIC_IDC_ITHRESHOLD_DISABLE); } + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + aplic_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); + return (ENXIO); + } + +fail: + return (error); +} + +static int +aplic_setup(device_t dev) +{ + struct aplic_irqsrc *isrcs; + struct aplic_softc *sc; + const char *name; + phandle_t node, xref, iparent; + pcell_t cell; + int error = ENXIO; + u_int irq; + int rid; + device_t rootdev; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + sc->dev = dev; + + if ((OF_getencprop(node, "riscv,num-sources", &sc->ndev, + sizeof(sc->ndev))) < 0) { + device_printf(dev, "Error: could not get number of devices\n"); + return (error); + } + + if (sc->ndev > APLIC_MAX_IRQS) { + device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); + return (error); + } + + /* Request memory resources */ + rid = 0; + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->mem_res == NULL) { + device_printf(dev, + "Error: could not allocate memory resources\n"); + return (error); + } + + /* Register the interrupt sources */ + isrcs = sc->isrcs; + name = device_get_nameunit(sc->dev); + for (irq = 1; irq <= sc->ndev; irq++) { + isrcs[irq].irq = irq; + + error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, + 0, "%s,%u", name, irq); + if (error != 0) + goto fail; + + aplic_write(sc, APLIC_SRC_CFG(irq), + APLIC_SRC_CFG_SM_DETACHED); + } + rootdev = intr_irq_root_device(INTR_ROOT_IRQ); iparent = OF_xref_from_node(ofw_bus_get_node(rootdev)); cell = IRQ_EXTERNAL_SUPERVISOR; @@ -429,20 +527,22 @@ return (ENXIO); } + if (sc->has_imsic) { + error = aplic_setup_imsic_mode(dev); + if (error) + goto fail; + } else { + error = aplic_setup_direct_mode(dev); + if (error) + goto fail; + } + xref = OF_xref_from_node(node); - pic = intr_pic_register(sc->dev, xref); - if (pic == NULL) { + sc->pic = intr_pic_register(sc->dev, xref); + if (sc->pic == NULL) { error = ENXIO; goto fail; } - - error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, - aplic_intr, NULL, sc, &sc->ih); - if (error != 0) { - device_printf(dev, "Unable to setup IRQ resource\n"); - return (ENXIO); - } - fail: return (error); } @@ -450,15 +550,18 @@ static int aplic_attach(device_t dev) { + struct aplic_softc *sc; int rc; + sc = device_get_softc(dev); + /* APLIC with IMSIC on hart is not supported */ if (ofw_bus_has_prop(dev, "msi-parent")) { - device_printf(dev, "APLIC with IMSIC is unsupported\n"); - return (ENXIO); - } + sc->has_imsic = 1; + } else + sc->has_imsic = 0; - rc = aplic_setup_direct_mode(dev); + rc = aplic_setup(dev); return (rc); } @@ -527,10 +630,23 @@ device_printf(dev, "Bind irq %d to cpu%d (hart %d)\n", src->irq, cpu, hartid); - aplic_write(sc, APLIC_TARGET(src->irq), - APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO)); - aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), - APLIC_IDC_IDELIVERY_ENABLE); + if (!sc->has_imsic) { + aplic_write(sc, APLIC_TARGET(src->irq), + APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO)); + aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), + APLIC_IDC_IDELIVERY_ENABLE); + } else { + if (imsic_alloc_aplic_msi(sc->parent, cpu, src->irq, + &src->msi)) { + device_printf(dev, "MSI allocation for irq %d failed\n", + src->irq); + return (ENXIO); + } + aplic_write(sc, APLIC_TARGET(src->irq), + (((hartid << 18) | (src->msi & 0x7ff)) & ~(0x3f000ul))); + imsic_enable_irq(sc->parent, cpu, src->msi); + } + aplic_enable_intr(dev, isrc); return (0); diff --git a/sys/riscv/riscv/imsic.c b/sys/riscv/riscv/imsic.c new file mode 100644 --- /dev/null +++ b/sys/riscv/riscv/imsic.c @@ -0,0 +1,928 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Himanshu Chauhan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +static pic_disable_intr_t imsic_disable_intr; +static pic_enable_intr_t imsic_enable_intr; +static pic_map_intr_t imsic_map_intr; +static pic_setup_intr_t imsic_setup_intr; +static pic_post_ithread_t imsic_post_ithread; +static pic_pre_ithread_t imsic_pre_ithread; +static pic_bind_intr_t imsic_bind_intr; +#ifdef SMP +static pic_init_secondary_t imsic_init_secondary; +#endif + +#define IMSIC_MAX_MSI 2047 /* Maximum MSI number in IMSIC */ +#define IMSIC_MSI_PPN_SHIFT 12 /* Number of bits to shift for PPN */ +#define IMSIC_PAGE_SHIFT 12 +#define IMSIC_PAGE_SIZE (1UL << 12) /* Size of interrupt file region */ + +#define INVALID_IRQ 0 /* 0 is invalid IRQ in IMSIC */ + +#define IMSIC_CSR_SISELECT 0x150 +#define IMSIC_CSR_SIREG 0x151 +#define IMSIC_CSR_STOPEI 0x15c +#define IMSIC_CSR_STOPI 0xdb0 + +#ifdef DECLARE_CSR +DECLARE_CSR(siselect, IMSIC_SISELCT) +DECLARE_CSR(sireg, IMSIC_SIREG) +DECLARE_CSR(stopei, IMSIC_STOPEI) +DECLARE_CSR(stopi, IMSIC_STOPI) +#endif + +#define IMSIC_EIDELIVERY 0x70 +#define IMSIC_EITHRESHOLD 0x72 + +#define IMSIC_EIP_BASE 0x80 +#define IMSIC_EIE_BASE 0xc0 + +#if __riscv_xlen == 32 +#define IMSIC_INTR_TO_EIP_REG(_n) IMSIC_EIP_BASE + (_n / 32) +#define IMSIC_INTR_TO_EIE_REG(_n) IMSIC_EIE_BASE + (_n / 32) +#elif __riscv_xlen == 64 +/* AIA spec (3.8.3), odd registers are invalid if xlen is 64 */ +#define IMSIC_INTR_TO_EIP_REG(_n) IMSIC_EIP_BASE + ((_n / 64) * 2 +#define IMSIC_INTR_TO_EIE_REG(_n) IMSIC_EIE_BASE + ((_n / 64) * 2) +#else +#error "Undefined __riscv_xlen" +#endif + +#define IMSIC_INTR_DISABLED 0x0 +#define IMSIC_INTR_FILE_EN 0x1 +#define IMSIC_INTR_APLIC_EN 0x40000000UL +#define IMSIC_ITHRESHOLD_DISABLE 0 + +/* + * MSI to be used as IPI. Lowest MSI has highest priority. + * Using one to give IPI the highest priority. + */ +#define IMSIC_IPI_MSI 1 + +#define IMSIC_FILL(p) BIT_FILL(IMSIC_MAX_MSI, p) +#define IMSIC_ISEMPTY(p) BIT_EMPTY(IMSIC_MAX_MSI, p) +#define IMSIC_FFS(p) BIT_FFS(IMSIC_MAX_MSI, p) +#define IMSIC_SETMSI(n, p) BIT_SET(IMSIC_MAX_MSI, n, p) +#define IMSIC_CLRMSI(n, p) BIT_CLR(IMSIC_MAX_MSI, n, p) + +#define IMSIC_ALLOCED_MSI(_sc, _cpu) \ + &_sc->pcpu_data[_cpu].allocated_msis + +#define IMSIC_IRQ(_sc, _cpu, _irq) \ +((struct imsic_irq *)(&_sc->pcpu_data[_cpu].isrcs[_irq])) + +#define IMSIC_MSI_ISRC(_sc, _cpu, _irq) \ +(&_sc->pcpu_data[_cpu].isrcs[_irq].isrc) + +#define IMSIC_ISRC_TO_IRQ(_isrc) \ +((struct imsic_irq *)_isrc) + +#define IMSIC_ACCESS_LOCK(_sc, _cpu) \ + mtx_lock_spin(&_sc->pcpu_data[_cpu].access_lock) + +#define IMSIC_ACCESS_UNLOCK(_sc, _cpu) \ + mtx_unlock_spin(&_sc->pcpu_data[_cpu].access_lock) + +struct imsic_msi_config { + unsigned long lhxs; /* Guest index shift */ + unsigned long lhxw; /* Hart index width */ + unsigned long hhxs; /* Group index shift */ + unsigned long hhxw; /* Group index width */ + unsigned long msi_base; /* Base address for PPN */ + unsigned long msi_sz; /* Total MSI region size for all harts */ +}; + +__BITSET_DEFINE(_msiset, IMSIC_MAX_MSI); +typedef struct _msiset msiset_t; + +struct imsic_irq { + struct intr_irqsrc isrc; + u_int msi; + u_int aplic_irq; +}; + +struct imsic_pcpu { + struct mtx access_lock; + msiset_t allocated_msis; + struct imsic_irq isrcs[IMSIC_MAX_MSI]; + uint32_t ipi_init_done; +}; + +struct imsic_softc { + device_t dev; + struct intr_pic *pic; + struct resource *mem_res; + struct resource *irq_res; + struct imsic_pcpu pcpu_data[MAXCPU]; + struct imsic_msi_config msi_config; + int interrupts_extended; + int ndev; + cpuset_t target_cpus; + struct intr_irqsrc ipi_isrc; + uint32_t pending_ipis[MAXCPU]; +}; + +static uint64_t *v_msi_addr[MAXCPU] = { 0 }; +static int imsic_present; +static int imsic_ipi_enabled; + +static inline void +imsic_csr_write(u_int reg, uint64_t val) +{ + csr_write(siselect, reg); + csr_write(sireg, val); +} + +static inline uint64_t +imsic_csr_read(u_int reg) +{ + uint64_t _v; + + csr_write(siselect, reg); + _v = csr_read(sireg); + + return (_v); +} + +static inline int +riscv_cpu_to_hartid(int cpu) +{ + return pcpu_find(cpu)->pc_hart; +} + +static inline int +riscv_hartid_to_cpu(int hartid) +{ + int cpu; + + CPU_FOREACH(cpu) { + if (riscv_cpu_to_hartid(cpu) == hartid) + return cpu; + } + + return (-1); +} + +static int +fdt_get_hartid(device_t dev, phandle_t imsic) +{ + int hartid; + + /* Check the interrupt controller layout. */ + if (OF_searchencprop(imsic, "#interrupt-cells", &hartid, + sizeof(hartid)) == -1) { + device_printf(dev, + "Could not find #interrupt-cells for phandle %u\n", imsic); + return (-1); + } + + /* + * The parent of the interrupt-controller is the CPU we are + * interested in, so search for its hart ID. + */ + if (OF_searchencprop(OF_parent(imsic), "reg", (pcell_t *)&hartid, + sizeof(hartid)) == -1) { + device_printf(dev, "Could not find hartid\n"); + return (-1); + } + + return (hartid); +} + +/* + * Clear the current pending interrupt and return the + * interrupt that was cleared. + */ +static inline u_long +imsic_get_pending_interrupt(void) +{ + u_long c_intr = 0; + + /* Value written is ignored. */ + c_intr = csr_read(stopei); + csr_write(stopei, ~c_intr); + + return c_intr; +} + +static inline void +imsic_enable_interrupt(u_int interrupt) +{ + uint64_t eie_v; + + eie_v = imsic_csr_read(IMSIC_INTR_TO_EIE_REG(interrupt)); + eie_v |= (0x1UL << (interrupt % 64)); + + imsic_csr_write(IMSIC_INTR_TO_EIE_REG(interrupt), eie_v); +} + +static inline void +imsic_disable_interrupt(u_int interrupt) +{ + uint64_t eie_v; + + eie_v = imsic_csr_read(IMSIC_INTR_TO_EIE_REG(interrupt)); + eie_v &= ~(0x1UL << (interrupt % 64)); + + imsic_csr_write(IMSIC_INTR_TO_EIE_REG(interrupt), eie_v); +} + +#ifdef SMP +static int +imsic_handle_ipi(struct imsic_softc *sc) +{ + uint32_t ipi_bitmap; + u_int cpu, ipi; + int bit; + + cpu = PCPU_GET(cpuid); + + mb(); + + ipi_bitmap = atomic_readandclear_32(&sc->pending_ipis[cpu]); + if (ipi_bitmap == 0) + return (FILTER_HANDLED); + + mb(); + + while ((bit = ffs(ipi_bitmap))) { + ipi = (bit - 1); + ipi_bitmap &= ~(1u << ipi); + + intr_ipi_dispatch(ipi); + } + + return (FILTER_HANDLED); +} +#endif + +static int +imsic_intr(void *arg) +{ + struct imsic_softc *sc; + u_long msi, aplic_irq; + struct trapframe *tf; + int cpu = PCPU_GET(cpuid); + + sc = arg; + + msi = imsic_get_pending_interrupt(); + msi >>= 16; + + if (!msi) + return (FILTER_STRAY); + +#ifdef SMP + if (msi == IMSIC_IPI_MSI) + return imsic_handle_ipi(sc); +#endif + + aplic_irq = IMSIC_IRQ(sc, cpu, msi)->aplic_irq; + if (aplic_irq == INVALID_IRQ) { + tf = curthread->td_intr_frame; + if (intr_isrc_dispatch(IMSIC_MSI_ISRC(sc, cpu, msi), tf) != 0) { + device_printf(sc->dev, "Stray irq %lu detected\n", msi); + return (FILTER_STRAY); + } + } else { + intr_child_irq_handler(sc->pic, aplic_irq); + } + + return (FILTER_HANDLED); +} + +static void +imsic_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct imsic_irq *src; + + src = IMSIC_ISRC_TO_IRQ(isrc); + + /* Disable the interrupt source */ + imsic_disable_interrupt(src->msi); +} + +static void +imsic_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct imsic_irq *src; + + src = IMSIC_ISRC_TO_IRQ(isrc); + + /* Enable the interrupt source */ + imsic_enable_interrupt(src->msi); +} + +static int +imsic_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + u_int cpu; + struct intr_map_data_fdt *daf; + struct imsic_softc *sc; + + cpu = PCPU_GET(cpuid); + + sc = device_get_softc(dev); + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + if (daf->ncells != 2 || daf->cells[0] > sc->ndev) { + device_printf(dev, "Invalid cell data\n"); + return (EINVAL); + } + + *isrcp = &sc->pcpu_data[cpu].isrcs[daf->cells[0]].isrc; + + return (0); +} + +#if 0 +static void imsic_enable_ipi_irq(void) +{ + imsic_enable_interrupt(IMSIC_IPI_MSI); +} +#endif + +static void imsic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + uint64_t *msi_vaddr; + struct imsic_softc *sc; + struct pcpu *pc; + u_int cpu; + + sc = device_get_softc(dev); + + KASSERT(isrc == &sc->ipi_isrc, ("%s: not the IPI isrc", __func__)); + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { + cpu = pc->pc_cpuid; + if (CPU_ISSET(cpu, &cpus)) + atomic_set_32(&sc->pending_ipis[cpu], 1u << ipi); + } + + CPU_FOREACH(cpu) { + if (CPU_ISSET(cpu, &cpus)) { + msi_vaddr = v_msi_addr[cpu]; + if (msi_vaddr) + /* Write MSI interrupt number */ + writel(msi_vaddr, IMSIC_IPI_MSI); + } + } +} + +static int +imsic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct imsic_softc *sc; + + sc = device_get_softc(dev); + + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u\n", __func__, ipi)); + + *isrcp = &sc->ipi_isrc; + + return (0); +} + +static int +imsic_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "riscv,imsics")) + return (ENXIO); + + if (!ofw_bus_has_prop(dev, "msi-controller")) + return (ENXIO); + + device_set_desc(dev, "Incoming MSI Controller"); + + return (BUS_PROBE_DEFAULT); +} + +#ifdef SMP +static void +imsic_init_secondary(device_t dev, uint32_t rootnum) +{ + cpuset_t cpus; + int cpu = PCPU_GET(cpuid); + int i; + + if (!imsic_present) + return; + + CPU_ZERO(&cpus); + CPU_FOREACH(i) { + if (i == cpu) + continue; + + CPU_SET(i, &cpus); + } + + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_INTR_FILE_EN); + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ITHRESHOLD_DISABLE); + imsic_enable_interrupt(IMSIC_IPI_MSI); +} +#endif + +static int +imsic_interrupts_extended(device_t dev) +{ + struct imsic_softc *sc; + phandle_t node; + pcell_t *cells; + int nintr, i, hartid, cpu; + int rc = ENXIO; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + sc->interrupts_extended = 0; + + nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", + sizeof(uint32_t), (void **)&cells); + if (nintr < 0) { + device_printf(dev, "interrupts-extended property not found.\n"); + goto _out; + } + + CPU_ZERO(&sc->target_cpus); + + for (i = 0; i < nintr; i+=2) { + if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) + continue; + + hartid = fdt_get_hartid(dev, OF_node_from_xref(cells[i])); + if (hartid < 0) { + device_printf(dev, "Target hartid not found\n"); + goto _fail; + } + + cpu = riscv_hartid_to_cpu(hartid); + if (cpu < 0) { + device_printf(dev, "Invalid cpu for hart %d\n", + hartid); + goto _fail; + } + + CPU_SET(cpu, &sc->target_cpus); + } + + sc->interrupts_extended = nintr; + + rc = 0; + + _fail: + OF_prop_free(cells); + _out: + return (rc); +} + +static int +imsic_parse_fdt_msi_config(device_t dev) +{ + struct imsic_softc *sc; + phandle_t node; + pcell_t *buf; + pcell_t reg[4]; + ssize_t len; + int rc = 0; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + if ((len = OF_getencprop_alloc(node, "riscv,guest-index-bits", + (void **)&buf)) < 0) { + sc->msi_config.lhxs = 0; + } else { + sc->msi_config.lhxs = buf[0]; + } + + OF_prop_free(buf); + + if ((len = OF_getencprop_alloc(node, "riscv,hart-index-bits", + (void *)&buf)) < 0) { + sc->msi_config.lhxw = flsl(sc->interrupts_extended); + if ((1UL << sc->msi_config.lhxw) < sc->interrupts_extended) + sc->msi_config.lhxw++; + } else + sc->msi_config.lhxw = buf[0]; + + if ((len = OF_getencprop_alloc(node, "riscv,group-index-bits", + (void *)&buf)) < 0) { + sc->msi_config.hhxw = 0; + } else + sc->msi_config.hhxw = buf[0]; + + OF_prop_free(buf); + + if ((len = OF_getencprop_alloc(node, "riscv,group-index-shift", + (void *)&buf)) < 0) { + sc->msi_config.hhxs = 2 * IMSIC_PAGE_SHIFT; + } else + sc->msi_config.hhxs = buf[0]; + + if (sc->msi_config.hhxs < (2 * IMSIC_PAGE_SHIFT)) + return (ENXIO); + sc->msi_config.hhxs -= 24; + + if ((OF_getencprop(node, "reg", ®[0], sizeof(reg))) < 0) { + device_printf(dev, "Failed to read regs property\n"); + } + + sc->msi_config.msi_base = ((uint64_t)reg[0] << 32) | reg[1]; + sc->msi_config.msi_sz = ((uint64_t)reg[2] << 32) | reg[3]; + + OF_prop_free(buf); + + return (rc); +} + +static int +imsic_configure_ipi(device_t dev) +{ + struct imsic_softc *sc; + vm_paddr_t msi_addr, ppn; + int msi_hart, msi_cpu, i; + uint64_t *v_msi; + char *env; + + env = kern_getenv("aia.imsic.ipi_disable"); + if (env) { + device_printf(dev, "IPI is disabled by configuration\n"); + imsic_ipi_enabled = 0; + return (ENXIO); + } + + sc = device_get_softc(dev); + + /* Get the MSI config needed to calculate MSI addresses for IPI */ + if (imsic_parse_fdt_msi_config(dev) < 0) { + device_printf(dev, "MSI configuration missgin\n"); + imsic_ipi_enabled = 0; + return (ENXIO); + } + + /* MSI regions in FDT are more than CPUs supported */ + if (sc->msi_config.msi_sz > ((1UL << IMSIC_PAGE_SHIFT) * MAXCPU)) { + device_printf(dev, "MSI config size is greater than expected\n"); + imsic_ipi_enabled = 0; + return (ENXIO); + } + + /* + * We are good to calculate MSI addresses for CPUs to send IPIs. + * + * msi_addr = (base ppn | (g << (hhsx + 12)) | (h << lhxs)) << 12 + * + * NOTE: We assume tha group is zero here. So we skip the middle + * part. + */ + CPU_FOREACH(msi_cpu) { + if (!CPU_ISSET(msi_cpu, &sc->target_cpus)) { + if (bootverbose) + device_printf(dev, + "CPU-%d is not set for MSI interrupts\n", + msi_cpu); + v_msi_addr[msi_cpu] = NULL; + continue; + } + + msi_hart = pcpu_find(msi_cpu)->pc_hart; + ppn = (sc->msi_config.msi_base >> IMSIC_MSI_PPN_SHIFT); + msi_addr = (ppn | (msi_hart << sc->msi_config.lhxs)) + << IMSIC_MSI_PPN_SHIFT; + + if (bootverbose) + device_printf(dev, "msi_addr_%d: 0x%lx\n", msi_cpu, + msi_addr); + + v_msi = (uint64_t *)pmap_mapdev((vm_paddr_t)msi_addr, + IMSIC_PAGE_SIZE); + if (!v_msi) { + if (bootverbose) + device_printf(dev, + "MSI address mapping failed.\n"); + + /* + * Event if a single pmap fails, we cannot do a writel + * and hence cannot deliver IPI. Disable IPIs via imsic. + */ + imsic_ipi_enabled = 0; + goto _setup_failed; + } + v_msi_addr[msi_cpu] = v_msi; + } + + /* Enable MSI IPI on this CPU */ + imsic_enable_interrupt(IMSIC_IPI_MSI); + + return (0); + + _setup_failed: + for (i = 0; i < MAXCPU; i++) { + if (v_msi_addr[i] != NULL) { + pmap_unmapdev(v_msi_addr[i], IMSIC_PAGE_SIZE); + v_msi_addr[i] = NULL; + } + } + + return (ENXIO); +} + +static int +imsic_attach(device_t dev) +{ + struct imsic_softc *sc; + phandle_t node; + phandle_t xref; + int i, j, irq, error, rid; + phandle_t iparent; + pcell_t cell; + device_t root_dev; + phandle_t root_node; + +#ifdef SMP + const char *name; +#endif + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + sc->dev = dev; + + if (imsic_interrupts_extended(dev)) { + device_printf(dev, "Interrupts not extended to supervisor.\n"); + return (ENXIO); + } + + imsic_present = 1; + + /* + * Enable interrupt delivery by interrupt file. APLIC will forward the + * interrupts to IMSIC as MSI writes. + */ + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_INTR_FILE_EN); + + /* Receive all interrupts */ + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ITHRESHOLD_DISABLE); + + for (i = 0; i < MAXCPU; i++) { + mtx_init(&sc->pcpu_data[i].access_lock, "imsic-pcpu-lock", NULL, + MTX_SPIN); + IMSIC_FILL(&sc->pcpu_data[i].allocated_msis); + + /* Clear the MSI bit which is an invalid MSI/IRQ */ + IMSIC_CLRMSI(INVALID_IRQ, &sc->pcpu_data[i].allocated_msis); + /* Mark IPI MSI as used. */ + IMSIC_CLRMSI(IMSIC_IPI_MSI, &sc->pcpu_data[i].allocated_msis); + + for (j = 0; j <= IMSIC_MAX_MSI; j++) { + sc->pcpu_data[i].isrcs[j].aplic_irq = INVALID_IRQ; + sc->pcpu_data[i].isrcs[j].msi = INVALID_IRQ; + } + } + + xref = OF_xref_from_node(node); + sc->pic = intr_pic_register(dev, xref); + if (sc->pic == NULL) { + device_printf(dev, "PIC registration unsuccessful\n"); + return (ENXIO); + } + + /* set self as slave of root intc on external supervisor IRQ */ + root_dev = intr_irq_root_device(INTR_ROOT_IRQ); + root_node = ofw_bus_get_node(root_dev); + iparent = OF_xref_from_node(root_node); + cell = IRQ_EXTERNAL_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + + /* Setup IRQs handler */ + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + imsic_intr, NULL, sc, (void **)&sc); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); + return (ENXIO); + } + +#ifdef SMP + name = device_get_nameunit(dev); + error = intr_isrc_register(&sc->ipi_isrc, sc->dev, INTR_ISRCF_IPI, + "%s,ipi", name); + if (error != 0) { + device_printf(dev, "Can't register interrupt: %d\n", error); + return (ENXIO); + } + + /* + * FIXME: Any other way to get pic_init_secondary get called from + * root PIC? + */ + intr_pic_claim_root(dev, xref, imsic_intr, sc, INTR_ROOT_IRQ+1); + + /* Setup IMSIC for IPIs */ + imsic_configure_ipi(dev); + + error = intr_ipi_pic_register(dev, 1); + if (error != 0) { + device_printf(dev, "Can't register as IPI source: %d\n", error); + return (ENXIO); + } + + device_printf(dev, "IMSIC registered as IPI device\n"); +#endif + + OF_device_register_xref(xref, dev); + + return (0); +} + +static void +imsic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); +} + +static void +imsic_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); +} + +static int +imsic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, + struct intr_map_data *data) +{ + device_printf(dev, "%s\n", __func__); + return (0); +} + +static int +imsic_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); + return (0); +} + +int +imsic_alloc_aplic_msi(device_t dev, int cpu, int irq, int *msi) +{ + struct imsic_softc *sc; + int error = 0; + + sc = device_get_softc(dev); + + IMSIC_ACCESS_LOCK(sc, cpu); + + /* If none allocated_msis == 0, no MSI left to allocate */ + if (IMSIC_ISEMPTY(&sc->pcpu_data[cpu].allocated_msis)) { + error = ENXIO; + goto _out; + } + + /* Find free MSI (first bit set) */ + *msi = IMSIC_FFS(&sc->pcpu_data[cpu].allocated_msis)-1; + + /* Clear the bit to mark as allocated */ + IMSIC_CLRMSI(*msi, &sc->pcpu_data[cpu].allocated_msis); + + /* Save MSI to APLIC IRQ to pass on to child handler */ + sc->pcpu_data[cpu].isrcs[*msi].msi = *msi; + sc->pcpu_data[cpu].isrcs[*msi].aplic_irq = irq; + + device_printf(dev, "%s: MSI-%d allocated to APLIC irq %d\n", + __func__, *msi, irq); + + error = 0; + + _out: + IMSIC_ACCESS_UNLOCK(sc, cpu); + + return (error); +} + +static void +smp_imsic_enable_interrupt(void *arg) +{ + uint64_t irq = (uint64_t)arg; + + imsic_enable_interrupt(irq); +} + +int +imsic_enable_irq(device_t dev, int cpu, int irq) +{ +#ifdef SMP + cpuset_t cpus; +#endif + if (cpu == PCPU_GET(cpuid)) { + imsic_enable_interrupt(irq); + } else { +#ifdef SMP + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + smp_rendezvous_cpus(cpus, NULL, smp_imsic_enable_interrupt, + NULL, (void *)((uint64_t)irq)); +#endif + } + + return 0; +} + +static device_method_t imsic_methods[] = { + DEVMETHOD(device_probe, imsic_probe), + DEVMETHOD(device_attach, imsic_attach), + + DEVMETHOD(pic_disable_intr, imsic_disable_intr), + DEVMETHOD(pic_enable_intr, imsic_enable_intr), + DEVMETHOD(pic_map_intr, imsic_map_intr), + DEVMETHOD(pic_pre_ithread, imsic_pre_ithread), + DEVMETHOD(pic_post_ithread, imsic_post_ithread), + DEVMETHOD(pic_post_filter, imsic_post_ithread), + DEVMETHOD(pic_setup_intr, imsic_setup_intr), + DEVMETHOD(pic_bind_intr, imsic_bind_intr), + +#ifdef SMP + DEVMETHOD(pic_ipi_send, imsic_ipi_send), + DEVMETHOD(pic_ipi_setup, imsic_ipi_setup), + + DEVMETHOD(pic_init_secondary, imsic_init_secondary), +#endif + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(imsic, imsic_driver, imsic_methods, sizeof(struct imsic_softc)); +EARLY_DRIVER_MODULE(imsic, simplebus, imsic_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY);