diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -54,6 +54,7 @@ riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/plic.c standard riscv/riscv/aplic.c standard +riscv/riscv/imsic.c standard riscv/riscv/pmap.c standard riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons diff --git a/sys/riscv/include/aplic.h b/sys/riscv/include/aplic.h new file mode 100644 --- /dev/null +++ b/sys/riscv/include/aplic.h @@ -0,0 +1,16 @@ +#ifndef _APLIC_H +#define _APLIC_H + +#include +#include + +typedef int (*imsic_aplic_reg_cb_t)(device_t parent, void *cookie); + +struct aplic_dev { + device_t aplic; + imsic_aplic_reg_cb_t callback; + TAILQ_ENTRY(aplic_dev) next; + void *cookie; +}; + +#endif diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h --- a/sys/riscv/include/cpufunc.h +++ b/sys/riscv/include/cpufunc.h @@ -46,6 +46,16 @@ #include +#define readb(va) (*(volatile uint8_t *) (va)) +#define readw(va) (*(volatile uint16_t *) (va)) +#define readl(va) (*(volatile uint32_t *) (va)) +#define readq(va) (*(volatile uint64_t *) (va)) + +#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) +#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) +#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) +#define writeq(va, d) (*(volatile uint64_t *) (va) = (d)) + static __inline register_t intr_disable(void) { diff --git a/sys/riscv/include/imsic.h b/sys/riscv/include/imsic.h new file mode 100644 --- /dev/null +++ b/sys/riscv/include/imsic.h @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2024 Himanshu Chauhan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _IMSIC_H_ +#define _IMSIC_H_ + +#include + +int imsic_register_aplic(struct aplic_dev *dev); +int imsic_alloc_msi(device_t dev, int cpu, int irq, int *msi); +int imsic_enable_irq(device_t dev, int cpu, int irq); + +#endif diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h --- a/sys/riscv/include/intr.h +++ b/sys/riscv/include/intr.h @@ -60,6 +60,20 @@ #ifdef SMP void riscv_setup_ipihandler(driver_filter_t *); void riscv_unmask_ipi(void); + +typedef void (*riscv_unmask_ipi_t)(void); +typedef void (*riscv_setup_ipi_handler_t)(driver_filter_t *); +typedef int (*riscv_send_ipi_t)(cpuset_t cpus); + +struct riscv_ipi_device { + char name[32]; + riscv_unmask_ipi_t unmask_ipi; + riscv_setup_ipi_handler_t setup_ipi_handler; + riscv_send_ipi_t send_ipi; + volatile uint32_t ready; +}; + +int riscv_register_ipi_device(struct riscv_ipi_device *dev); #endif enum { diff --git a/sys/riscv/riscv/aplic.c b/sys/riscv/riscv/aplic.c --- a/sys/riscv/riscv/aplic.c +++ b/sys/riscv/riscv/aplic.c @@ -38,6 +38,11 @@ #include #include #include +#include +#include + +#include +#include #include #include @@ -61,13 +66,18 @@ struct aplic_irqsrc { struct intr_irqsrc isrc; u_int irq; + u_int msi; }; struct aplic_softc { + struct aplic_dev apd; device_t dev; + device_t parent; + struct intr_pic *pic; struct resource *intc_res; struct aplic_irqsrc isrcs[APLIC_MAX_IRQS]; int ndev; + int has_imsic; }; #define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */ @@ -173,6 +183,19 @@ device_printf(sc->dev, "Stray irq %u detected\n", irq); } +static int +aplic_imsic_intr(void *arg, unsigned long irq) +{ + struct aplic_softc *sc; + struct trapframe *tf; + + sc = arg; + tf = curthread->td_intr_frame; + aplic_irq_dispatch(sc, irq, 0, tf); + + return (FILTER_HANDLED); +} + static int aplic_intr(void *arg) { @@ -265,6 +288,27 @@ return (BUS_PROBE_DEFAULT); } +static int +aplic_registration_imsc_callback(device_t parent, void *cookie) +{ + struct aplic_softc *sc; + device_t dev = (device_t)cookie; + int err; + + sc = device_get_softc(dev); + + sc->parent = parent; + + err = intr_pic_add_handler(parent, sc->pic, aplic_imsic_intr, sc, + 1, 1023); + if (err != 0) { + device_printf(dev, "Failed to install child handler\n"); + return (ENXIO); + } + + return 0; +} + /* * Setup APLIC in direct mode. */ @@ -273,7 +317,7 @@ { struct aplic_irqsrc *isrcs; struct aplic_softc *sc; - struct intr_pic *pic; + struct aplic_dev *apd; uint32_t irq; uint32_t cpu; uint32_t hartid; @@ -309,9 +353,16 @@ return (ENXIO); } - /* Set APLIC in direct mode and enable all interrupts */ - aplic_write(sc, APLIC_DOMAIN_CFG, - APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE); + if (sc->has_imsic) { + aplic_write(sc, APLIC_DOMAIN_CFG, + APLIC_MODE_MSI | APLIC_DOMAIN_CFG_IE); + device_printf(dev, "Configuring APLIC with IMSIC.\n"); + } else { + device_printf(dev, "Configuring APLIC in Direct mode.\n"); + /* Set APLIC in direct mode and enable all interrupts */ + aplic_write(sc, APLIC_DOMAIN_CFG, + APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE); + } /* Register the interrupt sources */ isrcs = sc->isrcs; @@ -327,23 +378,32 @@ APLIC_SRC_CFG_SM_DETACHED); } - /* Turn off the interrupt delivery for all CPUs */ - CPU_FOREACH(cpu) { - hartid = riscv_cpu_to_hartid(cpu); - aplic_write(sc, APLIC_IDC_IDELIVERY(hartid), - APLIC_IDC_IDELIVERY_DISABLE); - aplic_write(sc, APLIC_IDC_ITHRESHOLD(hartid), - APLIC_IDC_ITHRESHOLD_DISABLE); + if (!sc->has_imsic) { + /* Turn off the interrupt delivery for all CPUs */ + CPU_FOREACH(cpu) { + hartid = riscv_cpu_to_hartid(cpu); + aplic_write(sc, APLIC_IDC_IDELIVERY(hartid), + APLIC_IDC_IDELIVERY_DISABLE); + aplic_write(sc, APLIC_IDC_ITHRESHOLD(hartid), + APLIC_IDC_ITHRESHOLD_DISABLE); + } } xref = OF_xref_from_node(node); - pic = intr_pic_register(sc->dev, xref); - if (pic == NULL) { + sc->pic = intr_pic_register(sc->dev, xref); + if (sc->pic == NULL) { error = ENXIO; goto fail; } - intr_pic_claim_root(sc->dev, xref, aplic_intr, sc, 0); + if (!sc->has_imsic) + intr_pic_claim_root(sc->dev, xref, aplic_intr, sc, 0); + else { + apd = &sc->apd; + apd->callback = aplic_registration_imsc_callback; + apd->cookie = (void *)dev; + imsic_register_aplic(apd); + } csr_set(sie, SIE_SEIE); @@ -362,13 +422,16 @@ static int aplic_attach(device_t dev) { + struct aplic_softc *sc; int rc; + sc = device_get_softc(dev); + /* APLIC with IMSIC on hart is not supported */ if (ofw_bus_has_prop(dev, "msi-parent")) { - device_printf(dev, "APLIC with IMSIC is unsupported\n"); - return (ENXIO); - } + sc->has_imsic = 1; + } else + sc->has_imsic = 0; rc = aplic_setup_direct_mode(dev); @@ -436,9 +499,22 @@ hartid = riscv_cpu_to_hartid(cpu); - aplic_write(sc, APLIC_TARGET(src->irq), - APLIC_MK_IRQ_TARGET(hartid, APLIC_INTR_DEF_PRIO)); - aplic_write(sc, APLIC_IDC_IDELIVERY(hartid), APLIC_IDC_IDELIVERY_ENABLE); + if (!sc->has_imsic) { + aplic_write(sc, APLIC_TARGET(src->irq), + APLIC_MK_IRQ_TARGET(hartid, APLIC_INTR_DEF_PRIO)); + aplic_write(sc, APLIC_IDC_IDELIVERY(hartid), + APLIC_IDC_IDELIVERY_ENABLE); + } else { + if (imsic_alloc_msi(sc->parent, cpu, src->irq, &src->msi)) { + device_printf(dev, "MSI allocation for irq %d failed\n", + src->irq); + return (ENXIO); + } + aplic_write(sc, APLIC_TARGET(src->irq), + (((hartid << 18) | (src->msi & 0x7ff)) & ~(0x3f000ul))); + imsic_enable_irq(sc->parent, cpu, src->msi); + } + aplic_enable_intr(dev, isrc); return (0); diff --git a/sys/riscv/riscv/imsic.c b/sys/riscv/riscv/imsic.c new file mode 100644 --- /dev/null +++ b/sys/riscv/riscv/imsic.c @@ -0,0 +1,804 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Himanshu Chauhan + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +static pic_disable_intr_t imsic_disable_intr; +static pic_enable_intr_t imsic_enable_intr; +static pic_map_intr_t imsic_map_intr; +static pic_setup_intr_t imsic_setup_intr; +static pic_post_ithread_t imsic_post_ithread; +static pic_pre_ithread_t imsic_pre_ithread; +static pic_bind_intr_t imsic_bind_intr; + +#define IMSIC_MAX_MSI 2047 /* Maximum MSI number in IMSIC */ +#define IMSIC_MSI_PPN_SHIFT 12 /* Number of bits to shift for PPN */ +#define IMSIC_PAGE_SHIFT 12 +#define IMSIC_PAGE_SIZE (1UL << 12) /* Size of interrupt file region */ + +#define INVALID_IRQ (0) /* 0 is invalid IRQ in IMSIC */ + +#define IMSIC_CSR_SISELECT 0x150 +#define IMSIC_CSR_SIREG 0x151 +#define IMSIC_CSR_STOPEI 0x15c +#define IMSIC_CSR_STOPI 0xdb0 + +#ifdef DECLARE_CSR +DECLARE_CSR(siselect, IMSIC_SISELCT) +DECLARE_CSR(sireg, IMSIC_SIREG) +DECLARE_CSR(stopei, IMSIC_STOPEI) +DECLARE_CSR(stopi, IMSIC_STOPI) +#endif + +#define IMSIC_EIDELIVERY 0x70 +#define IMSIC_EITHRESHOLD 0x72 + +#define IMSIC_EIP_BASE 0x80 +#define IMSIC_EIE_BASE 0xc0 + +#if __riscv_xlen == 32 +#define IMSIC_INTR_TO_EIP_REG(_n) (IMSIC_EIP_BASE + (_n / 32)) +#elif __riscv_xlen == 64 +/* AIA spec (3.8.3), odd registers are invalid if xlen is 64 */ +#define IMSIC_INTR_TO_EIP_REG(_n) (IMSIC_EIP_BASE + ((_n / 64) * 2) +#else +#error "Undefined __riscv_xlen" +#endif + +#if __riscv_xlen == 32 +#define IMSIC_INTR_TO_EIE_REG(_n) (IMSIC_EIE_BASE + (_n / 32)) +#elif __riscv_xlen == 64 +/* AIA spec (3.8.3), odd registers are invalid if xlen is 64 */ +#define IMSIC_INTR_TO_EIE_REG(_n) (IMSIC_EIE_BASE + ((_n / 64) * 2)) +#else +#error "Undefined __riscv_xlen" +#endif + +#define IMSIC_INTR_DISABLED 0x0 +#define IMSIC_INTR_FILE_EN 0x1 +#define IMSIC_INTR_APLIC_EN 0x40000000UL +#define IMSIC_ITHRESHOLD_DISABLE (0) + +/* + * MSI to be used as IPI. Lowest MSI has highest priority. + * Using one to give IPI the highest priority. + */ +#define IMSIC_IPI_MSI (1) + +#define IMSIC_FILL(p) BIT_FILL(IMSIC_MAX_MSI, p) +#define IMSIC_ISEMPTY(p) BIT_EMPTY(IMSIC_MAX_MSI, p) +#define IMSIC_FFS(p) BIT_FFS(IMSIC_MAX_MSI, p) +#define IMSIC_SETMSI(n, p) BIT_SET(IMSIC_MAX_MSI, n, p) +#define IMSIC_CLRMSI(n, p) BIT_CLR(IMSIC_MAX_MSI, n, p) + +struct imsic_msi_config { + unsigned long lhxs; /* Guest index shift */ + unsigned long lhxw; /* Hart index width */ + unsigned long hhxs; /* Group index shift */ + unsigned long hhxw; /* Group index width */ + unsigned long msi_base; /* Base address for PPN */ + unsigned long msi_sz; /* Total MSI region size for all harts */ +}; + +__BITSET_DEFINE(_msiset, IMSIC_MAX_MSI); +typedef struct _msiset msiset_t; + +struct imsic_irqsrc { + struct intr_irqsrc isrc; + u_int msi; + u_int aplic_irq; +}; + +struct imsic_pcpu { + struct mtx access_lock; + msiset_t allocated_msis; + struct imsic_irqsrc isrcs[IMSIC_MAX_MSI]; + uint32_t ipi_init_done; +}; + +struct imsic_softc { + device_t dev; + device_t aplic_dev; + struct intr_pic *pic; + struct resource *intc_res; + struct imsic_pcpu pcpu_data[MAXCPU]; + struct imsic_msi_config msi_config; + int interrupts_extended; + int ndev; +}; + +static driver_filter_t *imsic_ipi_handler; + +static void imsic_enable_ipi_irq(void); +static void imsic_setup_ipi_handler(driver_filter_t *filt); +static int imsic_send_ipi(cpuset_t cpus); + +struct riscv_ipi_device imsic_ipi_dev = { + .name = "imsic-ipi-device", + .unmask_ipi = imsic_enable_ipi_irq, + .setup_ipi_handler = imsic_setup_ipi_handler, + .send_ipi = imsic_send_ipi, + .ready = 0, +}; + +static uint64_t nr_imsic_ipis; +static struct sysctl_ctx_list imsic_clist; +struct sysctl_oid *aia_poid = NULL; +static struct sysctl_oid *imsic_poid = NULL; +struct sysctl_ctx_list aia_clist; +static int imsic_ipi_enabled = 1; +static uint64_t *v_msi_addr[MAXCPU] = { 0 }; +static int imsic_present; + +/* APLIC children */ +static TAILQ_HEAD(, aplic_dev) aplic_list = TAILQ_HEAD_INITIALIZER(aplic_list); + +static inline void +imsic_csr_write(u_int reg, uint64_t val) +{ + csr_write(siselect, reg); + csr_write(sireg, val); +} + +static inline uint64_t +imsic_csr_read(u_int reg) +{ + uint64_t _v; + + csr_write(siselect, reg); + _v = csr_read(sireg); + + return (_v); +} + +/* + * Clear the current pending interrupt and return the + * interrupt that was cleared. + */ +static inline u_long +imsic_get_pending_interrupt(void) +{ + u_long c_intr = 0; + + /* Value written is ignored. */ + csr_swap(stopei, c_intr); + + return c_intr; +} + +static inline void +imsic_enable_interrupt(u_int interrupt) +{ + uint64_t eie_v; + + eie_v = imsic_csr_read(IMSIC_INTR_TO_EIE_REG(interrupt)); + eie_v |= (0x1UL << (interrupt % 64)); + imsic_csr_write(IMSIC_INTR_TO_EIE_REG(interrupt), eie_v); +} + +static inline void +imsic_disable_interrupt(u_int interrupt) +{ + uint64_t eie_v; + + eie_v = imsic_csr_read(IMSIC_INTR_TO_EIE_REG(interrupt)); + eie_v &= ~(0x1UL << (interrupt % 64)); + imsic_csr_write(IMSIC_INTR_TO_EIE_REG(interrupt), eie_v); +} + +static int +imsic_intr(void *arg) +{ + struct imsic_softc *sc; + u_long msi, aplic_irq; + struct trapframe *tf; + struct imsic_irqsrc *src; + int cpu = PCPU_GET(cpuid); + + sc = arg; + + msi = imsic_get_pending_interrupt(); + msi >>= 16; + + if (!msi) + return (FILTER_STRAY); + + if (msi == 1) { + atomic_fetchadd_64(&nr_imsic_ipis, 1); + if (imsic_ipi_handler) { + imsic_ipi_handler(NULL); + } + return (FILTER_HANDLED); + } + + aplic_irq = sc->pcpu_data[cpu].isrcs[msi].aplic_irq; + if (aplic_irq == INVALID_IRQ) { + src = &sc->pcpu_data[cpu].isrcs[msi]; + tf = curthread->td_intr_frame; + if (intr_isrc_dispatch(&src->isrc, tf) != 0) { + device_printf(sc->dev, "Stray irq %lu detected\n", msi); + return (FILTER_STRAY); + } + } else { + intr_child_irq_handler(sc->pic, aplic_irq); + } + + return (FILTER_HANDLED); +} + +static void +imsic_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct imsic_irqsrc *src; + + src = (struct imsic_irqsrc *)isrc; + + /* Disable the interrupt source */ + imsic_disable_interrupt(src->msi); +} + +static void +imsic_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct imsic_irqsrc *src; + + src = (struct imsic_irqsrc *)isrc; + + /* Enable the interrupt source */ + imsic_enable_interrupt(src->msi); +} + +static int +imsic_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + return 0; +} + +static void imsic_enable_ipi_irq(void) +{ + imsic_enable_interrupt(IMSIC_IPI_MSI); +} + +static void +imsic_setup_ipi_handler(driver_filter_t *filt) +{ + imsic_ipi_handler = filt; +} + +static int imsic_send_ipi(cpuset_t cpus) +{ + int cpu; + uint64_t *msi_vaddr; + + CPU_FOREACH(cpu) { + if (CPU_ISSET(cpu, &cpus)) { + msi_vaddr = v_msi_addr[cpu]; + if (msi_vaddr) { + /* Write MSI interrupt number */ + writel(msi_vaddr, IMSIC_IPI_MSI); + } + } + } + + return 0; +} + +static int +imsic_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "riscv,imsics")) + return (ENXIO); + + if (!ofw_bus_has_prop(dev, "msi-controller")) + return (ENXIO); + + device_set_desc(dev, "Incoming MSI Controller"); + + return (BUS_PROBE_DEFAULT); +} + +#ifdef SMP +static void +imsic_setup_on_each_cpu(void *arg) +{ + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_INTR_FILE_EN); + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ITHRESHOLD_DISABLE); + imsic_enable_interrupt(IMSIC_IPI_MSI); +} + +static int +smp_imsic_setup(SYSCTL_HANDLER_ARGS) +{ + cpuset_t cpus; + int cpu = PCPU_GET(cpuid); + int i; + + if (!imsic_present) + return (0); + + CPU_ZERO(&cpus); + CPU_FOREACH(i) { + if (i == cpu) + continue; + CPU_SET(i, &cpus); + } + + /* Setup imsic registers on all other CPUs */ + smp_rendezvous_cpus(cpus, NULL, imsic_setup_on_each_cpu, + NULL, NULL); + + /* imsic on all CPUs are ready to participate in IPIs */ + imsic_ipi_dev.ready = 1; + + return (0); +} +SYSINIT(smp_imsic_setup, SI_SUB_SMP, SI_ORDER_SECOND, smp_imsic_setup, NULL); +#endif + +static int +sysctl_read_nr_ipis(SYSCTL_HANDLER_ARGS) +{ + uint64_t val; + int err; + + /* get realtime value */ + val = atomic_fetchadd_64(&nr_imsic_ipis, 0); + + err = sysctl_handle_64(oidp, &val, 0, req); + if (err || !req->newptr) /* error || read request */ + return (err); + + /* write request */ + return (EINVAL); +} + +static int +imsic_interrupts_extended(device_t dev) +{ + struct imsic_softc *sc; + phandle_t node; + pcell_t *buf; + ssize_t len; + int rc = ENXIO; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + sc->interrupts_extended = 0; + + if ((len = OF_getencprop_alloc(node, "interrupts-extended", + (void *)&buf)) < 0) { + device_printf(dev, "interrupts-extended property not found.\n"); + goto _out; + } + + if (buf[1] != IRQ_EXTERNAL_SUPERVISOR) { + device_printf(dev,"Interrupts not extended to supervisor.\n"); + goto _fail; + } + + sc->interrupts_extended = len/(sizeof(pcell_t) * 2); + + rc = 0; + + _fail: + OF_prop_free(buf); + _out: + return (rc); +} + +static int +imsic_parse_fdt_msi_config(device_t dev) +{ + struct imsic_softc *sc; + phandle_t node; + pcell_t *buf; + pcell_t reg[4]; + ssize_t len; + int rc = 0; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + if ((len = OF_getencprop_alloc(node, "riscv,guest-index-bits", + (void **)&buf)) < 0) { + sc->msi_config.lhxs = 0; + } else { + sc->msi_config.lhxs = buf[0]; + } + + OF_prop_free(buf); + + if ((len = OF_getencprop_alloc(node, "riscv,hart-index-bits", + (void *)&buf)) < 0) { + sc->msi_config.lhxw = flsl(sc->interrupts_extended); + if ((1UL << sc->msi_config.lhxw) < sc->interrupts_extended) + sc->msi_config.lhxw++; + } else + sc->msi_config.lhxw = buf[0]; + + if ((len = OF_getencprop_alloc(node, "riscv,group-index-bits", + (void *)&buf)) < 0) { + sc->msi_config.hhxw = 0; + } else + sc->msi_config.hhxw = buf[0]; + + OF_prop_free(buf); + + if ((len = OF_getencprop_alloc(node, "riscv,group-index-shift", + (void *)&buf)) < 0) { + sc->msi_config.hhxs = 2 * IMSIC_PAGE_SHIFT; + } else + sc->msi_config.hhxs = buf[0]; + + if (sc->msi_config.hhxs < (2 * IMSIC_PAGE_SHIFT)) + return (ENXIO); + sc->msi_config.hhxs -= 24; + + if ((OF_getencprop(node, "reg", ®[0], sizeof(reg))) < 0) { + device_printf(dev, "Failed to read regs property\n"); + } + + sc->msi_config.msi_base = ((uint64_t)reg[0] << 32) | reg[1]; + sc->msi_config.msi_sz = ((uint64_t)reg[2] << 32) | reg[3]; + + OF_prop_free(buf); + + return (rc); +} + +static int +imsic_configure_ipi(device_t dev) +{ + struct imsic_softc *sc; + vm_paddr_t msi_addr, ppn; + int msi_hart, msi_cpu, i; + uint64_t *v_msi; + char *env; + + env = kern_getenv("aia.imsic.ipi_disable"); + if (env) { + imsic_ipi_enabled = 0; + return (ENXIO); + } + + sc = device_get_softc(dev); + + /* Get the MSI config needed to calculate MSI addresses for IPI */ + if (imsic_parse_fdt_msi_config(dev) < 0) { + imsic_ipi_enabled = 0; + return (ENXIO); + } + + /* MSI regions in FDT are more than CPUs supported */ + if (sc->msi_config.msi_sz > ((1UL << IMSIC_PAGE_SHIFT) * MAXCPU)) { + imsic_ipi_enabled = 0; + return (ENXIO); + } + + /* + * We are good to calculate MSI addresses for CPUs to send IPIs. + * + * msi_addr = (base ppn | (g << (hhsx + 12)) | (h << lhxs)) << 12 + * + * NOTE: We assume tha group is zero here. So we skip the middle + * part. + */ + CPU_FOREACH(msi_cpu) { + msi_hart = pcpu_find(msi_cpu)->pc_hart; + ppn = (sc->msi_config.msi_base >> IMSIC_MSI_PPN_SHIFT); + msi_addr = (ppn | (msi_hart << sc->msi_config.lhxs)) + << IMSIC_MSI_PPN_SHIFT; + + if (bootverbose) { + device_printf(dev, "msi_addr_%d: 0x%lx\n", msi_cpu, + msi_addr); + } + + v_msi = (uint64_t *)pmap_mapdev((vm_paddr_t)msi_addr, + IMSIC_PAGE_SIZE); + if (!v_msi) { + if (bootverbose) { + device_printf(dev, + "MSI address mapping failed.\n"); + } + /* + * Event if a single pmap fails, we cannot do a writel + * and hence cannot deliver IPI. Disable IPIs via imsic. + */ + imsic_ipi_enabled = 0; + goto _setup_failed; + } + v_msi_addr[msi_cpu] = v_msi; + } + + /* Enable MSI IPI on this CPU */ + imsic_enable_interrupt(IMSIC_IPI_MSI); + + imsic_ipi_handler = NULL; + + /* + * Register IPI device only if we have all the mappings. + * If not, the IPIs will keep working via sbi ecall. + */ + if (imsic_ipi_enabled) + riscv_register_ipi_device(&imsic_ipi_dev); + + return (0); + + _setup_failed: + for (i = 0; i < MAXCPU; i++) { + if (v_msi_addr[i] != NULL) { + pmap_unmapdev(v_msi_addr[i], IMSIC_PAGE_SIZE); + v_msi_addr[i] = NULL; + } + } + + return (ENXIO); +} + +static void +imsic_setup_sysctl(void) +{ + if (aia_poid == NULL) { + aia_poid = SYSCTL_ADD_NODE(&aia_clist, + SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, "aia", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); + } + + imsic_poid = SYSCTL_ADD_NODE(&imsic_clist, SYSCTL_CHILDREN(aia_poid), + OID_AUTO, "imsic", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "imsic root"); + SYSCTL_ADD_INT(&imsic_clist, SYSCTL_CHILDREN(imsic_poid), + OID_AUTO, "ipi_enabled", CTLFLAG_RD, &imsic_ipi_enabled, 0, + "Whether IPIs using IMSIC are enabled"); + SYSCTL_ADD_PROC(&imsic_clist, SYSCTL_CHILDREN(imsic_poid), + OID_AUTO, "ipi_count", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, NULL, 0, + sysctl_read_nr_ipis, "QU", "Number of IPIs triggered using IMSIC"); +} + +static int +imsic_attach(device_t dev) +{ + struct aplic_dev *aplic; + struct imsic_softc *sc; + phandle_t node; + phandle_t xref; + int i, j; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + sc->dev = dev; + + if (imsic_interrupts_extended(dev)) { + device_printf(dev, "Interrupts not extended to supervisor.\n"); + return (ENXIO); + } + + imsic_present = 1; + + xref = OF_xref_from_node(node); + sc->pic = intr_pic_register(dev, xref); + if (sc->pic == NULL) { + device_printf(dev, "PIC registration unsuccessful\n"); + return (ENXIO); + } + + /* + * Enable interrupt delivery by interrupt file. APLIC will forward the + * interrupts to IMSIC as MSI writes. + */ + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_INTR_FILE_EN); + + /* Receive all interrupts */ + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ITHRESHOLD_DISABLE); + + for (i = 0; i < MAXCPU; i++) { + mtx_init(&sc->pcpu_data[i].access_lock, "imsic-pcpu-lock", NULL, + MTX_SPIN); + IMSIC_FILL(&sc->pcpu_data[i].allocated_msis); + + /* Clear the MSI bit which is an invalid MSI/IRQ */ + IMSIC_CLRMSI(INVALID_IRQ, &sc->pcpu_data[i].allocated_msis); + /* Mark IPI MSI as used. */ + IMSIC_CLRMSI(IMSIC_IPI_MSI, &sc->pcpu_data[i].allocated_msis); + + for (j = 0; j <= IMSIC_MAX_MSI; j++) { + sc->pcpu_data[i].isrcs[j].aplic_irq = INVALID_IRQ; + sc->pcpu_data[i].isrcs[j].msi = INVALID_IRQ; + } + } + + intr_pic_claim_root(sc->dev, xref, imsic_intr, sc, 0); + +#ifdef SMP + /* Setup IMSIC for IPIs */ + imsic_configure_ipi(dev); +#endif + + /* Call APLIC so that it can register as child of IMSIC */ + TAILQ_FOREACH(aplic, &aplic_list, next) { + aplic->callback(dev, aplic->cookie); + } + + /* Setup OIDs to publish IPI details */ + imsic_setup_sysctl(); + + return (0); +} + +static void +imsic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); +} + +static void +imsic_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); +} + +static int +imsic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, + struct intr_map_data *data) +{ + device_printf(dev, "%s\n", __func__); + return (0); +} + +static int +imsic_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + device_printf(dev, "%s\n", __func__); + return (0); +} + +int +imsic_register_aplic(struct aplic_dev *dev) +{ + TAILQ_INSERT_TAIL(&aplic_list, dev, next); + + return 0; +} + +int +imsic_alloc_msi(device_t dev, int cpu, int irq, int *msi) +{ + struct imsic_softc *sc; + int error = 0; + + sc = device_get_softc(dev); + + mtx_lock_spin(&sc->pcpu_data[cpu].access_lock); + + /* If none allocated_msis == 0, no MSI left to allocate */ + if (IMSIC_ISEMPTY(&sc->pcpu_data[cpu].allocated_msis)) { + error = ENXIO; + goto _out; + } + + /* Find free MSI (first bit set) */ + *msi = IMSIC_FFS(&sc->pcpu_data[cpu].allocated_msis)-1; + + /* Clear the bit to mark as allocated */ + IMSIC_CLRMSI(*msi, &sc->pcpu_data[cpu].allocated_msis); + + /* Save MSI to APLIC IRQ to pass on to child handler */ + sc->pcpu_data[cpu].isrcs[*msi].msi = *msi; + sc->pcpu_data[cpu].isrcs[*msi].aplic_irq = irq; + + error = 0; + + _out: + mtx_unlock_spin(&sc->pcpu_data[cpu].access_lock); + + return (error); +} + +static void +smp_imsic_enable_interrupt(void *arg) +{ + uint64_t irq = (uint64_t)arg; + + imsic_enable_interrupt(irq); +} + +int +imsic_enable_irq(device_t dev, int cpu, int irq) +{ + cpuset_t cpus; + + if (cpu == PCPU_GET(cpuid)) { + imsic_enable_interrupt(irq); + } else { +#ifdef SMP + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + smp_rendezvous_cpus(cpus, NULL, smp_imsic_enable_interrupt, + NULL, (void *)((uint64_t)irq)); +#endif + } + + return 0; +} + +static device_method_t imsic_methods[] = { + DEVMETHOD(device_probe, imsic_probe), + DEVMETHOD(device_attach, imsic_attach), + + DEVMETHOD(pic_disable_intr, imsic_disable_intr), + DEVMETHOD(pic_enable_intr, imsic_enable_intr), + DEVMETHOD(pic_map_intr, imsic_map_intr), + DEVMETHOD(pic_pre_ithread, imsic_pre_ithread), + DEVMETHOD(pic_post_ithread, imsic_post_ithread), + DEVMETHOD(pic_post_filter, imsic_post_ithread), + DEVMETHOD(pic_setup_intr, imsic_setup_intr), + DEVMETHOD(pic_bind_intr, imsic_bind_intr), + + DEVMETHOD_END +}; + +static driver_t imsic_driver = { + "imsic", + imsic_methods, + sizeof(struct imsic_softc), +}; + +EARLY_DRIVER_MODULE(imsic, simplebus, imsic_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/riscv/riscv/intr_machdep.c b/sys/riscv/riscv/intr_machdep.c --- a/sys/riscv/riscv/intr_machdep.c +++ b/sys/riscv/riscv/intr_machdep.c @@ -67,6 +67,8 @@ struct intc_irqsrc isrcs[INTC_NIRQS]; +static struct riscv_ipi_device *ipi_device = NULL; + static void riscv_mask_irq(void *source) { @@ -183,14 +185,22 @@ void riscv_setup_ipihandler(driver_filter_t *filt) { - - riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, - INTR_TYPE_MISC, NULL); + if (ipi_device) + ipi_device->setup_ipi_handler(filt); + /* + * register for software interrupts as initial + * ipi will be from sbi + */ + riscv_setup_intr("ipi", filt, NULL, NULL, + IRQ_SOFTWARE_SUPERVISOR, INTR_TYPE_MISC, NULL); } void riscv_unmask_ipi(void) { + if (ipi_device && ipi_device->ready) { + ipi_device->unmask_ipi(); + } csr_set(sie, SIE_SSIE); } @@ -200,17 +210,40 @@ ipi_send(struct pcpu *pc, int ipi) { u_long mask; + cpuset_t cpus; CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi); atomic_set_32(&pc->pc_pending_ipis, ipi); - mask = (1 << pc->pc_hart); - sbi_send_ipi(&mask); + if (ipi_device && ipi_device->ready) { + CPU_ZERO(&cpus); + CPU_SET(pc->pc_cpuid, &cpus); + ipi_device->send_ipi(cpus); + } else { + mask = (1 << pc->pc_hart); + + sbi_send_ipi(&mask); + } CTR1(KTR_SMP, "%s: sent", __func__); } +int riscv_register_ipi_device(struct riscv_ipi_device *dev) +{ + if (ipi_device) { + if (bootverbose) + printf("IPI device is already registered\n"); + return (EBUSY); + } + + ipi_device = dev; + + printf("%s set as IPI device\n", dev->name); + + return (0); +} + void ipi_all_but_self(u_int ipi) { @@ -251,7 +284,11 @@ mask |= (1 << pc->pc_hart); } } - sbi_send_ipi(&mask); + + if (ipi_device && ipi_device->ready) + ipi_device->send_ipi(cpus); + else + sbi_send_ipi(&mask); } #endif