Page MenuHomeFreeBSD
Files F173567547

D59695.diff
No OneTemporary
Actions

D59695.diff
View Options

Index: sys/conf/files.riscv
===================================================================
--- sys/conf/files.riscv
+++ sys/conf/files.riscv
@@ -71,6 +71,7 @@
riscv/riscv/exec_machdep.c standard
riscv/riscv/fpe.c standard
riscv/riscv/gdb_machdep.c optional gdb
+riscv/riscv/imsic.c standard
riscv/riscv/intc.c standard
riscv/riscv/identcpu.c standard
riscv/riscv/locore.S standard no-obj
Index: sys/riscv/include/imsic.h
===================================================================
--- /dev/null
+++ sys/riscv/include/imsic.h
@@ -0,0 +1,31 @@
+/*-
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_IMSIC_H_
+#define _MACHINE_IMSIC_H_
+
+int imsic_alloc_aplic_msi(device_t dev, int cpu, int aplic_irq, int *msi);
+
+#endif /* !_MACHINE_IMSIC_H_ */
Index: sys/riscv/include/intr.h
===================================================================
--- sys/riscv/include/intr.h
+++ sys/riscv/include/intr.h
@@ -36,7 +36,12 @@
#define _MACHINE_INTR_MACHDEP_H_
#ifndef NIRQ
-#define NIRQ 1024
+/*
+ * Max interrupts per spec:
+ * APLIC is 1023 total
+ * IMSIC is 2047 per core
+ */
+#define NIRQ 8192
#endif
#ifndef LOCORE
Index: sys/riscv/riscv/aplic.c
===================================================================
--- sys/riscv/riscv/aplic.c
+++ sys/riscv/riscv/aplic.c
@@ -1,6 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
* Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev>
*
* Redistribution and use in source and binary forms, with or without
@@ -34,15 +35,19 @@
#include <sys/proc.h>
#include <sys/rman.h>
#include <sys/smp.h>
+#include <sys/intr.h>
#include <machine/bus.h>
#include <machine/intr.h>
#include <machine/riscvreg.h>
+#include <machine/imsic.h>
#include <dev/ofw/openfirm.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
#include "pic_if.h"
#define APLIC_MAX_IRQS 1023
@@ -61,6 +66,8 @@
struct aplic_irqsrc {
struct intr_irqsrc isrc;
u_int irq;
+ u_int msi;
+ u_int type;
};
struct aplic_softc {
@@ -69,125 +76,136 @@
struct resource *irq_res;
void *ih;
struct aplic_irqsrc isrcs[APLIC_MAX_IRQS + 1];
- unsigned int hart_indices[MAXCPU];
+ u_int hart_indices[MAXCPU];
+ device_t parent;
+ struct intr_pic *pic;
int ndev;
+ bool has_imsic;
};
-#define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */
-#define APLIC_DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */
-#define APLIC_DOMAIN_CFG_BE (1UL << 0) /* Endianess */
-
-#define APLIC_MODE_DIRECT 0 /* Direct delivery mode */
-#define APLIC_MODE_MSI 1 /* MSI delivery mode */
-
-#define APLIC_SRC_CFG_DLGT (1UL << 10) /* Source delegation */
-#define APLIC_SRC_CFG_SM_SHIFT 0
-#define APLIC_SRC_CFG_SM_MASK (0x7UL << APLIC_SRC_CFG_SM_SHIFT)
-
-#define APLIC_SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */
-#define APLIC_SRC_CFG_SM_DETACHED 1 /* Detached from source wire */
-#define APLIC_SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */
-#define APLIC_SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */
-#define APLIC_SRC_CFG_SM_LVL_HI 6 /* Asserted when high */
-#define APLIC_SRC_CFG_SM_LVL_LO 7 /* Asserted when low */
-
/* Register offsets in APLIC configuration space */
-#define APLIC_DOMAIN_CFG 0x0000
-#define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4))
-#define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4))
-#define APLIC_MMSIADDRCFG 0x1BC0
-#define APLIC_MMSIADDRCFGH 0x1BC4
-#define APLIC_SMSIADDRCFG 0x1BC8
-#define APLIC_SMSIADDRCFGH 0x1BCC
-#define APLIC_SETIPNUM 0x1CDC
-#define APLIC_CLRIPNUM 0x1DDC
-#define APLIC_SETIENUM 0x1EDC
-#define APLIC_CLRIENUM 0x1FDC
-#define APLIC_SETIPNUM_LE 0x2000
-#define APLIC_SETIPNUM_BE 0x2004
-#define APLIC_GENMSI 0x3000
-#define APLIC_IDC_BASE 0x4000
-
-#define APLIC_SETIE_BASE 0x1E00
-#define APLIC_CLRIE_BASE 0x1F00
+#define APLIC_DOMAIN_CFG 0x0000
+#define DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */
+#define DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */
+#define DOMAIN_CFG_BE (1UL << 0) /* Endianess */
+#define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4))
+#define SRC_CFG_DLGT (1UL << 10) /* Source delegation */
+#define SRC_CFG_SM_SHIFT 0
+#define SRC_CFG_SM_MASK (0x7UL << SRC_CFG_SM_SHIFT)
+#define SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */
+#define SRC_CFG_SM_DETACHED 1 /* Detached from source wire */
+#define SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */
+#define SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */
+#define SRC_CFG_SM_LVL_HI 6 /* Asserted when high */
+#define SRC_CFG_SM_LVL_LO 7 /* Asserted when low */
+#define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4))
+#define TARGET_HART_ID_S 18
+#define TARGET_HART_ID_M (0x3fff << TARGET_HART_ID_S)
+/* Direct delivery mode. */
+#define TARGET_IPRIO_S 0
+#define TARGET_IPRIO_M (0xff << TARGET_IPRIO_S)
+/* MSI delivery mode. */
+#define TARGET_GUEST_ID_S 12
+#define TARGET_GUEST_ID_M (0x3f << TARGET_GUEST_ID_S)
+#define TARGET_EIID_S 0
+#define TARGET_EIID_M (0x7ff << TARGET_EIID_S)
+#define APLIC_MMSIADDRCFG 0x1BC0
+#define APLIC_MMSIADDRCFGH 0x1BC4
+#define APLIC_SMSIADDRCFG 0x1BC8
+#define APLIC_SMSIADDRCFGH 0x1BCC
+#define APLIC_SETIPNUM 0x1CDC
+#define APLIC_CLRIPNUM 0x1DDC
+#define APLIC_SETIE_BASE 0x1E00
+#define APLIC_SETIENUM 0x1EDC
+#define APLIC_CLRIE_BASE 0x1F00
+#define APLIC_CLRIENUM 0x1FDC
+#define APLIC_SETIPNUM_LE 0x2000
+#define APLIC_SETIPNUM_BE 0x2004
+#define APLIC_GENMSI 0x3000
+#define APLIC_IDC_BASE 0x4000
/* Interrupt delivery control structure */
+#define APLIC_IDC_SIZE 0x20
#define APLIC_IDC_IDELIVERY_OFFS 0x0000
#define APLIC_IDC_IFORCE_OFFS 0x0004
#define APLIC_IDC_ITHRESHOLD_OFFS 0x0008
#define APLIC_IDC_TOPI_OFFS 0x0018
#define APLIC_IDC_CLAIMI_OFFS 0x001C
-
-#define APLIC_IDC_SZ 0x20
-
-#define APLIC_IDC_IDELIVERY_DISABLE 0
-#define APLIC_IDC_IDELIVERY_ENABLE 1
-#define APLIC_IDC_ITHRESHOLD_DISABLE 0
-
-#define APLIC_IDC_CLAIMI_PRIO_MASK 0xff
-#define APLIC_IDC_CLAIMI_IRQ_SHIFT 16
-#define APLIC_IDC_CLAIMI_IRQ_MASK 0x3ff
-
-#define APLIC_IDC_CLAIMI_IRQ(_claimi) \
- (((_claimi) >> APLIC_IDC_CLAIMI_IRQ_SHIFT) \
- & APLIC_IDC_CLAIMI_IRQ_MASK)
-
-#define APLIC_IDC_CLAIMI_PRIO(_claimi) ((_claimi) & APLIC_IDC_CLAIMI_PRIO_MASK)
-
#define APLIC_IDC_REG(_sc, _cpu, _field) \
- (APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \
- ((_sc->hart_indices[_cpu]) * APLIC_IDC_SZ))
-
-#define APLIC_IDC_IDELIVERY(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, IDELIVERY)
-
-#define APLIC_IDC_IFORCE(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, IFORCE)
-
-#define APLIC_IDC_ITHRESHOLD(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD)
-
-#define APLIC_IDC_TOPI(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, TOPI)
-
-#define APLIC_IDC_CLAIMI(_sc, _cpu) \
- APLIC_IDC_REG(_sc, _cpu, CLAIMI)
-
-#define APLIC_MK_IRQ_TARGET(_sc, _cpu, _prio) \
- (_sc->hart_indices[_cpu] << 18 | ((_prio) & 0xff))
+ APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \
+ ((_sc->hart_indices[_cpu]) * APLIC_IDC_SIZE)
+
+#define APLIC_IDC_IDELIVERY(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, IDELIVERY)
+#define IDELIVERY_DISABLE 0
+#define IDELIVERY_ENABLE 1
+#define APLIC_IDC_IFORCE(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, IFORCE)
+#define APLIC_IDC_ITHRESHOLD(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD)
+#define ITHRESHOLD_DISABLE 0
+#define APLIC_IDC_TOPI(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, TOPI)
+#define APLIC_IDC_CLAIMI(_sc, _cpu) APLIC_IDC_REG(_sc, _cpu, CLAIMI)
+#define CLAIMI_PRIO_MASK 0xff
+#define CLAIMI_IRQ_SHIFT 16
+#define CLAIMI_IRQ_MASK 0x3ff
+#define CLAIMI_PRIO(_r) ((_r) & CLAIMI_PRIO_MASK)
+#define CLAIMI_IRQ(_r) (((_r) >> CLAIMI_IRQ_SHIFT) & CLAIMI_IRQ_MASK)
#define aplic_read(sc, reg) bus_read_4(sc->mem_res, (reg))
#define aplic_write(sc, reg, val) bus_write_4(sc->mem_res, (reg), (val))
static u_int aplic_irq_cpu;
-static inline int
+static inline uint32_t
riscv_cpu_to_hartid(int cpu)
{
- return pcpu_find(cpu)->pc_hart;
+ return (pcpu_find(cpu)->pc_hart);
}
static inline int
-riscv_hartid_to_cpu(int hartid)
+riscv_hartid_to_cpu(uint32_t hartid)
{
int cpu;
CPU_FOREACH(cpu) {
if (riscv_cpu_to_hartid(cpu) == hartid)
- return cpu;
+ return (cpu);
}
return (-1);
}
static int
-fdt_get_hartid(device_t dev, phandle_t aplic)
+fdt_get_parent_imsic(device_t dev)
+{
+ struct aplic_softc *sc;
+ phandle_t parent_xref, node;
+ int rc;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ rc = OF_getencprop(node, "msi-parent", &parent_xref,
+ sizeof(parent_xref));
+ if (rc <= 0) {
+ device_printf(dev, "can't read parent node property\n");
+ return (ENXIO);
+ }
+
+ sc->parent = OF_device_from_xref(parent_xref);
+ if (sc->parent == NULL) {
+ device_printf(dev, "can't find parent MSI controller\n");
+ return (ENXIO);
+ }
+
+ return (0);
+}
+
+static int
+fdt_get_hartid(device_t dev, phandle_t aplic, uint32_t *hartid)
{
- int hartid;
/* Check the interrupt controller layout. */
- if (OF_searchencprop(aplic, "#interrupt-cells", &hartid,
- sizeof(hartid)) == -1) {
+ if (OF_searchencprop(aplic, "#interrupt-cells", hartid,
+ sizeof(*hartid)) == -1) {
device_printf(dev,
"Could not find #interrupt-cells for phandle %u\n", aplic);
return (-1);
@@ -197,13 +215,13 @@
* The parent of the interrupt-controller is the CPU we are
* interested in, so search for its hart ID.
*/
- if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)&hartid,
- sizeof(hartid)) == -1) {
+ if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)hartid,
+ sizeof(*hartid)) == -1) {
device_printf(dev, "Could not find hartid\n");
return (-1);
}
- return (hartid);
+ return (0);
}
static inline void
@@ -219,6 +237,19 @@
device_printf(sc->dev, "Stray irq %u detected\n", irq);
}
+static int
+aplic_imsic_intr(void *arg, unsigned long irq)
+{
+ struct aplic_softc *sc;
+ struct trapframe *tf;
+
+ sc = arg;
+ tf = curthread->td_intr_frame;
+ aplic_irq_dispatch(sc, irq, 0, tf);
+
+ return (FILTER_HANDLED);
+}
+
static int
aplic_intr(void *arg)
{
@@ -233,8 +264,8 @@
/* Claim all pending interrupts. */
while ((claimi = aplic_read(sc, APLIC_IDC_CLAIMI(sc, cpu))) != 0) {
- prio = APLIC_IDC_CLAIMI_PRIO(claimi);
- irq = APLIC_IDC_CLAIMI_IRQ(claimi);
+ prio = CLAIMI_PRIO(claimi);
+ irq = CLAIMI_IRQ(claimi);
KASSERT((irq != 0), ("Invalid IRQ 0"));
@@ -277,6 +308,8 @@
{
struct intr_map_data_fdt *daf;
struct aplic_softc *sc;
+ u_int type;
+ u_int irq;
sc = device_get_softc(dev);
@@ -289,7 +322,11 @@
return (EINVAL);
}
- *isrcp = &sc->isrcs[daf->cells[0]].isrc;
+ irq = daf->cells[0];
+ type = daf->cells[1];
+
+ sc->isrcs[irq].type = type;
+ *isrcp = &sc->isrcs[irq].isrc;
return (0);
}
@@ -313,16 +350,112 @@
*/
static int
aplic_setup_direct_mode(device_t dev)
+{
+ struct aplic_softc *sc;
+ phandle_t node;
+ pcell_t *cells;
+ uint32_t hartid;
+ int nintr, idc;
+ int cpu, i;
+ int error;
+
+ error = ENXIO;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ device_printf(dev, "Configuring APLIC in Direct mode.\n");
+ /* Set APLIC in direct mode and enable all interrupts */
+ aplic_write(sc, APLIC_DOMAIN_CFG, DOMAIN_CFG_IE);
+
+ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
+ sizeof(uint32_t), (void **)&cells);
+ if (nintr <= 0) {
+ device_printf(dev, "Could not read interrupts-extended\n");
+ goto fail;
+ }
+
+ /* interrupts-extended is a list of phandles and interrupt types. */
+ for (i = 0, idc = 0; i < nintr; i += 2, idc++) {
+ /* Skip M-mode external interrupts */
+ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
+ continue;
+
+ /* Get the hart ID from the CLIC's phandle. */
+ error = fdt_get_hartid(dev, OF_node_from_xref(cells[i]),
+ &hartid);
+ if (error != 0) {
+ OF_prop_free(cells);
+ goto fail;
+ }
+
+ /* Get the corresponding cpuid. */
+ cpu = riscv_hartid_to_cpu(hartid);
+ if (cpu == -1) {
+ device_printf(dev, "Invalid cpu for hart %d\n", hartid);
+ OF_prop_free(cells);
+ goto fail;
+ }
+
+ sc->hart_indices[cpu] = idc;
+ }
+
+ /* Turn off the interrupt delivery for all CPUs within or out domain */
+ CPU_FOREACH(cpu) {
+ aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
+ IDELIVERY_DISABLE);
+ aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu),
+ ITHRESHOLD_DISABLE);
+ }
+
+ error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
+ aplic_intr, NULL, sc, &sc->ih);
+ if (error != 0) {
+ device_printf(dev, "Unable to setup IRQ resource\n");
+ return (ENXIO);
+ }
+
+fail:
+ return (error);
+}
+
+static int
+aplic_setup_imsic_mode(device_t dev)
+{
+ struct aplic_softc *sc;
+ int error = ENXIO;
+
+ sc = device_get_softc(dev);
+
+ device_printf(dev, "Configuring APLIC with IMSIC.\n");
+
+ if ((error = fdt_get_parent_imsic(dev)) != 0)
+ return (error);
+
+ error = intr_pic_add_handler(sc->parent, sc->pic,
+ aplic_imsic_intr, sc, 1, APLIC_MAX_IRQS);
+ if (error != 0) {
+ device_printf(dev, "Failed to install child handler, "
+ "error %d\n", error);
+ return (error);
+ }
+
+ aplic_write(sc, APLIC_DOMAIN_CFG, DOMAIN_CFG_DM | DOMAIN_CFG_IE);
+
+ return (0);
+}
+
+static int
+aplic_setup(device_t dev)
{
struct aplic_irqsrc *isrcs;
struct aplic_softc *sc;
- struct intr_pic *pic;
const char *name;
phandle_t node, xref, iparent;
- pcell_t *cells, cell;
+ pcell_t cell;
int error = ENXIO;
u_int irq;
- int cpu, hartid, rid, i, nintr, idc;
+ int rid;
device_t rootdev;
sc = device_get_softc(dev);
@@ -351,10 +484,6 @@
return (error);
}
- /* Set APLIC in direct mode and enable all interrupts */
- aplic_write(sc, APLIC_DOMAIN_CFG,
- APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE);
-
/* Register the interrupt sources */
isrcs = sc->isrcs;
name = device_get_nameunit(sc->dev);
@@ -366,48 +495,7 @@
if (error != 0)
goto fail;
- aplic_write(sc, APLIC_SRC_CFG(irq),
- APLIC_SRC_CFG_SM_DETACHED);
- }
-
- nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
- sizeof(uint32_t), (void **)&cells);
- if (nintr <= 0) {
- device_printf(dev, "Could not read interrupts-extended\n");
- goto fail;
- }
-
- /* interrupts-extended is a list of phandles and interrupt types. */
- for (i = 0, idc = 0; i < nintr; i += 2, idc++) {
- /* Skip M-mode external interrupts */
- if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
- continue;
-
- /* Get the hart ID from the CLIC's phandle. */
- hartid = fdt_get_hartid(dev, OF_node_from_xref(cells[i]));
- if (hartid < 0) {
- OF_prop_free(cells);
- goto fail;
- }
-
- /* Get the corresponding cpuid. */
- cpu = riscv_hartid_to_cpu(hartid);
- if (cpu < 0) {
- device_printf(dev, "Invalid cpu for hart %d\n", hartid);
- OF_prop_free(cells);
- goto fail;
- }
-
- sc->hart_indices[cpu] = idc;
- }
- OF_prop_free(cells);
-
- /* Turn off the interrupt delivery for all CPUs within or out domain */
- CPU_FOREACH(cpu) {
- aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
- APLIC_IDC_IDELIVERY_DISABLE);
- aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu),
- APLIC_IDC_ITHRESHOLD_DISABLE);
+ aplic_write(sc, APLIC_SRC_CFG(irq), SRC_CFG_SM_DETACHED);
}
rootdev = intr_irq_root_device(INTR_ROOT_IRQ);
@@ -429,20 +517,19 @@
return (ENXIO);
}
+ if (sc->has_imsic)
+ error = aplic_setup_imsic_mode(dev);
+ else
+ error = aplic_setup_direct_mode(dev);
+ if (error != 0)
+ goto fail;
+
xref = OF_xref_from_node(node);
- pic = intr_pic_register(sc->dev, xref);
- if (pic == NULL) {
+ sc->pic = intr_pic_register(sc->dev, xref);
+ if (sc->pic == NULL) {
error = ENXIO;
goto fail;
}
-
- error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
- aplic_intr, NULL, sc, &sc->ih);
- if (error != 0) {
- device_printf(dev, "Unable to setup IRQ resource\n");
- return (ENXIO);
- }
-
fail:
return (error);
}
@@ -450,15 +537,15 @@
static int
aplic_attach(device_t dev)
{
+ struct aplic_softc *sc;
int rc;
- /* APLIC with IMSIC on hart is not supported */
- if (ofw_bus_has_prop(dev, "msi-parent")) {
- device_printf(dev, "APLIC with IMSIC is unsupported\n");
- return (ENXIO);
- }
+ sc = device_get_softc(dev);
- rc = aplic_setup_direct_mode(dev);
+ if (ofw_bus_has_prop(dev, "msi-parent"))
+ sc->has_imsic = true;
+
+ rc = aplic_setup(dev);
return (rc);
}
@@ -466,13 +553,44 @@
static void
aplic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
{
+
aplic_disable_intr(dev, isrc);
}
static void
aplic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
{
+ struct aplic_softc *sc;
+ struct aplic_irqsrc *src;
+
+ sc = device_get_softc(dev);
+ src = (struct aplic_irqsrc *)isrc;
+
aplic_enable_intr(dev, isrc);
+
+ if (!src->msi)
+ return;
+
+ /*
+ * The section "4.9.2 Special consideration for level-sensitive
+ * interrupt sources" of the RISC-V AIA specification says:
+ *
+ * A second option is for the interrupt service routine to write the
+ * APLIC’s source identity number for the interrupt to the domain’s
+ * setipnum register just before exiting. This will cause the
+ * interrupt’s pending bit to be set to one again if the source is
+ * still asserting an interrupt, but not if the source is not asserting
+ * an interrupt.
+ */
+
+ switch (src->type) {
+ case IRQ_TYPE_LEVEL_HIGH:
+ case IRQ_TYPE_LEVEL_LOW:
+ aplic_write(sc, APLIC_SETIPNUM_LE, src->irq);
+ break;
+ default:
+ break;
+ }
}
static int
@@ -481,13 +599,38 @@
{
struct aplic_irqsrc *src;
struct aplic_softc *sc;
+ uint32_t reg;
CPU_ZERO(&isrc->isrc_cpu);
sc = device_get_softc(dev);
src = (struct aplic_irqsrc *)isrc;
- aplic_write(sc, APLIC_SRC_CFG(src->irq), APLIC_SRC_CFG_SM_EDGE_RSE);
+ if (bootverbose)
+ device_printf(dev, "configuring irq %d type %x\n", src->irq,
+ src->type);
+
+ switch (src->type) {
+ case IRQ_TYPE_LEVEL_HIGH:
+ reg = SRC_CFG_SM_LVL_HI;
+ break;
+ case IRQ_TYPE_LEVEL_LOW:
+ reg = SRC_CFG_SM_LVL_LO;
+ break;
+ case IRQ_TYPE_EDGE_RISING:
+ reg = SRC_CFG_SM_EDGE_RSE;
+ break;
+ case IRQ_TYPE_EDGE_FALLING:
+ reg = SRC_CFG_SM_EDGE_FLL;
+ break;
+ case IRQ_TYPE_EDGE_BOTH:
+ reg = SRC_CFG_SM_EDGE_FLL | SRC_CFG_SM_EDGE_RSE;
+ break;
+ default:
+ panic("unknown level");
+ };
+
+ aplic_write(sc, APLIC_SRC_CFG(src->irq), reg);
/*
* In uniprocessor system, bind_intr will not be called.
@@ -506,6 +649,7 @@
struct aplic_softc *sc;
struct aplic_irqsrc *src;
uint32_t cpu, hartid;
+ uint32_t reg;
sc = device_get_softc(dev);
src = (struct aplic_irqsrc *)isrc;
@@ -517,9 +661,8 @@
cpu = aplic_irq_cpu = intr_irq_next_cpu(aplic_irq_cpu,
&all_cpus);
CPU_SETOF(cpu, &isrc->isrc_cpu);
- } else {
+ } else
cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
- }
hartid = riscv_cpu_to_hartid(cpu);
@@ -527,10 +670,28 @@
device_printf(dev, "Bind irq %d to cpu%d (hart %d)\n", src->irq,
cpu, hartid);
- aplic_write(sc, APLIC_TARGET(src->irq),
- APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO));
- aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
- APLIC_IDC_IDELIVERY_ENABLE);
+ if (sc->has_imsic) {
+ if (imsic_alloc_aplic_msi(sc->parent, cpu, src->irq,
+ &src->msi)) {
+ device_printf(dev, "MSI allocation for irq %d failed\n",
+ src->irq);
+ return (ENXIO);
+ }
+
+ if (bootverbose)
+ device_printf(dev, "Setting up IRQ %d on CPU %d "
+ "MSI ID %d\n", src->irq, cpu, src->msi);
+
+ reg = hartid << TARGET_HART_ID_S;
+ reg |= src->msi << TARGET_EIID_S;
+ aplic_write(sc, APLIC_TARGET(src->irq), reg);
+ } else {
+ reg = sc->hart_indices[cpu] << TARGET_HART_ID_S;
+ reg |= (APLIC_INTR_DEF_PRIO << TARGET_IPRIO_S) & TARGET_IPRIO_M;
+ aplic_write(sc, APLIC_TARGET(src->irq), reg);
+ aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), IDELIVERY_ENABLE);
+ }
+
aplic_enable_intr(dev, isrc);
return (0);
Index: sys/riscv/riscv/imsic.c
===================================================================
--- /dev/null
+++ sys/riscv/riscv/imsic.c
@@ -0,0 +1,808 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/module.h>
+#include <sys/proc.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+#include <sys/mutex.h>
+#include <sys/intr.h>
+
+#include <machine/bus.h>
+#include <machine/intr.h>
+#include <machine/imsic.h>
+#include <machine/smp.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "pic_if.h"
+#include "msi_if.h"
+
+#define IMSIC_MSI_PPN_SHIFT 12 /* Number of bits to shift for PPN. */
+#define IMSIC_PAGE_SHIFT 12 /* Interrupt file page shift. */
+
+#define IMSIC_EIDELIVERY 0x70
+#define EIDELIVERY_EN (1 << 0)
+#define IMSIC_EITHRESHOLD 0x72
+#define ITHRESHOLD_DIS 0
+#define IMSIC_EIP(n) (0x80 + ((n / 64) * 2))
+#define IMSIC_EIE(n) (0xC0 + ((n / 64) * 2))
+
+#define IMSIC_MAX_MSI 2048 /* Max MSI ID per core (per spec). */
+BITSET_DEFINE(msiset, IMSIC_MAX_MSI);
+#define IMSIC_FILL(p) BIT_FILL(IMSIC_MAX_MSI, p)
+#define IMSIC_ISEMPTY(p) BIT_EMPTY(IMSIC_MAX_MSI, p)
+#define IMSIC_FFS(p) BIT_FFS(IMSIC_MAX_MSI, p)
+#define IMSIC_SET(n, p) BIT_SET(IMSIC_MAX_MSI, n, p)
+#define IMSIC_CLR(n, p) BIT_CLR(IMSIC_MAX_MSI, n, p)
+#define IMSIC_LOCK(_sc, _cpu) mtx_lock_spin(&(_sc)->pcpu_data[_cpu].lock)
+#define IMSIC_UNLOCK(_sc, _cpu) mtx_unlock_spin(&(_sc)->pcpu_data[_cpu].lock)
+
+#define ISRC_TO_IRQ(_isrc) ((struct imsic_irq *)(_isrc))
+
+struct imsic_msi_config {
+ u_int lhxs; /* Low hart index shift */
+ u_int lhxw; /* Low hart index width */
+ u_int hhxs; /* High hart index shift */
+ u_int hhxw; /* High hart index width */
+ vm_paddr_t msi_base; /* Base address for PPN */
+ unsigned long msi_sz; /* Total MSI region size for all harts */
+};
+
+#ifdef SMP
+struct smp_action {
+ int irq;
+};
+#endif
+
+struct imsic_irq {
+ struct intr_irqsrc isrc;
+ int cpuid;
+ int msi;
+};
+
+struct imsic_pcpu {
+ struct mtx lock;
+ struct msiset allocated_msis;
+ struct imsic_irq *irqs[IMSIC_MAX_MSI];
+ uint16_t aplic_irq[IMSIC_MAX_MSI];
+};
+
+struct imsic_softc {
+ device_t dev;
+ struct intr_pic *pic;
+ struct resource *mem_res;
+ struct resource *irq_res;
+ struct imsic_pcpu *pcpu_data;
+ struct imsic_msi_config msi_config;
+ cpuset_t target_cpus;
+ int nr_parent_irqs;
+ int ndev;
+};
+
+static inline void
+imsic_csr_write(int reg, uint64_t val)
+{
+
+ csr_write(siselect, reg);
+ csr_write(sireg, val);
+}
+
+static inline uint64_t
+imsic_csr_read(int reg)
+{
+ uint64_t val;
+
+ csr_write(siselect, reg);
+ val = csr_read(sireg);
+
+ return (val);
+}
+
+static inline uint32_t
+riscv_cpu_to_hartid(int cpu)
+{
+
+ return (pcpu_find(cpu)->pc_hart);
+}
+
+static inline int
+riscv_hartid_to_cpu(uint32_t hartid)
+{
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ if (riscv_cpu_to_hartid(cpu) == hartid)
+ return (cpu);
+ }
+
+ return (-1);
+}
+
+static int
+fdt_get_hartid(device_t dev, phandle_t imsic, uint32_t *hartid)
+{
+
+ /* Check the interrupt controller layout. */
+ if (OF_searchencprop(imsic, "#interrupt-cells", hartid,
+ sizeof(*hartid)) == -1) {
+ device_printf(dev,
+ "Could not find #interrupt-cells for phandle %u\n", imsic);
+ return (-1);
+ }
+
+ /*
+ * The parent of the interrupt-controller is the CPU we are
+ * interested in, so search for its hart ID.
+ */
+ if (OF_searchencprop(OF_parent(imsic), "reg", (pcell_t *)hartid,
+ sizeof(*hartid)) == -1) {
+ device_printf(dev, "Could not find hartid\n");
+ return (-1);
+ }
+
+ return (0);
+}
+
+static int
+imsic_intr(void *arg)
+{
+ struct imsic_softc *sc;
+ struct trapframe *tf;
+ struct imsic_irq *irq;
+ uint16_t aplic_irq;
+ u_long msi;
+ int cpu;
+
+ cpu = PCPU_GET(cpuid);
+
+ sc = arg;
+
+ while ((msi = csr_swap(stopei, 0))) {
+ msi >>= 16;
+
+ aplic_irq = sc->pcpu_data[cpu].aplic_irq[msi];
+ if (aplic_irq) {
+ intr_child_irq_handler(sc->pic, aplic_irq);
+ continue;
+ }
+
+ /* This IRQ is handled by IMSIC. */
+ irq = sc->pcpu_data[cpu].irqs[msi];
+ tf = curthread->td_intr_frame;
+ if (intr_isrc_dispatch(&irq->isrc, tf) != 0) {
+ device_printf(sc->dev, "Stray irq %lu detected\n", msi);
+ return (FILTER_STRAY);
+ }
+ }
+
+ return (FILTER_HANDLED);
+}
+
+static inline void
+imsic_enable_interrupt(int irq)
+{
+ uint64_t reg;
+
+ reg = imsic_csr_read(IMSIC_EIE(irq));
+ reg |= (0x1UL << (irq % 64));
+ imsic_csr_write(IMSIC_EIE(irq), reg);
+}
+
+static inline void
+imsic_disable_interrupt(int irq)
+{
+ uint64_t reg;
+
+ reg = imsic_csr_read(IMSIC_EIE(irq));
+ reg &= ~(0x1UL << (irq % 64));
+ imsic_csr_write(IMSIC_EIE(irq), reg);
+}
+
+#ifdef SMP
+static void
+smp_imsic_enable_interrupt(void *arg)
+{
+ struct smp_action *act;
+
+ act = arg;
+
+ imsic_enable_interrupt(act->irq);
+}
+
+static void
+smp_imsic_disable_interrupt(void *arg)
+{
+ struct smp_action *act;
+
+ act = arg;
+
+ imsic_disable_interrupt(act->irq);
+}
+#endif
+
+static int
+imsic_enable_irq(device_t dev, int cpu, int irq)
+{
+#ifdef SMP
+ struct smp_action act;
+ cpuset_t cpus;
+#endif
+ if (cpu == PCPU_GET(cpuid)) {
+ imsic_enable_interrupt(irq);
+ } else {
+#ifdef SMP
+ CPU_ZERO(&cpus);
+ CPU_SET(cpu, &cpus);
+ act.irq = irq;
+ smp_rendezvous_cpus(cpus, NULL, smp_imsic_enable_interrupt,
+ NULL, &act);
+#endif
+ }
+
+ return (0);
+}
+
+static int
+imsic_disable_irq(device_t dev, int cpu, int irq)
+{
+#ifdef SMP
+ struct smp_action act;
+ cpuset_t cpus;
+#endif
+ if (cpu == PCPU_GET(cpuid)) {
+ imsic_disable_interrupt(irq);
+ } else {
+#ifdef SMP
+ CPU_ZERO(&cpus);
+ CPU_SET(cpu, &cpus);
+ act.irq = irq;
+ smp_rendezvous_cpus(cpus, NULL, smp_imsic_disable_interrupt,
+ NULL, &act);
+#endif
+ }
+
+ return (0);
+}
+
+static int
+imsic_nr_parent_irqs(device_t dev)
+{
+ struct imsic_softc *sc;
+ phandle_t node;
+ pcell_t *cells;
+ uint32_t hartid;
+ int error;
+ int nintr;
+ int cpu;
+ int rc, i;
+
+ rc = ENXIO;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ sc->nr_parent_irqs = 0;
+
+ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
+ sizeof(uint32_t), (void **)&cells);
+ if (nintr < 0) {
+ device_printf(dev, "interrupts-extended property not found.\n");
+ goto out;
+ }
+
+ CPU_ZERO(&sc->target_cpus);
+
+ for (i = 0; i < nintr; i += 2) {
+ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
+ continue;
+
+ error = fdt_get_hartid(dev, OF_node_from_xref(cells[i]),
+ &hartid);
+ if (error != 0) {
+ device_printf(dev, "Target hartid not found\n");
+ goto fail;
+ }
+
+ cpu = riscv_hartid_to_cpu(hartid);
+ if (cpu == -1) {
+ device_printf(dev, "Invalid cpu for hart %d\n",
+ hartid);
+ continue;
+ }
+
+ CPU_SET(cpu, &sc->target_cpus);
+ sc->nr_parent_irqs += 1;
+ }
+
+ rc = 0;
+
+fail:
+ OF_prop_free(cells);
+out:
+ return (rc);
+}
+
+static int
+imsic_parse_fdt_msi_config(device_t dev)
+{
+ struct imsic_softc *sc;
+ phandle_t node;
+ pcell_t *buf;
+ ssize_t len;
+ int error;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+
+ if ((len = OF_getencprop_alloc(node, "riscv,guest-index-bits",
+ (void **)&buf)) < 0)
+ sc->msi_config.lhxs = 0;
+ else
+ sc->msi_config.lhxs = buf[0];
+
+ OF_prop_free(buf);
+
+ if ((len = OF_getencprop_alloc(node, "riscv,hart-index-bits",
+ (void *)&buf)) < 0) {
+ sc->msi_config.lhxw = fls(sc->nr_parent_irqs);
+ if ((1UL << sc->msi_config.lhxw) < sc->nr_parent_irqs)
+ sc->msi_config.lhxw++;
+ } else
+ sc->msi_config.lhxw = buf[0];
+
+ OF_prop_free(buf);
+
+ if ((len = OF_getencprop_alloc(node, "riscv,group-index-bits",
+ (void *)&buf)) < 0)
+ sc->msi_config.hhxw = 0;
+ else
+ sc->msi_config.hhxw = buf[0];
+
+ OF_prop_free(buf);
+
+ if ((len = OF_getencprop_alloc(node, "riscv,group-index-shift",
+ (void *)&buf)) < 0)
+ sc->msi_config.hhxs = 2 * IMSIC_PAGE_SHIFT;
+ else
+ sc->msi_config.hhxs = buf[0];
+
+ OF_prop_free(buf);
+
+ if (sc->msi_config.hhxs < (2 * IMSIC_PAGE_SHIFT))
+ return (ENXIO);
+
+ /* Now find the MSI base. */
+ error = fdt_regsize(node, &sc->msi_config.msi_base,
+ &sc->msi_config.msi_sz);
+
+ return (error);
+}
+
+static int
+imsic_probe(device_t dev)
+{
+
+ if (!ofw_bus_status_okay(dev))
+ return (ENXIO);
+
+ if (!ofw_bus_is_compatible(dev, "riscv,imsics"))
+ return (ENXIO);
+
+ if (!ofw_bus_has_prop(dev, "msi-controller"))
+ return (ENXIO);
+
+ device_set_desc(dev, "Incoming MSI Controller");
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+imsic_attach(device_t dev)
+{
+ struct imsic_softc *sc;
+ phandle_t node;
+ phandle_t xref;
+ int i, irq, error, rid;
+ phandle_t iparent;
+ pcell_t cell;
+ device_t root_dev;
+ phandle_t root_node;
+
+ sc = device_get_softc(dev);
+ node = ofw_bus_get_node(dev);
+ sc->dev = dev;
+
+ sc->pcpu_data = mallocarray(mp_maxid + 1, sizeof(struct imsic_pcpu),
+ M_DEVBUF, M_WAITOK | M_ZERO);
+
+ if (imsic_nr_parent_irqs(dev)) {
+ device_printf(dev, "Could not parse DTS.\n");
+ return (ENXIO);
+ }
+
+ error = imsic_parse_fdt_msi_config(dev);
+ if (error) {
+ device_printf(dev, "Could not read config.\n");
+ return (ENXIO);
+ }
+
+ /*
+ * Enable interrupt delivery by interrupt file. APLIC will forward the
+ * interrupts to IMSIC as MSI writes.
+ */
+ imsic_csr_write(IMSIC_EIDELIVERY, EIDELIVERY_EN);
+
+ /* Receive all interrupts */
+ imsic_csr_write(IMSIC_EITHRESHOLD, ITHRESHOLD_DIS);
+
+ /* Per cpu MSI IDs. */
+ CPU_FOREACH(i) {
+ mtx_init(&sc->pcpu_data[i].lock, "imsic-pcpu-lock",
+ NULL, MTX_SPIN);
+ IMSIC_FILL(&sc->pcpu_data[i].allocated_msis);
+ /* MSI ID 0 is not a valid MSI. */
+ IMSIC_CLR(0, &sc->pcpu_data[i].allocated_msis);
+ }
+
+ xref = OF_xref_from_node(node);
+ sc->pic = intr_pic_register(dev, xref);
+ if (sc->pic == NULL) {
+ device_printf(dev, "PIC registration unsuccessful\n");
+ return (ENXIO);
+ }
+ intr_msi_register(dev, xref);
+
+ /* Find our parent interrupt controller. */
+ root_dev = device_get_parent(dev);
+ root_node = ofw_bus_get_node(root_dev);
+ iparent = OF_xref_from_node(root_node);
+ cell = IRQ_EXTERNAL_SUPERVISOR;
+ irq = ofw_bus_map_intr(dev, iparent, 1, &cell);
+ error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1);
+ if (error != 0) {
+ device_printf(dev, "Unable to register IRQ resource\n");
+ return (ENXIO);
+ }
+
+ rid = 0;
+ sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+ RF_ACTIVE | RF_SHAREABLE);
+ if (sc->irq_res == NULL) {
+ device_printf(dev, "Unable to alloc IRQ resource\n");
+ return (ENXIO);
+ }
+
+ /* Setup IRQ handler. */
+ error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE,
+ imsic_intr, NULL, sc, (void **)&sc);
+ if (error != 0) {
+ device_printf(dev, "Unable to setup IRQ resource\n");
+ return (ENXIO);
+ }
+
+ OF_device_register_xref(xref, dev);
+
+ return (0);
+}
+
+static int
+imsic_find_msi(device_t dev, int cpu, int *msi)
+{
+ struct imsic_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ IMSIC_LOCK(sc, cpu);
+ if (IMSIC_ISEMPTY(&sc->pcpu_data[cpu].allocated_msis)) {
+ /* No MSIs left to allocate. */
+ IMSIC_UNLOCK(sc, cpu);
+ return (ESRCH);
+ }
+ /* Find free MSI (first bit set). */
+ *msi = IMSIC_FFS(&sc->pcpu_data[cpu].allocated_msis) - 1;
+ /* Clear the bit to mark as allocated. */
+ IMSIC_CLR(*msi, &sc->pcpu_data[cpu].allocated_msis);
+ IMSIC_UNLOCK(sc, cpu);
+
+ return (0);
+}
+
+int
+imsic_alloc_aplic_msi(device_t dev, int cpu, int aplic_irq, int *msi)
+{
+ struct imsic_softc *sc;
+ int error;
+ int m;
+
+ sc = device_get_softc(dev);
+
+ error = imsic_find_msi(dev, cpu, &m);
+ if (error != 0)
+ return (error);
+
+ if (msi != NULL)
+ *msi = m;
+
+ sc->pcpu_data[cpu].aplic_irq[m] = aplic_irq;
+ imsic_enable_irq(dev, cpu, m);
+
+ return (0);
+}
+
+static int
+imsic_alloc_msi(device_t dev, int cpu, struct intr_irqsrc **isrc)
+{
+ struct imsic_softc *sc;
+ struct imsic_irq *irq;
+ const char *name;
+ int error;
+ int m;
+
+ sc = device_get_softc(dev);
+
+ error = imsic_find_msi(dev, cpu, &m);
+ if (error != 0)
+ return (error);
+
+ irq = sc->pcpu_data[cpu].irqs[m];
+ if (irq == NULL)
+ irq = malloc(sizeof(struct imsic_irq), M_DEVBUF,
+ M_ZERO | M_WAITOK);
+ irq->msi = m;
+ irq->cpuid = cpu;
+ sc->pcpu_data[cpu].irqs[m] = irq;
+
+ name = device_get_nameunit(dev);
+ error = intr_isrc_register(&irq->isrc, sc->dev, INTR_ISRCF_PPI,
+ "%s,%d", name, m);
+ if (error != 0)
+ return (error);
+
+ if (isrc != NULL)
+ *isrc = &irq->isrc;
+
+ return (0);
+}
+
+static void
+imsic_release_msi(device_t dev, struct imsic_irq *src)
+{
+ struct imsic_softc *sc;
+ struct imsic_irq *irq;
+ int error;
+ int cpu;
+ int m;
+
+ sc = device_get_softc(dev);
+ cpu = src->cpuid;
+ m = src->msi;
+
+ irq = sc->pcpu_data[cpu].irqs[m];
+
+ error = intr_isrc_deregister(&irq->isrc);
+ if (error) {
+ device_printf(dev, "could not deregister irq\n");
+ return;
+ }
+
+ free(irq, M_DEVBUF);
+ sc->pcpu_data[cpu].irqs[m] = NULL;
+
+ /* The MSI bit m is now available on this cpu. */
+ IMSIC_LOCK(sc, cpu);
+ IMSIC_SET(m, &sc->pcpu_data[cpu].allocated_msis);
+ IMSIC_UNLOCK(sc, cpu);
+}
+
+static void
+imsic_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct imsic_irq *src;
+
+ src = ISRC_TO_IRQ(isrc);
+
+ /* Disable the interrupt source */
+ imsic_disable_interrupt(src->msi);
+}
+
+static void
+imsic_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct imsic_irq *src;
+
+ src = ISRC_TO_IRQ(isrc);
+
+ /* Enable the interrupt source */
+ imsic_enable_interrupt(src->msi);
+}
+
+static void
+imsic_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct imsic_irq *src;
+
+ src = ISRC_TO_IRQ(isrc);
+
+ imsic_disable_irq(dev, src->cpuid, src->msi);
+}
+
+static void
+imsic_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct imsic_irq *src;
+
+ src = ISRC_TO_IRQ(isrc);
+
+ imsic_enable_irq(dev, src->cpuid, src->msi);
+}
+
+#ifdef SMP
+static void
+imsic_pic_init_secondary(device_t dev, uint32_t rootnum)
+{
+
+ /*
+ * Enable interrupt delivery by interrupt file. APLIC will forward the
+ * interrupts to IMSIC as MSI writes.
+ */
+ imsic_csr_write(IMSIC_EIDELIVERY, EIDELIVERY_EN);
+
+ /* Receive all interrupts */
+ imsic_csr_write(IMSIC_EITHRESHOLD, ITHRESHOLD_DIS);
+}
+#endif
+
+static int
+imsic_msi_alloc_msi(device_t dev, device_t child, int count, int maxcount,
+ device_t *pic, struct intr_irqsrc **isrc)
+{
+ struct imsic_irq *src;
+ int error;
+ int cpu;
+ int i;
+ int j;
+
+ cpu = PCPU_GET(cpuid);
+
+ for (i = 0; i < count; i++) {
+ error = imsic_alloc_msi(dev, cpu, &isrc[i]);
+ if (error) {
+ for (j = 0; j < i; j++) {
+ src = ISRC_TO_IRQ(isrc[j]);
+ imsic_release_msi(dev, src);
+ }
+ return (error);
+ }
+ }
+
+ return (0);
+}
+
+static int
+imsic_msi_release_msi(device_t dev, device_t child, int count,
+ struct intr_irqsrc **isrc)
+{
+ struct imsic_irq *src;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ src = ISRC_TO_IRQ(isrc[i]);
+ imsic_release_msi(dev, src);
+ }
+
+ return (0);
+}
+
+static int
+imsic_msi_alloc_msix(device_t dev, device_t child, device_t *pic,
+ struct intr_irqsrc **isrc)
+{
+ int error;
+ int cpu;
+
+ cpu = PCPU_GET(cpuid);
+
+ error = imsic_alloc_msi(dev, cpu, isrc);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+static int
+imsic_msi_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
+{
+ struct imsic_irq *src;
+
+ src = ISRC_TO_IRQ(isrc);
+
+ imsic_release_msi(dev, src);
+
+ return (0);
+}
+
+static int
+imsic_msi_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
+ uint64_t *addr, uint32_t *data)
+{
+ struct imsic_softc *sc;
+ struct imsic_irq *irq;
+ vm_paddr_t msi_addr;
+ uint32_t hartid;
+
+ sc = device_get_softc(dev);
+ irq = (struct imsic_irq *)isrc;
+ hartid = riscv_cpu_to_hartid(irq->cpuid);
+
+ msi_addr = (sc->msi_config.msi_base >> IMSIC_MSI_PPN_SHIFT);
+ msi_addr |= ((hartid & ((1 << sc->msi_config.lhxw) - 1))
+ << sc->msi_config.lhxs);
+ msi_addr |= (((hartid >> sc->msi_config.lhxw) &
+ ((1 << sc->msi_config.hhxw) - 1)) << sc->msi_config.hhxs);
+ msi_addr <<= IMSIC_MSI_PPN_SHIFT;
+
+ *addr = msi_addr;
+ *data = irq->msi;
+
+ return (0);
+}
+
+static device_method_t imsic_methods[] = {
+ DEVMETHOD(device_probe, imsic_probe),
+ DEVMETHOD(device_attach, imsic_attach),
+
+ DEVMETHOD(pic_disable_intr, imsic_pic_disable_intr),
+ DEVMETHOD(pic_enable_intr, imsic_pic_enable_intr),
+ DEVMETHOD(pic_pre_ithread, imsic_pic_pre_ithread),
+ DEVMETHOD(pic_post_ithread, imsic_pic_post_ithread),
+ DEVMETHOD(pic_post_filter, imsic_pic_post_ithread),
+#ifdef SMP
+ DEVMETHOD(pic_init_secondary, imsic_pic_init_secondary),
+#endif
+
+ /* MSI/MSI-X */
+ DEVMETHOD(msi_alloc_msi, imsic_msi_alloc_msi),
+ DEVMETHOD(msi_release_msi, imsic_msi_release_msi),
+ DEVMETHOD(msi_alloc_msix, imsic_msi_alloc_msix),
+ DEVMETHOD(msi_release_msix, imsic_msi_release_msix),
+ DEVMETHOD(msi_map_msi, imsic_msi_map_msi),
+
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(imsic, imsic_driver, imsic_methods, sizeof(struct imsic_softc));
+EARLY_DRIVER_MODULE(imsic, intc, imsic_driver, 0, 0,
+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY);

File Metadata

Mime Type
text/plain
Expires
Sun, Sep 27, 9:42 PM (2 h, 11 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39762583
Default Alt Text
D59695.diff (38 KB)

Event Timeline