Page MenuHomeFreeBSD
Files F171246880

D59259.id185371.diff
No OneTemporary
Actions

D59259.id185371.diff
View Options

diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -388,7 +388,9 @@
powerpc/powernv/platform_powernv.c optional powernv
powerpc/powernv/powernv_centaur.c optional powernv
powerpc/powernv/powernv_xscom.c optional powernv
-powerpc/powernv/xive.c optional powernv
+powerpc/powernv/xive.c optional powernv | pseries powerpc64 | pseries powerpc64le
+powerpc/powernv/xive_if.m optional powernv | pseries powerpc64 | pseries powerpc64le
+powerpc/powernv/xive_native.c optional powernv
powerpc/powerpc/altivec.c standard
powerpc/powerpc/autoconf.c standard
powerpc/powerpc/bus_machdep.c standard
@@ -461,6 +463,7 @@
powerpc/pseries/rtas_pci.c optional pseries pci
powerpc/pseries/vdevice.c optional pseries powerpc64 | pseries powerpc64le
powerpc/pseries/xics.c optional pseries powerpc64 | pseries powerpc64le
+powerpc/pseries/xive_phyp.c optional pseries powerpc64 | pseries powerpc64le
powerpc/psim/iobus.c optional psim
powerpc/psim/ata_iobus.c optional ata psim
powerpc/psim/openpic_iobus.c optional psim
diff --git a/sys/powerpc/powernv/platform_powernv.c b/sys/powerpc/powernv/platform_powernv.c
--- a/sys/powerpc/powernv/platform_powernv.c
+++ b/sys/powerpc/powernv/platform_powernv.c
@@ -58,8 +58,6 @@
extern void *ap_pcpu;
#endif
-void (*powernv_smp_ap_extra_init)(void);
-
static int powernv_probe(platform_t);
static int powernv_attach(platform_t);
void powernv_mem_regions(platform_t, struct mem_region *phys, int *physsz,
@@ -69,7 +67,6 @@
static int powernv_smp_first_cpu(platform_t, struct cpuref *cpuref);
static int powernv_smp_next_cpu(platform_t, struct cpuref *cpuref);
static int powernv_smp_get_bsp(platform_t, struct cpuref *cpuref);
-static void powernv_smp_ap_init(platform_t);
#ifdef SMP
static int powernv_smp_start_cpu(platform_t, struct pcpu *cpu);
static void powernv_smp_cpu_off(platform_t plat);
@@ -88,7 +85,6 @@
PLATFORMMETHOD(platform_numa_mem_regions, powernv_numa_mem_regions),
PLATFORMMETHOD(platform_timebase_freq, powernv_timebase_freq),
- PLATFORMMETHOD(platform_smp_ap_init, powernv_smp_ap_init),
PLATFORMMETHOD(platform_smp_first_cpu, powernv_smp_first_cpu),
PLATFORMMETHOD(platform_smp_next_cpu, powernv_smp_next_cpu),
PLATFORMMETHOD(platform_smp_get_bsp, powernv_smp_get_bsp),
@@ -571,14 +567,6 @@
opal_call(OPAL_CEC_REBOOT);
}
-static void
-powernv_smp_ap_init(platform_t platform)
-{
-
- if (powernv_smp_ap_extra_init != NULL)
- powernv_smp_ap_extra_init();
-}
-
static void
powernv_cpu_idle(sbintime_t sbt)
{
diff --git a/sys/powerpc/powernv/xive.h b/sys/powerpc/powernv/xive.h
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/powernv/xive.h
@@ -0,0 +1,158 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2019 Justin Hibbits
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ */
+
+#ifndef _POWERPC_POWERNV_XIVE_H_
+#define _POWERPC_POWERNV_XIVE_H_
+
+#include <sys/malloc.h>
+#include <sys/pcpu.h>
+#include <sys/_cpuset.h>
+#include <sys/_mutex.h>
+
+#include <vm/vm.h>
+
+struct resource;
+struct trapframe;
+
+#define XIVE_PRIORITY 7 /* Random non-zero number */
+#define XIVE_NPRIORITIES 8 /* Hardware supports priorities 0-7 */
+#define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */
+
+/* TIMA rings */
+#define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */
+#define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */
+#define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */
+
+/* Per-ring registers */
+#define XIVE_TM_NSR 0x00
+#define XIVE_TM_CPPR 0x01
+#define XIVE_TM_IPB 0x02
+#define XIVE_TM_LSMFB 0x03
+#define XIVE_TM_ACK_CNT 0x04
+#define XIVE_TM_INC 0x05
+#define XIVE_TM_AGE 0x06
+#define XIVE_TM_PIPR 0x07
+
+#define TM_WORD0 0x0
+#define TM_WORD2 0x8
+#define TM_QW2W2_VP 0x80000000
+
+/* Special-command offsets from the TIMA base. */
+#define XIVE_TM_SPC_ACK 0x800
+#define TM_QW3NSR_HE_SHIFT 14
+#define TM_QW3_NSR_HE_NONE 0
+#define TM_QW3_NSR_HE_POOL 1
+#define TM_QW3_NSR_HE_PHYS 2
+#define TM_QW3_NSR_HE_LSI 3
+#define XIVE_TM_SPC_PULL_POOL_CTX 0x828
+
+/* Offsets within an ESB page. */
+#define XIVE_IRQ_LOAD_EOI 0x000
+#define XIVE_IRQ_STORE_EOI 0x400
+#define XIVE_IRQ_PQ_00 0xc00
+#define XIVE_IRQ_PQ_01 0xd00
+#define XIVE_IRQ_PQ_10 0xe00
+
+#define XIVE_IRQ_VAL_P 0x02
+#define XIVE_IRQ_VAL_Q 0x01
+
+/* Backend-neutral xive_irq::flags. */
+#define XIVE_IRQ_FLAG_H_INT_ESB 0x00000010
+#define XIVE_IRQ_FLAG_SHIFT_BUG 0x00000008
+#define XIVE_IRQ_FLAG_LSI 0x00000004
+#define XIVE_IRQ_FLAG_STORE_EOI 0x00000002
+#define XIVE_IRQ_FLAG_TRIGGER_PAGE 0x00000001
+
+struct xive_softc {
+ struct mtx sc_mtx;
+ struct resource *sc_mem;
+ vm_size_t sc_prov_page_size;
+ uint32_t sc_offset;
+ uint32_t sc_eq_shift;
+ uint8_t sc_prio;
+};
+
+struct xive_queue {
+ uint32_t *q_page;
+ uint32_t *q_eoi_page;
+ uint32_t q_toggle;
+ uint32_t q_size;
+ uint32_t q_index;
+ uint32_t q_mask;
+};
+
+struct xive_irq {
+ uint32_t girq;
+ uint32_t lirq;
+ uint64_t vp;
+ uint64_t flags;
+ uint8_t prio;
+ bool masked;
+ vm_offset_t eoi_page;
+ vm_offset_t trig_page;
+ vm_size_t esb_size;
+ int chip;
+};
+
+struct xive_cpu {
+ uint64_t vp;
+ uint64_t flags;
+ struct xive_irq ipi_data;
+ struct xive_queue queue;
+ uint64_t cam;
+ uint32_t chip;
+};
+
+MALLOC_DECLARE(M_XIVE);
+DPCPU_DECLARE(struct xive_cpu, xive_cpu_data);
+
+/*
+ * Called by each backend's attach method after MMIO + per-CPU VP setup:
+ * allocates queue pages, sets up IPIs, registers the PIC, and runs
+ * ap_init on the BSP.
+ */
+int xive_core_attach(device_t dev);
+
+/* Populate an IRQ descriptor via XIVE_IRQ_POPULATE; map ESB pages. */
+int xive_init_irq(device_t dev, struct xive_irq *irqd, u_int lisn);
+
+/* MMIO helpers usable by backends (e.g., for pool-ring context ops). */
+uint16_t xive_read_2(struct xive_softc *sc, bus_size_t offset);
+void xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val);
+uint64_t xive_read_mmap8(vm_offset_t addr);
+void xive_write_mmap8(vm_offset_t addr, uint64_t val);
+
+/* PIC method implementations. */
+void xive_bind(device_t, u_int, cpuset_t, void **);
+void xive_dispatch(device_t, struct trapframe *);
+void xive_enable(device_t, u_int, u_int, void **);
+void xive_eoi(device_t, u_int, void *);
+void xive_ipi(device_t, u_int);
+void xive_mask(device_t, u_int, void *);
+void xive_unmask(device_t, u_int, void *);
+void xive_translate_code(device_t, u_int, int, enum intr_trigger *,
+ enum intr_polarity *);
+
+/* Shared per-CPU init: unmask this CPU's IPI and enable interrupts (CPPR). */
+void xive_ap_init_common(device_t);
+
+/*
+ * Base driver_t. xive_native and xive_phyp derive from this via
+ * DEFINE_CLASS_1 and inherit all shared PIC methods (including the
+ * default pic_ap_init = xive_ap_init_common; native overrides).
+ */
+DECLARE_CLASS(xive_driver);
+
+#endif /* _POWERPC_POWERNV_XIVE_H_ */
diff --git a/sys/powerpc/powernv/xive.c b/sys/powerpc/powernv/xive.c
--- a/sys/powerpc/powernv/xive.c
+++ b/sys/powerpc/powernv/xive.c
@@ -25,12 +25,34 @@
* SUCH DAMAGE.
*/
+/*
+ * XIVE Exploitation-mode driver core.
+ *
+ * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
+ * mode, and "Exploitation mode". XICS emulation mode is compatible with the
+ * POWER8 and earlier XICS interrupt controller. Exploitation mode gives us
+ * raw access to the XIVE MMIO for interrupt delivery, improving performance
+ * significantly.
+ *
+ * This file holds the backend-neutral core: PIC KOBJ methods, in-memory
+ * event queue handling, ESB-page MMIO for EOI/mask/unmask, and the shared
+ * attach helper. Two backends supply the source/queue setup differences:
+ *
+ * xive_native.c (powernv) - OPAL calls, hypervisor-privileged.
+ * xive_phyp.c (pseries) - PAPR H_INT_* hypercalls, guest OS.
+ *
+ * Each backend registers XIVE_* methods (see xive_if.m) on its driver
+ * and sets sc->sc_offset to the TIMA ring it operates on (QW3_HV on
+ * powernv, QW1_OS on pseries) before calling xive_core_attach().
+ *
+ * See xive.h for register definitions and struct layouts.
+ */
+
#include <sys/cdefs.h>
#include "opt_platform.h"
#include <sys/param.h>
#include <sys/systm.h>
-#include <sys/module.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/endian.h>
@@ -50,83 +72,25 @@
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
-#ifdef POWERNV
-#include <powerpc/powernv/opal.h>
-#endif
-
+#include "xive.h"
#include "pic_if.h"
+#include "xive_if.h"
-#define XIVE_PRIORITY 7 /* Random non-zero number */
-#define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */
-
-/* Registers */
-#define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */
-#define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */
-#define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */
-
-#define XIVE_TM_NSR 0x00
-#define XIVE_TM_CPPR 0x01
-#define XIVE_TM_IPB 0x02
-#define XIVE_TM_LSMFB 0x03
-#define XIVE_TM_ACK_CNT 0x04
-#define XIVE_TM_INC 0x05
-#define XIVE_TM_AGE 0x06
-#define XIVE_TM_PIPR 0x07
-
-#define TM_WORD0 0x0
-#define TM_WORD2 0x8
-#define TM_QW2W2_VP 0x80000000
-
-#define XIVE_TM_SPC_ACK 0x800
-#define TM_QW3NSR_HE_SHIFT 14
-#define TM_QW3_NSR_HE_NONE 0
-#define TM_QW3_NSR_HE_POOL 1
-#define TM_QW3_NSR_HE_PHYS 2
-#define TM_QW3_NSR_HE_LSI 3
-#define XIVE_TM_SPC_PULL_POOL_CTX 0x828
-
-#define XIVE_IRQ_LOAD_EOI 0x000
-#define XIVE_IRQ_STORE_EOI 0x400
-#define XIVE_IRQ_PQ_00 0xc00
-#define XIVE_IRQ_PQ_01 0xd00
-
-#define XIVE_IRQ_VAL_P 0x02
-#define XIVE_IRQ_VAL_Q 0x01
-
-struct xive_softc;
-struct xive_irq;
-
-extern void (*powernv_smp_ap_extra_init)(void);
-
-/* Private support */
-static void xive_setup_cpu(void);
-static void xive_smp_cpu_startup(void);
-static void xive_init_irq(struct xive_irq *irqd, u_int irq);
-static struct xive_irq *xive_configure_irq(u_int irq);
-static int xive_provision_page(struct xive_softc *sc);
-
-/* Interfaces */
-static int xive_probe(device_t);
-static int xive_attach(device_t);
-static int xics_probe(device_t);
-static int xics_attach(device_t);
-
-static void xive_bind(device_t, u_int, cpuset_t, void **);
-static void xive_dispatch(device_t, struct trapframe *);
-static void xive_enable(device_t, u_int, u_int, void **);
-static void xive_eoi(device_t, u_int, void *);
-static void xive_ipi(device_t, u_int);
-static void xive_mask(device_t, u_int, void *);
-static void xive_unmask(device_t, u_int, void *);
-static void xive_translate_code(device_t dev, u_int irq, int code,
- enum intr_trigger *trig, enum intr_polarity *pol);
-
-static device_method_t xive_methods[] = {
- /* Device interface */
- DEVMETHOD(device_probe, xive_probe),
- DEVMETHOD(device_attach, xive_attach),
-
- /* PIC interface */
+MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
+
+DPCPU_DEFINE(struct xive_cpu, xive_cpu_data);
+
+static int xive_ipi_vector = -1;
+
+static struct xive_irq *xive_configure_irq(device_t dev, u_int lisn);
+
+/*
+ * Base class holding backend-neutral PIC methods. Never attached
+ * directly; xive_native and xive_phyp derive from this via
+ * DEFINE_CLASS_1 in their respective files. pic_ap_init defaults to
+ * xive_ap_init_common; native overrides to add pool-ring context mgmt.
+ */
+static device_method_t xive_methods[] = {
DEVMETHOD(pic_bind, xive_bind),
DEVMETHOD(pic_dispatch, xive_dispatch),
DEVMETHOD(pic_enable, xive_enable),
@@ -135,312 +99,84 @@
DEVMETHOD(pic_mask, xive_mask),
DEVMETHOD(pic_unmask, xive_unmask),
DEVMETHOD(pic_translate_code, xive_translate_code),
-
- DEVMETHOD_END
-};
-
-static device_method_t xics_methods[] = {
- /* Device interface */
- DEVMETHOD(device_probe, xics_probe),
- DEVMETHOD(device_attach, xics_attach),
-
+ DEVMETHOD(pic_ap_init, xive_ap_init_common),
DEVMETHOD_END
};
-struct xive_softc {
- struct mtx sc_mtx;
- struct resource *sc_mem;
- vm_size_t sc_prov_page_size;
- uint32_t sc_offset;
-};
-
-struct xive_queue {
- uint32_t *q_page;
- uint32_t *q_eoi_page;
- uint32_t q_toggle;
- uint32_t q_size;
- uint32_t q_index;
- uint32_t q_mask;
-};
-
-struct xive_irq {
- uint32_t girq;
- uint32_t lirq;
- uint64_t vp;
- uint64_t flags;
-#define OPAL_XIVE_IRQ_SHIFT_BUG 0x00000008
-#define OPAL_XIVE_IRQ_LSI 0x00000004
-#define OPAL_XIVE_IRQ_STORE_EOI 0x00000002
-#define OPAL_XIVE_IRQ_TRIGGER_PAGE 0x00000001
- uint8_t prio;
- vm_offset_t eoi_page;
- vm_offset_t trig_page;
- vm_size_t esb_size;
- int chip;
-};
-
-struct xive_cpu {
- uint64_t vp;
- uint64_t flags;
- struct xive_irq ipi_data;
- struct xive_queue queue; /* We only use a single queue for now. */
- uint64_t cam;
- uint32_t chip;
-};
-
-static driver_t xive_driver = {
- "xive",
- xive_methods,
- sizeof(struct xive_softc)
-};
-
-static driver_t xics_driver = {
- "xivevc",
- xics_methods,
- 0
-};
-
-EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, 0, 0, BUS_PASS_INTERRUPT - 1);
-EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT);
-
-MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
-
-DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data);
-
-static int xive_ipi_vector = -1;
+DEFINE_CLASS_0(xive, xive_driver, xive_methods, sizeof(struct xive_softc));
/*
- * XIVE Exploitation mode driver.
- *
- * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
- * mode, and "Exploitation mode". XICS emulation mode is compatible with the
- * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate
- * hypervisor calls and memory accesses. Exploitation mode gives us raw access
- * to the XIVE MMIO, improving performance significantly.
- *
- * The XIVE controller is a very bizarre interrupt controller. It uses queues
- * in memory to pass interrupts around, and maps itself into 512GB of physical
- * device address space, giving each interrupt in the system one or more pages
- * of address space. An IRQ is tied to a virtual processor, which could be a
- * physical CPU thread, or a guest CPU thread (LPAR running on a physical
- * thread). Thus, the controller can route interrupts directly to guest OSes
- * bypassing processing by the hypervisor, thereby improving performance of the
- * guest OS.
- *
- * An IRQ, in addition to being tied to a virtual processor, has one or two
- * page mappings: an EOI page, and an optional trigger page. The trigger page
- * could be the same as the EOI page. Level-sensitive interrupts (LSIs) don't
- * have a trigger page, as they're external interrupts controlled by physical
- * lines. MSIs and IPIs have trigger pages. An IPI is really just another IRQ
- * in the XIVE, which is triggered by software.
- *
- * An interesting behavior of the XIVE controller is that oftentimes the
- * contents of an address location don't actually matter, but the direction of
- * the action is the signifier (read vs write), and the address is significant.
- * Hence, masking and unmasking an interrupt is done by reading different
- * addresses in the EOI page, and triggering an interrupt consists of writing to
- * the trigger page.
- *
- * Additionally, the MMIO region mapped is CPU-sensitive, just like the
- * per-processor register space (private access) in OpenPIC. In order for a CPU
- * to receive interrupts it must itself configure its CPPR (Current Processor
- * Priority Register), it cannot be set by any other processor. This
- * necessitates the xive_smp_cpu_startup() function.
- *
- * Queues are pages of memory, sized powers-of-two, that are shared with the
- * XIVE. The XIVE writes into the queue with an alternating polarity bit, which
- * flips when the queue wraps.
- */
-
-/*
- * Offset-based read/write interfaces.
+ * Offset-based read/write into the current TIMA ring.
*/
-static uint16_t
+uint16_t
xive_read_2(struct xive_softc *sc, bus_size_t offset)
{
return (bus_read_2(sc->sc_mem, sc->sc_offset + offset));
}
-static void
+void
xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val)
{
bus_write_1(sc->sc_mem, sc->sc_offset + offset, val);
}
-/* EOI and Trigger page access interfaces. */
-static uint64_t
+/* EOI and trigger page access. */
+uint64_t
xive_read_mmap8(vm_offset_t addr)
{
return (*(volatile uint64_t *)addr);
}
-static void
+void
xive_write_mmap8(vm_offset_t addr, uint64_t val)
{
*(uint64_t *)(addr) = val;
}
-/* Device interfaces. */
-static int
-xive_probe(device_t dev)
+/*
+ * ESB accessors. A source normally exposes its ESB as MMIO pages that we
+ * map at setup time, but PAPR may hand back a source with no MMIO at all
+ * (QEMU does this for every LSI), whose state bits are only reachable
+ * through the backend.
+ */
+static uint64_t
+xive_esb_read(device_t dev, struct xive_irq *irqd, uint32_t offset)
{
- if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
- return (ENXIO);
-
- device_set_desc(dev, "External Interrupt Virtualization Engine");
+ if (irqd->flags & XIVE_IRQ_FLAG_H_INT_ESB)
+ return (XIVE_ESB_OP(dev, irqd, offset, 0, false));
- /* Make sure we always win against the xicp driver. */
- return (BUS_PROBE_DEFAULT);
+ return (xive_read_mmap8(irqd->eoi_page + offset));
}
-static int
-xics_probe(device_t dev)
+static void
+xive_esb_write(device_t dev, struct xive_irq *irqd, uint32_t offset,
+ uint64_t val)
{
- if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
- return (ENXIO);
-
- device_set_desc(dev, "External Interrupt Virtualization Engine Root");
- return (BUS_PROBE_DEFAULT);
-}
-
-static int
-xive_attach(device_t dev)
-{
- struct xive_softc *sc = device_get_softc(dev);
- struct xive_cpu *xive_cpud;
- phandle_t phandle = ofw_bus_get_node(dev);
- int64_t vp_block;
- int error;
- int rid;
- int i, order;
- uint64_t vp_id;
- int64_t ipi_irq;
-
- opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
-
- error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
- (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
-
- rid = 1; /* Get the Hypervisor-level register set. */
- sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
- &rid, RF_ACTIVE);
- sc->sc_offset = XIVE_TM_QW3_HV;
-
- mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
-
- /* Workaround for qemu single-thread powernv */
- if (mp_maxid == 0)
- order = 1;
+ if (irqd->flags & XIVE_IRQ_FLAG_H_INT_ESB)
+ XIVE_ESB_OP(dev, irqd, offset, val, true);
else
- order = fls(mp_maxid + (mp_maxid - 1)) - 1;
-
- do {
- vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
- if (vp_block == OPAL_BUSY)
- DELAY(10);
- else if (vp_block == OPAL_XIVE_PROVISIONING)
- xive_provision_page(sc);
- else
- break;
- } while (1);
-
- if (vp_block < 0) {
- device_printf(dev,
- "Unable to allocate VP block. Opal error %d\n",
- (int)vp_block);
- bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
- return (ENXIO);
- }
-
- /*
- * Set up the VPs. Try to do as much as we can in attach, to lessen
- * what's needed at AP spawn time.
- */
- CPU_FOREACH(i) {
- vp_id = pcpu_find(i)->pc_hwref;
-
- xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
- xive_cpud->vp = vp_id + vp_block;
- opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
- vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
-
- xive_cpud->cam = be64toh(xive_cpud->cam);
- xive_cpud->chip = be64toh(xive_cpud->chip);
-
- /* Allocate the queue page and populate the queue state data. */
- xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE,
- M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
- xive_cpud->queue.q_size = 1 << PAGE_SHIFT;
- xive_cpud->queue.q_mask =
- ((xive_cpud->queue.q_size / sizeof(int)) - 1);
- xive_cpud->queue.q_toggle = 0;
- xive_cpud->queue.q_index = 0;
- do {
- error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
- OPAL_XIVE_VP_ENABLED, 0);
- } while (error == OPAL_BUSY);
- error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id,
- XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT,
- OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
-
- do {
- ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ,
- xive_cpud->chip);
- } while (ipi_irq == OPAL_BUSY);
-
- if (ipi_irq < 0)
- device_printf(root_pic,
- "Failed allocating IPI. OPAL error %d\n",
- (int)ipi_irq);
- else {
- xive_init_irq(&xive_cpud->ipi_data, ipi_irq);
- xive_cpud->ipi_data.vp = vp_id;
- xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
- opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq,
- xive_cpud->ipi_data.vp, XIVE_PRIORITY,
- MAX_XIVE_IRQS);
- }
- }
-
- powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
- 1 /* Number of IPIs */, FALSE);
- root_pic = dev;
-
- xive_setup_cpu();
- powernv_smp_ap_extra_init = xive_smp_cpu_startup;
-
- return (0);
-}
-
-static int
-xics_attach(device_t dev)
-{
- phandle_t phandle = ofw_bus_get_node(dev);
-
- /* The XIVE (root PIC) will handle all our interrupts */
- powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
- MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
-
- return (0);
+ xive_write_mmap8(irqd->eoi_page + offset, val);
}
/*
* PIC I/F methods.
*/
-static void
+void
xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
{
+ struct xive_softc *sc = device_get_softc(dev);
struct xive_irq *irqd;
- int cpu;
- int ncpus, i, error;
+ int cpu, ncpus, i, error;
if (*priv == NULL)
- *priv = xive_configure_irq(irq);
+ *priv = xive_configure_irq(dev, irq);
+ if (*priv == NULL)
+ return;
irqd = *priv;
@@ -462,13 +198,10 @@
ncpus++;
}
- opal_call(OPAL_XIVE_SYNC, OPAL_XIVE_SYNC_QUEUE, irq);
+ XIVE_SYNC(dev, irq);
- irqd->vp = pcpu_find(cpu)->pc_hwref;
- error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp,
- XIVE_PRIORITY, irqd->lirq);
-
- if (error < 0)
+ error = XIVE_IRQ_CONFIGURE(dev, irq, cpu, sc->sc_prio, irqd->lirq);
+ if (error != 0)
panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
xive_eoi(dev, irq, irqd);
@@ -492,7 +225,7 @@
return (i & 0x7fffffff);
}
-static void
+void
xive_dispatch(device_t dev, struct trapframe *tf)
{
struct xive_softc *sc;
@@ -541,11 +274,12 @@
xive_write_1(sc, XIVE_TM_CPPR, 0xff);
}
-static void
+void
xive_enable(device_t dev, u_int irq, u_int vector, void **priv)
{
+ struct xive_softc *sc = device_get_softc(dev);
struct xive_irq *irqd;
- cell_t status, cpu;
+ int cpu, error;
if (irq == MAX_XIVE_IRQS) {
if (xive_ipi_vector == -1)
@@ -553,30 +287,32 @@
return;
}
if (*priv == NULL)
- *priv = xive_configure_irq(irq);
+ *priv = xive_configure_irq(dev, irq);
+ if (*priv == NULL)
+ return;
irqd = *priv;
/* Bind to this CPU to start */
- cpu = PCPU_GET(hwref);
+ cpu = PCPU_GET(cpuid);
irqd->lirq = vector;
for (;;) {
- status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu,
- XIVE_PRIORITY, vector);
- if (status != OPAL_BUSY)
+ error = XIVE_IRQ_CONFIGURE(dev, irq, cpu, sc->sc_prio,
+ vector);
+ if (error != EBUSY)
break;
DELAY(10);
}
- if (status != 0)
- panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
- cpu, status);
+ if (error != 0)
+ panic("XIVE irq_configure %d -> cpu %d failed: %d", irq,
+ cpu, error);
xive_unmask(dev, irq, *priv);
}
-static void
+void
xive_eoi(device_t dev, u_int irq, void *priv)
{
struct xive_irq *rirq;
@@ -589,18 +325,27 @@
} else
rirq = priv;
- if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI)
- xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0);
- else if (rirq->flags & OPAL_XIVE_IRQ_LSI)
- xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI);
- else {
- eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
+ if (rirq->masked)
+ return;
+
+ if (rirq->flags & XIVE_IRQ_FLAG_STORE_EOI)
+ xive_esb_write(dev, rirq, XIVE_IRQ_STORE_EOI, 0);
+ else if (rirq->flags & XIVE_IRQ_FLAG_LSI) {
+ /*
+ * LSIs are re-triggered in hardware while still asserted, so
+ * the EOI cycle is all that is needed. That matters for an
+ * H_INT_ESB source, which has no trigger page to re-arm by
+ * hand and is always an LSI in practice.
+ */
+ xive_esb_read(dev, rirq, XIVE_IRQ_LOAD_EOI);
+ } else {
+ eoi_val = xive_esb_read(dev, rirq, XIVE_IRQ_PQ_00);
if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0)
xive_write_mmap8(rirq->trig_page, 0);
}
}
-static void
+void
xive_ipi(device_t dev, u_int cpu)
{
struct xive_cpu *xive_cpud;
@@ -612,7 +357,7 @@
xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0);
}
-static void
+void
xive_mask(device_t dev, u_int irq, void *priv)
{
struct xive_irq *rirq;
@@ -623,22 +368,28 @@
rirq = priv;
- if (!(rirq->flags & OPAL_XIVE_IRQ_LSI))
+ if (!(rirq->flags & XIVE_IRQ_FLAG_LSI))
return;
- xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01);
+ xive_esb_read(dev, rirq, XIVE_IRQ_PQ_10);
+ rirq->masked = true;
}
-static void
+void
xive_unmask(device_t dev, u_int irq, void *priv)
{
struct xive_irq *rirq;
rirq = priv;
- xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
+ if (rirq->masked) {
+ rirq->masked = false;
+ xive_esb_read(dev, rirq, XIVE_IRQ_LOAD_EOI);
+ return;
+ }
+ xive_esb_read(dev, rirq, XIVE_IRQ_PQ_00);
}
-static void
+void
xive_translate_code(device_t dev, u_int irq, int code,
enum intr_trigger *trig, enum intr_polarity *pol)
{
@@ -659,57 +410,50 @@
}
}
-/* Private functions. */
/*
- * Setup the current CPU. Called by the BSP at driver attachment, and by each
- * AP at wakeup (via xive_smp_cpu_startup()).
+ * Per-CPU init shared by both backends. Native's pic_ap_init wraps this
+ * with pool-ring context management first; phyp uses it directly.
*/
-static void
-xive_setup_cpu(void)
+void
+xive_ap_init_common(device_t dev)
{
- struct xive_softc *sc;
- struct xive_cpu *cpup;
- uint32_t val;
+ struct xive_softc *sc = device_get_softc(dev);
+ struct xive_cpu *cpup = DPCPU_PTR(xive_cpu_data);
- cpup = DPCPU_PTR(xive_cpu_data);
+ xive_unmask(dev, cpup->ipi_data.girq, &cpup->ipi_data);
+ xive_write_1(sc, XIVE_TM_CPPR, 0xff);
+}
- sc = device_get_softc(root_pic);
+/*
+ * Populate an IRQ structure and map its ESB pages into KVA.
+ * The backend's irq_populate leaves eoi_page/trig_page as physical
+ * addresses; we pmap_mapdev them into KVA before storing back.
+ */
+int
+xive_init_irq(device_t dev, struct xive_irq *irqd, u_int lisn)
+{
+ vm_paddr_t eoi_phys, trig_phys;
+ int error;
- val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
- if (val & TM_QW2W2_VP)
- bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
+ bzero(irqd, sizeof(*irqd));
- bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
- bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
- TM_QW2W2_VP | cpup->cam);
+ error = XIVE_IRQ_POPULATE(dev, irqd, lisn);
+ if (error != 0)
+ return (error);
+ irqd->girq = lisn;
- xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data);
- xive_write_1(sc, XIVE_TM_CPPR, 0xff);
-}
+ /* Nothing to map: the backend reaches this source's ESB for us. */
+ if (irqd->flags & XIVE_IRQ_FLAG_H_INT_ESB) {
+ irqd->eoi_page = 0;
+ irqd->trig_page = 0;
+ return (0);
+ }
-/* Populate an IRQ structure, mapping the EOI and trigger pages. */
-static void
-xive_init_irq(struct xive_irq *irqd, u_int irq)
-{
- uint64_t eoi_phys, trig_phys;
- uint32_t esb_shift;
-
- opal_call(OPAL_XIVE_GET_IRQ_INFO, irq,
- vtophys(&irqd->flags), vtophys(&eoi_phys),
- vtophys(&trig_phys), vtophys(&esb_shift),
- vtophys(&irqd->chip));
-
- irqd->flags = be64toh(irqd->flags);
- eoi_phys = be64toh(eoi_phys);
- trig_phys = be64toh(trig_phys);
- esb_shift = be32toh(esb_shift);
- irqd->chip = be32toh(irqd->chip);
-
- irqd->girq = irq;
- irqd->esb_size = 1 << esb_shift;
- irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
+ eoi_phys = (vm_paddr_t)irqd->eoi_page;
+ trig_phys = (vm_paddr_t)irqd->trig_page;
- if (eoi_phys == trig_phys)
+ irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
+ if (trig_phys == eoi_phys)
irqd->trig_page = irqd->eoi_page;
else if (trig_phys != 0)
irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys,
@@ -717,53 +461,96 @@
else
irqd->trig_page = 0;
- opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp),
- vtophys(&irqd->prio), vtophys(&irqd->lirq));
-
- irqd->vp = be64toh(irqd->vp);
- irqd->prio = be64toh(irqd->prio);
- irqd->lirq = be32toh(irqd->lirq);
+ return (0);
}
-/* Allocate an IRQ struct before populating it. */
+/* Allocate an IRQ struct and populate it. NULL if the source is unusable. */
static struct xive_irq *
-xive_configure_irq(u_int irq)
+xive_configure_irq(device_t dev, u_int lisn)
{
struct xive_irq *irqd;
- irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK);
-
- xive_init_irq(irqd, irq);
+ irqd = malloc(sizeof(*irqd), M_XIVE, M_WAITOK);
+ if (xive_init_irq(dev, irqd, lisn) != 0) {
+ free(irqd, M_XIVE);
+ return (NULL);
+ }
return (irqd);
}
/*
- * Part of the OPAL API. OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages,
- * provisioned through this call.
+ * Shared attach helper. Backend attach:
+ * - claims its DT node and MMIO resource,
+ * - resets the controller and does any per-CPU VP setup,
+ * - sets sc->sc_offset (TIMA ring),
+ * - calls xive_core_attach() which handles queue pages, IPI setup,
+ * powerpc_register_pic(), and running ap_init on the BSP.
+ * The backend's XIVE_* methods are dispatched via KOBJ from the driver.
*/
-static int
-xive_provision_page(struct xive_softc *sc)
+int
+xive_core_attach(device_t dev)
{
- void *prov_page;
- int error;
+ struct xive_softc *sc = device_get_softc(dev);
+ struct xive_cpu *xive_cpud;
+ phandle_t phandle = ofw_bus_get_node(dev);
+ vm_size_t qsize;
+ u_int ipi_lisn;
+ int error, i;
- do {
- prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
- 0, BUS_SPACE_MAXADDR,
- sc->sc_prov_page_size, sc->sc_prov_page_size);
+ mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
- error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
- vtophys(prov_page));
- } while (error == OPAL_XIVE_PROVISIONING);
+ CPU_FOREACH(i) {
+ xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
- return (0);
-}
+ /*
+ * Allocate the queue page and populate the queue state. The
+ * queue must be naturally aligned to its own size, which is
+ * not necessarily PAGE_SIZE: PAPR advertises the sizes it
+ * accepts, and KVM only takes 64K.
+ */
+ qsize = 1UL << sc->sc_eq_shift;
+ xive_cpud->queue.q_page = contigmalloc(qsize, M_XIVE,
+ M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, qsize, 0);
+ xive_cpud->queue.q_size = qsize;
+ xive_cpud->queue.q_mask =
+ ((xive_cpud->queue.q_size / sizeof(int)) - 1);
+ xive_cpud->queue.q_toggle = 0;
+ xive_cpud->queue.q_index = 0;
-/* The XIVE_TM_CPPR register must be set by each thread */
-static void
-xive_smp_cpu_startup(void)
-{
+ error = XIVE_QUEUE_CONFIG(dev, xive_cpud, i,
+ vtophys(xive_cpud->queue.q_page), sc->sc_eq_shift,
+ sc->sc_prio);
+ if (error != 0)
+ device_printf(dev,
+ "queue_config CPU %d failed: %d\n", i, error);
+
+ error = XIVE_IPI_ALLOC(dev, xive_cpud, i, &ipi_lisn);
+ if (error != 0) {
+ device_printf(dev,
+ "IPI alloc for CPU %d failed: %d\n", i, error);
+ continue;
+ }
+
+ error = xive_init_irq(dev, &xive_cpud->ipi_data, ipi_lisn);
+ if (error != 0) {
+ device_printf(dev,
+ "IPI init for CPU %d failed: %d\n", i, error);
+ continue;
+ }
+ xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
+ error = XIVE_IRQ_CONFIGURE(dev, ipi_lisn, i, sc->sc_prio,
+ MAX_XIVE_IRQS);
+ if (error != 0)
+ device_printf(dev,
+ "IPI configure for CPU %d failed: %d\n", i, error);
+ }
- xive_setup_cpu();
+ powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
+ 1 /* Number of IPIs */, FALSE);
+ root_pic = dev;
+
+ PIC_AP_INIT(dev);
+
+ return (0);
}
diff --git a/sys/powerpc/powernv/xive_if.m b/sys/powerpc/powernv/xive_if.m
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/powernv/xive_if.m
@@ -0,0 +1,90 @@
+#-
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright 2026 Justin Hibbits
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+
+CODE {
+#include <sys/bus.h>
+#include <sys/types.h>
+
+#include <vm/vm.h>
+
+struct xive_cpu;
+struct xive_irq;
+}
+
+# Backend contract for the shared XIVE core (xive.c). Two backends
+# implement this interface: xive_native (OPAL, powernv) and xive_phyp
+# (PAPR H_INT_*, pseries). Every call but esb_op runs on a cold path
+# (attach, IRQ setup, IRQ bind), and esb_op is only reached by sources
+# that already pay for a hypercall, so KOBJ dispatch is never the cost
+# that matters.
+#
+# Backends translate the FreeBSD cpuid to whatever target encoding
+# their transport requires. Return 0 on success, EBUSY when a retry
+# is warranted, or another errno-like value on failure.
+INTERFACE xive;
+
+# Bind a queue page to (cpu, prio) so the controller can post events.
+METHOD int queue_config {
+ device_t dev;
+ struct xive_cpu *cpud;
+ int cpu;
+ vm_paddr_t qpage;
+ int qshift;
+ uint8_t prio;
+};
+
+# Allocate a LISN suitable for use as this CPU's IPI.
+METHOD int ipi_alloc {
+ device_t dev;
+ struct xive_cpu *cpud;
+ int cpu;
+ u_int *lisn_out;
+};
+
+# Fill irqd from the controller: flags, EOI/trigger page phys addrs,
+# esb_shift, chip, current target/prio/sw_irq.
+METHOD int irq_populate {
+ device_t dev;
+ struct xive_irq *irqd;
+ u_int lisn;
+};
+
+# Route lisn to (cpu, prio) with the given sw_irq (logical) tag.
+METHOD int irq_configure {
+ device_t dev;
+ u_int lisn;
+ int cpu;
+ uint8_t prio;
+ uint32_t sw_irq;
+};
+
+# Wait for pending events on lisn to drain.
+METHOD int sync {
+ device_t dev;
+ u_int lisn;
+};
+
+# Load from or store to a source's ESB, for sources that expose no ESB
+# MMIO and must be driven by the transport instead. Only called for irqs
+# flagged XIVE_IRQ_FLAG_H_INT_ESB, so a backend that never sets that flag
+# need not implement this. Returns the loaded value, 0 for a store, or
+# ~0 if the operation failed.
+METHOD uint64_t esb_op {
+ device_t dev;
+ struct xive_irq *irqd;
+ uint32_t offset;
+ uint64_t data;
+ bool store;
+};
diff --git a/sys/powerpc/powernv/xive_native.c b/sys/powerpc/powernv/xive_native.c
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/powernv/xive_native.c
@@ -0,0 +1,351 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2019 Justin Hibbits
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * XIVE native (OPAL) backend for POWERNV.
+ *
+ * Attaches to "ibm,opal-xive-pe" (presenter) and "ibm,opal-xive-vc"
+ * (virtualization controller root). Provides the backend ops that
+ * the shared XIVE core in xive.c calls; supplies pool-ring context
+ * management in its own pic_ap_init.
+ */
+
+#include <sys/cdefs.h>
+#include "opt_platform.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/intr_machdep.h>
+#include <machine/md_var.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <powerpc/powernv/opal.h>
+
+#include "xive.h"
+#include "pic_if.h"
+#include "xive_if.h"
+
+static int xive_native_probe(device_t);
+static int xive_native_attach(device_t);
+static int xive_native_vc_probe(device_t);
+static int xive_native_vc_attach(device_t);
+static void xive_native_ap_init(device_t);
+static int xive_native_provision_page(struct xive_softc *sc);
+
+static int xive_native_queue_config(device_t dev, struct xive_cpu *cpud,
+ int cpu, vm_paddr_t qpage, int qshift, uint8_t prio);
+static int xive_native_ipi_alloc(device_t dev, struct xive_cpu *cpud,
+ int cpu, u_int *lisn_out);
+static int xive_native_irq_populate(device_t dev, struct xive_irq *irqd,
+ u_int lisn);
+static int xive_native_irq_configure(device_t dev, u_int lisn, int cpu,
+ uint8_t prio, uint32_t sw_irq);
+static int xive_native_sync(device_t dev, u_int lisn);
+
+static device_method_t xive_native_methods[] = {
+ DEVMETHOD(device_probe, xive_native_probe),
+ DEVMETHOD(device_attach, xive_native_attach),
+
+ /* Override the base's default pic_ap_init for pool-ring context. */
+ DEVMETHOD(pic_ap_init, xive_native_ap_init),
+
+ DEVMETHOD(xive_queue_config, xive_native_queue_config),
+ DEVMETHOD(xive_ipi_alloc, xive_native_ipi_alloc),
+ DEVMETHOD(xive_irq_populate, xive_native_irq_populate),
+ DEVMETHOD(xive_irq_configure, xive_native_irq_configure),
+ DEVMETHOD(xive_sync, xive_native_sync),
+
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(xive, xive_native_driver, xive_native_methods,
+ sizeof(struct xive_softc), xive_driver);
+
+static device_method_t xive_native_vc_methods[] = {
+ DEVMETHOD(device_probe, xive_native_vc_probe),
+ DEVMETHOD(device_attach, xive_native_vc_attach),
+
+ DEVMETHOD_END
+};
+
+static driver_t xive_native_vc_driver = {
+ "xivevc",
+ xive_native_vc_methods,
+ 0
+};
+
+EARLY_DRIVER_MODULE(xive, ofwbus, xive_native_driver, 0, 0,
+ BUS_PASS_INTERRUPT - 1);
+EARLY_DRIVER_MODULE(xivevc, ofwbus, xive_native_vc_driver, 0, 0,
+ BUS_PASS_INTERRUPT);
+
+static int
+xive_native_probe(device_t dev)
+{
+
+ if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
+ return (ENXIO);
+
+ device_set_desc(dev, "External Interrupt Virtualization Engine");
+
+ /* Make sure we always win against the xicp driver. */
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+xive_native_vc_probe(device_t dev)
+{
+
+ if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
+ return (ENXIO);
+
+ device_set_desc(dev, "External Interrupt Virtualization Engine Root");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+xive_native_attach(device_t dev)
+{
+ struct xive_softc *sc = device_get_softc(dev);
+ struct xive_cpu *xive_cpud;
+ phandle_t phandle = ofw_bus_get_node(dev);
+ int64_t vp_block;
+ int error, rid, i, order;
+ uint64_t vp_id;
+
+ opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
+
+ error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
+ (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
+ if (error < 0)
+ sc->sc_prov_page_size = 0;
+
+ rid = 1; /* Get the Hypervisor-level register set. */
+ sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+ RF_ACTIVE);
+ if (sc->sc_mem == NULL) {
+ device_printf(dev, "Unable to allocate HV MMIO resource\n");
+ return (ENXIO);
+ }
+ sc->sc_offset = XIVE_TM_QW3_HV;
+ sc->sc_prio = XIVE_PRIORITY;
+ sc->sc_eq_shift = PAGE_SHIFT;
+
+ /* Workaround for qemu single-thread powernv */
+ if (mp_maxid == 0)
+ order = 1;
+ else
+ order = fls(mp_maxid + (mp_maxid - 1)) - 1;
+
+ for (;;) {
+ vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
+ if (vp_block == OPAL_BUSY)
+ DELAY(10);
+ else if (vp_block == OPAL_XIVE_PROVISIONING)
+ xive_native_provision_page(sc);
+ else
+ break;
+ }
+
+ if (vp_block < 0) {
+ device_printf(dev,
+ "Unable to allocate VP block. Opal error %d\n",
+ (int)vp_block);
+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
+ return (ENXIO);
+ }
+
+ /* Per-CPU VP setup. Queue setup happens in the shared core. */
+ CPU_FOREACH(i) {
+ vp_id = pcpu_find(i)->pc_hwref;
+ xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
+ xive_cpud->vp = vp_id + vp_block;
+
+ opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
+ vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
+ xive_cpud->cam = be64toh(xive_cpud->cam);
+ xive_cpud->chip = be64toh(xive_cpud->chip);
+
+ do {
+ error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
+ OPAL_XIVE_VP_ENABLED, 0);
+ } while (error == OPAL_BUSY);
+ }
+
+ return (xive_core_attach(dev));
+}
+
+static int
+xive_native_vc_attach(device_t dev)
+{
+ phandle_t phandle = ofw_bus_get_node(dev);
+
+ /* The XIVE (root PIC) will handle all our interrupts. */
+ powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
+ MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
+
+ return (0);
+}
+
+/*
+ * HV-privileged pic_ap_init: load our own pool VP context before enabling
+ * the CPPR. Runs on the BSP at end of xive_core_attach and on every AP
+ * from mp_machdep.c via PIC_AP_INIT(root_pic).
+ */
+static void
+xive_native_ap_init(device_t dev)
+{
+ struct xive_softc *sc = device_get_softc(dev);
+ struct xive_cpu *cpup = DPCPU_PTR(xive_cpu_data);
+ uint32_t val;
+
+ val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
+ if (val & TM_QW2W2_VP)
+ bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
+
+ bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
+ bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
+ TM_QW2W2_VP | cpup->cam);
+
+ xive_ap_init_common(dev);
+}
+
+/*
+ * XIVE KOBJ methods.
+ */
+static int
+xive_native_queue_config(device_t dev __unused, struct xive_cpu *cpud __unused,
+ int cpu, vm_paddr_t qpage, int qshift, uint8_t prio)
+{
+ uint64_t hwref = pcpu_find(cpu)->pc_hwref;
+ int64_t rc;
+
+ rc = opal_call(OPAL_XIVE_SET_QUEUE_INFO, hwref, prio, qpage, qshift,
+ OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
+ return (rc == 0 ? 0 : EIO);
+}
+
+static int
+xive_native_ipi_alloc(device_t dev __unused, struct xive_cpu *cpud,
+ int cpu __unused, u_int *lisn_out)
+{
+ int64_t ipi;
+
+ do {
+ ipi = opal_call(OPAL_XIVE_ALLOCATE_IRQ, cpud->chip);
+ } while (ipi == OPAL_BUSY);
+
+ if (ipi < 0)
+ return (EIO);
+ *lisn_out = (u_int)ipi;
+ return (0);
+}
+
+static int
+xive_native_irq_populate(device_t dev __unused, struct xive_irq *irqd,
+ u_int lisn)
+{
+ uint64_t flags, eoi_phys, trig_phys, vp;
+ uint32_t esb_shift, chip, lirq;
+ uint8_t prio;
+
+ opal_call(OPAL_XIVE_GET_IRQ_INFO, lisn,
+ vtophys(&flags), vtophys(&eoi_phys), vtophys(&trig_phys),
+ vtophys(&esb_shift), vtophys(&chip));
+
+ irqd->flags = be64toh(flags); /* backend flags == core flags */
+ irqd->eoi_page = (vm_offset_t)be64toh(eoi_phys);
+ irqd->trig_page = (vm_offset_t)be64toh(trig_phys);
+ irqd->esb_size = 1U << be32toh(esb_shift);
+ irqd->chip = be32toh(chip);
+
+ opal_call(OPAL_XIVE_GET_IRQ_CONFIG, lisn, vtophys(&vp),
+ vtophys(&prio), vtophys(&lirq));
+ irqd->vp = be64toh(vp);
+ irqd->prio = be64toh(prio);
+ irqd->lirq = be32toh(lirq);
+
+ return (0);
+}
+
+static int
+xive_native_irq_configure(device_t dev __unused, u_int lisn, int cpu,
+ uint8_t prio, uint32_t sw_irq)
+{
+ uint64_t hwref = pcpu_find(cpu)->pc_hwref;
+ int64_t rc;
+
+ rc = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, lisn, hwref, prio, sw_irq);
+ if (rc == OPAL_BUSY)
+ return (EBUSY);
+ return (rc == 0 ? 0 : EIO);
+}
+
+static int
+xive_native_sync(device_t dev __unused, u_int lisn)
+{
+
+ opal_call(OPAL_XIVE_SYNC, OPAL_XIVE_SYNC_QUEUE, lisn);
+ return (0);
+}
+
+/*
+ * OPAL_XIVE_ALLOCATE_VP_BLOCK may need pages of memory donated to satisfy
+ * larger block orders. Called from the attach loop on OPAL_XIVE_PROVISIONING.
+ */
+static int
+xive_native_provision_page(struct xive_softc *sc)
+{
+ void *prov_page;
+ int error;
+
+ do {
+ prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
+ 0, BUS_SPACE_MAXADDR,
+ sc->sc_prov_page_size, sc->sc_prov_page_size);
+
+ error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
+ vtophys(prov_page));
+ } while (error == OPAL_XIVE_PROVISIONING);
+
+ return (0);
+}
diff --git a/sys/powerpc/pseries/phyp-hvcall.h b/sys/powerpc/pseries/phyp-hvcall.h
--- a/sys/powerpc/pseries/phyp-hvcall.h
+++ b/sys/powerpc/pseries/phyp-hvcall.h
@@ -331,9 +331,23 @@
#define H_GET_DMA_XLATES_L 0x324
/* Reserved ... */
#define H_REGISTER_PROC_TBL 0x37c
-#define MAX_HCALL_OPCODE H_REGISTER_PROC_TBL
+/* PAPR XIVE hypercalls (PAPR 2.9+). */
+#define H_INT_GET_SOURCE_INFO 0x3a8
+#define H_INT_SET_SOURCE_CONFIG 0x3ac
+#define H_INT_GET_SOURCE_CONFIG 0x3b0
+#define H_INT_GET_QUEUE_INFO 0x3b4
+#define H_INT_SET_QUEUE_CONFIG 0x3b8
+#define H_INT_GET_QUEUE_CONFIG 0x3bc
+#define H_INT_SET_OS_REPORTING_LINE 0x3c0
+#define H_INT_GET_OS_REPORTING_LINE 0x3c4
+#define H_INT_ESB 0x3c8
+#define H_INT_SYNC 0x3cc
+#define H_INT_RESET 0x3d0
+#define MAX_HCALL_OPCODE H_INT_RESET
int64_t phyp_hcall(uint64_t opcode, ...);
+int64_t phyp_hcall4(uint64_t opcode, uint64_t a1, uint64_t a2, uint64_t a3,
+ uint64_t a4, uint64_t a5, uint64_t a6, uint64_t *out);
int64_t phyp_pft_hcall(uint64_t opcode, uint64_t flags, uint64_t pteidx,
uint64_t pte_hi, uint64_t pte_lo, uint64_t *pteidx_out, uint64_t *ptelo_out,
uint64_t *r6);
diff --git a/sys/powerpc/pseries/phyp-hvcall.S b/sys/powerpc/pseries/phyp-hvcall.S
--- a/sys/powerpc/pseries/phyp-hvcall.S
+++ b/sys/powerpc/pseries/phyp-hvcall.S
@@ -47,6 +47,33 @@
blr /* return r3 = status */
ASEND(phyp_hcall)
+/*
+ * HV call returning four output values in R4-R7.
+ *
+ * C signature:
+ * int64_t phyp_hcall4(uint64_t opcode, uint64_t a1, uint64_t a2,
+ * uint64_t a3, uint64_t a4, uint64_t a5, uint64_t a6, uint64_t *out);
+ *
+ * On entry the caller places the output pointer in r10 (the 8th arg).
+ * We stash it, invoke the hypervisor, then write r4..r7 into *out.
+ */
+ASENTRY(phyp_hcall4)
+ mflr %r0
+ std %r0,16(%r1)
+ stdu %r1,-64(%r1)
+ std %r10,48(%r1) /* save output pointer */
+ hc /* invoke the hypervisor */
+ ld %r10,48(%r1) /* restore output pointer */
+ std %r4,0(%r10)
+ std %r5,8(%r10)
+ std %r6,16(%r10)
+ std %r7,24(%r10)
+ ld %r1,0(%r1)
+ ld %r0,16(%r1)
+ mtlr %r0
+ blr /* return r3 = status */
+ASEND(phyp_hcall4)
+
/*
* PFT HV calls take a special ABI (see PAPR 14.5.4.1)
*
diff --git a/sys/powerpc/pseries/xive_phyp.c b/sys/powerpc/pseries/xive_phyp.c
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/pseries/xive_phyp.c
@@ -0,0 +1,434 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2026 Justin Hibbits
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * XIVE PAPR (phyp) backend for pSeries.
+ *
+ * Attaches to "ibm,power-ivpe" and uses PAPR H_INT_* hypercalls to drive
+ * the XIVE from a KVM/PowerVM guest. Runs in XIVE-native mode: TIMA and
+ * ESB pages are mapped directly into the guest, only source/queue setup
+ * goes through hypercalls.
+ *
+ * IPI LISNs are drawn from the pool published in "ibm,xive-lisn-ranges".
+ */
+
+#include <sys/cdefs.h>
+#include "opt_platform.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/intr_machdep.h>
+#include <machine/md_var.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <powerpc/powernv/xive.h>
+#include <powerpc/pseries/phyp-hvcall.h>
+
+#include "pic_if.h"
+#include "xive_if.h"
+
+/*
+ * PAPR numbers hypercall flag bits MSB-first, so bit 0 is the sign bit.
+ * The hypervisor rejects any flag word with a bit set outside the small
+ * set it knows about, so these must be exact.
+ */
+#define XIVE_PAPR_BIT(b) (1ULL << (63 - (b)))
+
+/* PAPR XIVE source flags (returned by H_INT_GET_SOURCE_INFO). */
+#define XIVE_SRC_H_INT_ESB XIVE_PAPR_BIT(60)
+#define XIVE_SRC_LSI XIVE_PAPR_BIT(61)
+#define XIVE_SRC_TRIGGER XIVE_PAPR_BIT(62)
+#define XIVE_SRC_STORE_EOI XIVE_PAPR_BIT(63)
+
+/*
+ * H_INT_SET_QUEUE_CONFIG flags. There is no "enable" bit: a non-zero
+ * queue shift enables the queue, a zero shift resets it.
+ */
+#define XIVE_EQ_ALWAYS_NOTIFY XIVE_PAPR_BIT(63)
+
+/* H_INT_SET_SOURCE_CONFIG flags. */
+#define XIVE_SRC_SET_EISN XIVE_PAPR_BIT(62)
+#define XIVE_SRC_MASK XIVE_PAPR_BIT(63)
+
+/* H_INT_ESB flags. */
+#define XIVE_ESB_STORE XIVE_PAPR_BIT(63)
+
+struct xive_phyp_softc {
+ struct xive_softc xps_core; /* Must be first for softc alias. */
+ uint32_t xps_ipi_base;
+ uint32_t xps_ipi_count;
+ uint32_t xps_ipi_next; /* bump allocator */
+};
+
+static int xive_phyp_probe(device_t);
+static int xive_phyp_attach(device_t);
+
+static int xive_phyp_queue_config(device_t dev, struct xive_cpu *cpud,
+ int cpu, vm_paddr_t qpage, int qshift, uint8_t prio);
+static int xive_phyp_ipi_alloc(device_t dev, struct xive_cpu *cpud,
+ int cpu, u_int *lisn_out);
+static int xive_phyp_irq_populate(device_t dev, struct xive_irq *irqd,
+ u_int lisn);
+static int xive_phyp_irq_configure(device_t dev, u_int lisn, int cpu,
+ uint8_t prio, uint32_t sw_irq);
+static int xive_phyp_sync(device_t dev, u_int lisn);
+static uint64_t xive_phyp_esb_op(device_t dev, struct xive_irq *irqd,
+ uint32_t offset, uint64_t data, bool store);
+
+static device_method_t xive_phyp_methods[] = {
+ DEVMETHOD(device_probe, xive_phyp_probe),
+ DEVMETHOD(device_attach, xive_phyp_attach),
+
+ DEVMETHOD(xive_queue_config, xive_phyp_queue_config),
+ DEVMETHOD(xive_ipi_alloc, xive_phyp_ipi_alloc),
+ DEVMETHOD(xive_irq_populate, xive_phyp_irq_populate),
+ DEVMETHOD(xive_irq_configure, xive_phyp_irq_configure),
+ DEVMETHOD(xive_sync, xive_phyp_sync),
+ DEVMETHOD(xive_esb_op, xive_phyp_esb_op),
+
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_1(xive, xive_phyp_driver, xive_phyp_methods,
+ sizeof(struct xive_phyp_softc), xive_driver);
+
+EARLY_DRIVER_MODULE(xive_phyp, ofwbus, xive_phyp_driver, 0, 0,
+ BUS_PASS_INTERRUPT - 1);
+
+static int
+xive_phyp_probe(device_t dev)
+{
+
+ if (!ofw_bus_is_compatible(dev, "ibm,power-ivpe"))
+ return (ENXIO);
+
+ device_set_desc(dev, "PAPR External Interrupt Virtualization Engine");
+
+ /* Win against xicp when both bindings are present in the DT. */
+ return (BUS_PROBE_DEFAULT);
+}
+
+/*
+ * Pick the least urgent priority the hypervisor has not reserved for itself.
+ * The root node's "ibm,plat-res-int-priorities" lists the reserved ranges as
+ * <start, count> pairs; QEMU reserves 7 and up, so a guest using
+ * XIVE_PRIORITY gets H_P4 back from every H_INT_SET_*_CONFIG.
+ */
+static int
+xive_phyp_pick_priority(device_t dev, uint8_t *priop)
+{
+ pcell_t *ranges;
+ ssize_t nelems;
+ int found, i, prio;
+
+ nelems = OF_getencprop_alloc_multi(OF_finddevice("/"),
+ "ibm,plat-res-int-priorities", sizeof(pcell_t), (void **)&ranges);
+ if (nelems <= 0 || (nelems % 2) != 0) {
+ device_printf(dev,
+ "Missing or malformed ibm,plat-res-int-priorities\n");
+ if (nelems > 0)
+ OF_prop_free(ranges);
+ return (ENXIO);
+ }
+
+ found = -1;
+ for (prio = 0; prio < XIVE_NPRIORITIES; prio++) {
+ for (i = 0; i < nelems; i += 2) {
+ if ((uint32_t)prio >= ranges[i] &&
+ (uint32_t)prio - ranges[i] < ranges[i + 1])
+ break;
+ }
+ if (i == nelems)
+ found = prio;
+ }
+ OF_prop_free(ranges);
+
+ if (found < 0) {
+ device_printf(dev, "All interrupt priorities are reserved\n");
+ return (ENXIO);
+ }
+
+ *priop = found;
+ return (0);
+}
+
+/*
+ * Choose an event queue size from "ibm,xive-eq-sizes", a list of the page
+ * shifts the hypervisor accepts. PAGE_SIZE is not a safe assumption: KVM
+ * only takes 64K queues, and rejects anything else with H_HARDWARE.
+ */
+static int
+xive_phyp_pick_eq_shift(device_t dev, phandle_t node, uint32_t *shiftp)
+{
+ pcell_t *sizes;
+ ssize_t nelems;
+ uint32_t best;
+ int i;
+
+ nelems = OF_getencprop_alloc_multi(node, "ibm,xive-eq-sizes",
+ sizeof(pcell_t), (void **)&sizes);
+ if (nelems <= 0) {
+ device_printf(dev, "Missing ibm,xive-eq-sizes property\n");
+ return (ENXIO);
+ }
+
+ /* Smallest advertised queue that still holds a full page. */
+ best = 0;
+ for (i = 0; i < nelems; i++) {
+ if (sizes[i] < PAGE_SHIFT)
+ continue;
+ if (best == 0 || sizes[i] < best)
+ best = sizes[i];
+ }
+ OF_prop_free(sizes);
+
+ if (best == 0) {
+ device_printf(dev, "No usable ibm,xive-eq-sizes entry\n");
+ return (ENXIO);
+ }
+
+ *shiftp = best;
+ return (0);
+}
+
+static int
+xive_phyp_attach(device_t dev)
+{
+ struct xive_phyp_softc *psc = device_get_softc(dev);
+ struct xive_softc *sc = &psc->xps_core;
+ phandle_t phandle = ofw_bus_get_node(dev);
+ pcell_t lisn_range[2];
+ int rid, error;
+ int64_t rc;
+
+ rc = phyp_hcall(H_INT_RESET, 0ULL);
+ if (rc != H_SUCCESS) {
+ device_printf(dev, "H_INT_RESET failed: %ld\n", (long)rc);
+ return (ENXIO);
+ }
+
+ rid = 1; /* TIMA per-thread page. */
+ sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+ RF_ACTIVE);
+ if (sc->sc_mem == NULL) {
+ device_printf(dev, "Unable to allocate TIMA MMIO resource\n");
+ return (ENXIO);
+ }
+ sc->sc_offset = XIVE_TM_QW1_OS;
+
+ error = xive_phyp_pick_priority(dev, &sc->sc_prio);
+ if (error != 0) {
+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
+ return (error);
+ }
+
+ error = xive_phyp_pick_eq_shift(dev, phandle, &sc->sc_eq_shift);
+ if (error != 0) {
+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
+ return (error);
+ }
+
+ /*
+ * Consume the LISN pool for our IPIs. We only need mp_ncpus of them
+ * so a bump allocator over the first range is sufficient.
+ */
+ error = OF_getencprop(phandle, "ibm,xive-lisn-ranges",
+ lisn_range, sizeof(lisn_range));
+ if (error < (int)sizeof(lisn_range)) {
+ device_printf(dev, "Missing ibm,xive-lisn-ranges property\n");
+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
+ return (ENXIO);
+ }
+ psc->xps_ipi_base = lisn_range[0];
+ psc->xps_ipi_count = lisn_range[1];
+ psc->xps_ipi_next = 0;
+
+ if (psc->xps_ipi_count < (uint32_t)mp_ncpus) {
+ device_printf(dev,
+ "LISN pool (%u) smaller than mp_ncpus (%d)\n",
+ psc->xps_ipi_count, mp_ncpus);
+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
+ return (ENXIO);
+ }
+
+ return (xive_core_attach(dev));
+}
+
+/*
+ * XIVE KOBJ methods.
+ */
+static int
+xive_phyp_queue_config(device_t dev, struct xive_cpu *cpud __unused,
+ int cpu, vm_paddr_t qpage, int qshift, uint8_t prio)
+{
+ uint64_t hwref = pcpu_find(cpu)->pc_hwref;
+ int64_t rc;
+
+ rc = phyp_hcall(H_INT_SET_QUEUE_CONFIG, XIVE_EQ_ALWAYS_NOTIFY,
+ hwref, (uint64_t)prio, (uint64_t)qpage, (uint64_t)qshift);
+ if (rc == H_BUSY)
+ return (EBUSY);
+ if (rc != H_SUCCESS) {
+ device_printf(dev, "H_INT_SET_QUEUE_CONFIG(target %#lx, prio "
+ "%u, qpage %#lx, qshift %d) failed: %ld\n", (u_long)hwref,
+ prio, (u_long)qpage, qshift, (long)rc);
+ return (EIO);
+ }
+ return (0);
+}
+
+static int
+xive_phyp_ipi_alloc(device_t dev, struct xive_cpu *cpud __unused,
+ int cpu __unused, u_int *lisn_out)
+{
+ struct xive_phyp_softc *psc = device_get_softc(dev);
+
+ if (psc->xps_ipi_next >= psc->xps_ipi_count)
+ return (ENOSPC);
+ *lisn_out = psc->xps_ipi_base + psc->xps_ipi_next++;
+ return (0);
+}
+
+static int
+xive_phyp_irq_populate(device_t dev, struct xive_irq *irqd, u_int lisn)
+{
+ uint64_t out[4];
+ int64_t rc;
+
+ /*
+ * PAPR H_INT_GET_SOURCE_INFO returns in R4-R7:
+ * out[0]: flags (XIVE_SRC_*)
+ * out[1]: EOI ESB page phys
+ * out[2]: trigger page phys
+ * out[3]: esb shift
+ */
+ rc = phyp_hcall4(H_INT_GET_SOURCE_INFO, 0ULL, (uint64_t)lisn,
+ 0, 0, 0, 0, out);
+ if (rc != H_SUCCESS)
+ return (EIO);
+
+ irqd->flags = 0;
+ if (out[0] & XIVE_SRC_STORE_EOI)
+ irqd->flags |= XIVE_IRQ_FLAG_STORE_EOI;
+ if (out[0] & XIVE_SRC_LSI)
+ irqd->flags |= XIVE_IRQ_FLAG_LSI;
+ if (out[0] & XIVE_SRC_TRIGGER)
+ irqd->flags |= XIVE_IRQ_FLAG_TRIGGER_PAGE;
+ /*
+ * Such a source has no ESB MMIO at all, and reports -1 for both
+ * pages; the core reaches it through XIVE_ESB_OP instead. QEMU sets
+ * this for every LSI so that it can re-trigger a still-asserted
+ * level under KVM.
+ */
+ if (out[0] & XIVE_SRC_H_INT_ESB)
+ irqd->flags |= XIVE_IRQ_FLAG_H_INT_ESB;
+
+ /* The hypervisor returns -1 for a page that isn't present. */
+ irqd->eoi_page = (out[1] == (uint64_t)-1) ? 0 : (vm_offset_t)out[1];
+ irqd->trig_page = (out[2] == (uint64_t)-1) ? 0 : (vm_offset_t)out[2];
+ irqd->esb_size = 1U << out[3];
+ irqd->chip = 0; /* Guest has no chip affinity to speak of. */
+
+ /*
+ * H_INT_GET_SOURCE_CONFIG returns in R4-R7:
+ * out[0]: flags
+ * out[1]: target (hwref)
+ * out[2]: priority
+ * out[3]: sw_irq (logical IRQ)
+ */
+ rc = phyp_hcall4(H_INT_GET_SOURCE_CONFIG, 0ULL, (uint64_t)lisn,
+ 0, 0, 0, 0, out);
+ if (rc == H_SUCCESS) {
+ irqd->vp = out[1];
+ irqd->prio = (uint8_t)out[2];
+ irqd->lirq = (uint32_t)out[3];
+ }
+ return (0);
+}
+
+static int
+xive_phyp_irq_configure(device_t dev, u_int lisn, int cpu,
+ uint8_t prio, uint32_t sw_irq)
+{
+ uint64_t hwref = pcpu_find(cpu)->pc_hwref;
+ int64_t rc;
+
+ rc = phyp_hcall(H_INT_SET_SOURCE_CONFIG, XIVE_SRC_SET_EISN,
+ (uint64_t)lisn, hwref, (uint64_t)prio, (uint64_t)sw_irq);
+ if (rc == H_BUSY)
+ return (EBUSY);
+ if (rc != H_SUCCESS) {
+ device_printf(dev, "H_INT_SET_SOURCE_CONFIG(lisn %u, target "
+ "%#lx, prio %u) failed: %ld\n", lisn, (u_long)hwref, prio,
+ (long)rc);
+ return (EIO);
+ }
+ return (0);
+}
+
+/*
+ * Drive the ESB of a source that has no MMIO mapping. Called from the
+ * interrupt path, so it stays quiet on failure rather than risking a
+ * printf storm; a failing EOI shows up as a source that stops firing.
+ */
+static uint64_t
+xive_phyp_esb_op(device_t dev __unused, struct xive_irq *irqd,
+ uint32_t offset, uint64_t data, bool store)
+{
+ int64_t rc;
+
+ rc = phyp_hcall(H_INT_ESB, store ? XIVE_ESB_STORE : 0ULL,
+ (uint64_t)irqd->girq, (uint64_t)offset, data);
+ if (rc != H_SUCCESS)
+ return (~0ULL);
+
+ /* A store returns -1 in R4; report it as the 0 the core expects. */
+ return (rc == H_SUCCESS ? 0 : ~0ULL);
+}
+
+static int
+xive_phyp_sync(device_t dev __unused, u_int lisn)
+{
+ int64_t rc;
+
+ rc = phyp_hcall(H_INT_SYNC, 0ULL, (uint64_t)lisn);
+ return (rc == H_SUCCESS ? 0 : EIO);
+}

File Metadata

Mime Type
text/plain
Expires
Thu, Sep 10, 7:57 PM (10 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
38366376
Default Alt Text
D59259.id185371.diff (59 KB)

Event Timeline