Index: sys/arm64/arm64/gic_v3.c =================================================================== --- sys/arm64/arm64/gic_v3.c +++ sys/arm64/arm64/gic_v3.c @@ -63,6 +63,8 @@ #include "gic_v3_reg.h" #include "gic_v3_var.h" +static bus_read_ivar_t gic_v3_read_ivar; + #ifdef INTRNG static pic_disable_intr_t gic_v3_disable_intr; static pic_enable_intr_t gic_v3_enable_intr; @@ -101,6 +103,9 @@ /* Device interface */ DEVMETHOD(device_detach, gic_v3_detach), + /* Bus interface */ + DEVMETHOD(bus_read_ivar, gic_v3_read_ivar), + #ifdef INTRNG /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, gic_v3_disable_intr), @@ -179,6 +184,44 @@ }; #endif +#ifdef INTRNG +uint32_t +gic_r_read_4(device_t dev, bus_size_t offset) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + return (bus_read_4(sc->gic_redists.pcpu[PCPU_GET(cpuid)], offset)); +} + +uint64_t +gic_r_read_8(device_t dev, bus_size_t offset) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + return (bus_read_8(sc->gic_redists.pcpu[PCPU_GET(cpuid)], offset)); +} + +void +gic_r_write_4(device_t dev, bus_size_t offset, uint32_t val) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + bus_write_4(sc->gic_redists.pcpu[PCPU_GET(cpuid)], offset, val); +} + +void +gic_r_write_8(device_t dev, bus_size_t offset, uint64_t val) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + bus_write_8(sc->gic_redists.pcpu[PCPU_GET(cpuid)], offset, val); +} +#endif + /* * Device interface. */ @@ -240,6 +283,9 @@ /* Get the number of supported SPI interrupts */ typer = gic_d_read(sc, 4, GICD_TYPER); sc->gic_nirqs = GICD_TYPER_I_NUM(typer); +#ifdef INTRNG + sc->gic_nirqs += GIC_FIRST_SPI; +#endif if (sc->gic_nirqs > GIC_I_NUM_MAX) sc->gic_nirqs = GIC_I_NUM_MAX; @@ -327,17 +373,36 @@ return (0); } +static int +gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + switch (which) { + case GICV3_IVAR_REDIST_VADDR: + *result = (uintptr_t)rman_get_virtual( + sc->gic_redists.pcpu[PCPU_GET(cpuid)]); + return (0); + } + + return (ENOENT); +} + #ifdef INTRNG int arm_gic_v3_intr(void *arg) { struct gic_v3_softc *sc = arg; struct gic_v3_irqsrc *gi; + struct intr_pic *pic; uint64_t active_irq; struct trapframe *tf; bool first; first = true; + pic = sc->gic_pic; while (1) { if (CPU_MATCH_ERRATA_CAVIUM_THUNDER_1_1) { @@ -357,6 +422,11 @@ active_irq = gic_icc_read(IAR1); } + if (active_irq >= GIC_FIRST_LPI) { + intr_child_irq_handler(pic, active_irq); + continue; + } + if (__predict_false(active_irq >= sc->gic_nirqs)) return (FILTER_HANDLED); @@ -383,7 +453,8 @@ device_printf(sc->dev, "Stray irq %lu disabled\n", active_irq); } - } + } else + panic("Unknown IRQ %lu", active_irq); } } Index: sys/arm64/arm64/gic_v3_fdt.c =================================================================== --- sys/arm64/arm64/gic_v3_fdt.c +++ sys/arm64/arm64/gic_v3_fdt.c @@ -139,7 +139,8 @@ #ifdef INTRNG xref = OF_xref_from_node(ofw_bus_get_node(dev)); - if (intr_pic_register(dev, xref) == NULL) { + sc->gic_pic = intr_pic_register(dev, xref); + if (sc->gic_pic == NULL) { device_printf(dev, "could not register PIC\n"); goto error; } Index: sys/arm64/arm64/gic_v3_reg.h =================================================================== --- sys/arm64/arm64/gic_v3_reg.h +++ sys/arm64/arm64/gic_v3_reg.h @@ -142,6 +142,8 @@ #define GICR_PROPBASER_CACHE_NIWAWB 0x5UL #define GICR_PROPBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PROPBASER_CACHE_NIRAWAWB 0x7UL +#define GICR_PROPBASER_CACHE_MASK \ + (0x7UL << GICR_PROPBASER_CACHE_SHIFT) /* * Shareability @@ -179,6 +181,8 @@ #define GICR_PENDBASER_CACHE_NIWAWB 0x5UL #define GICR_PENDBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PENDBASER_CACHE_NIRAWAWB 0x7UL +#define GICR_PENDBASER_CACHE_MASK \ + (0x7UL << GICR_PENDBASER_CACHE_SHIFT) /* * Shareability @@ -217,6 +221,26 @@ #define GITS_CTLR (0x0000) #define GITS_CTLR_EN (1 << 0) +#define GITS_IIDR (0x0004) +#define GITS_IIDR_PRODUCT_SHIFT 24 +#define GITS_IIDR_PRODUCT_MASK (0xff << GITS_IIDR_PRODUCT_SHIFT) +#define GITS_IIDR_VARIANT_SHIFT 16 +#define GITS_IIDR_VARIANT_MASK (0xf << GITS_IIDR_VARIANT_SHIFT) +#define GITS_IIDR_REVISION_SHIFT 12 +#define GITS_IIDR_REVISION_MASK (0xf << GITS_IIDR_REVISION_SHIFT) +#define GITS_IIDR_IMPLEMENTOR_SHIFT 0 +#define GITS_IIDR_IMPLEMENTOR_MASK (0xfff << GITS_IIDR_IMPLEMENTOR_SHIFT) + +#define GITS_IIDR_RAW(impl, prod, var, rev) \ + ((prod) << GITS_IIDR_PRODUCT_SHIFT | \ + (var) << GITS_IIDR_VARIANT_SHIFT | \ + (rev) << GITS_IIDR_REVISION_SHIFT | \ + (impl) << GITS_IIDR_IMPLEMENTOR_SHIFT) + +#define GITS_IIDR_IMPL_CAVIUM (0x34c) +#define GITS_IIDR_PROD_THUNDER (0xa1) +#define GITS_IIDR_VAR_THUNDER_1 (0x0) + #define GITS_CBASER (0x0080) #define GITS_CBASER_VALID (1UL << 63) /* @@ -239,7 +263,7 @@ #define GITS_CBASER_CACHE_NIWAWB 0x5UL #define GITS_CBASER_CACHE_NIRAWAWT 0x6UL #define GITS_CBASER_CACHE_NIRAWAWB 0x7UL -#define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_TYPE_SHIFT) +#define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable Index: sys/arm64/arm64/gic_v3_var.h =================================================================== --- sys/arm64/arm64/gic_v3_var.h +++ sys/arm64/arm64/gic_v3_var.h @@ -36,10 +36,12 @@ DECLARE_CLASS(gic_v3_driver); +#ifndef INTRNG #define LPI_FLAGS_CONF_FLUSH (1UL << 0) #define LPI_CONFTAB_SIZE PAGE_SIZE_64K /* 1 bit per LPI + 1 KB more for the obligatory PPI, SGI, SPI stuff */ #define LPI_PENDTAB_SIZE ((LPI_CONFTAB_SIZE / 8) + 0x400) +#endif #ifdef INTRNG struct gic_v3_irqsrc { @@ -86,6 +88,7 @@ boolean_t gic_registered; #ifdef INTRNG + struct intr_pic *gic_pic; struct gic_v3_irqsrc *gic_irqs; #endif }; @@ -96,27 +99,39 @@ MALLOC_DECLARE(M_GIC_V3); +/* ivars */ +enum { + GICV3_IVAR_REDIST_VADDR, +}; + +__BUS_ACCESSOR(gicv3, redist_vaddr, GICV3, REDIST_VADDR, void *); + /* Device methods */ int gic_v3_attach(device_t dev); int gic_v3_detach(device_t dev); int arm_gic_v3_intr(void *); +#ifdef INTRNG +uint32_t gic_r_read_4(device_t, bus_size_t); +uint64_t gic_r_read_8(device_t, bus_size_t); +void gic_r_write_4(device_t, bus_size_t, uint32_t var); +void gic_r_write_8(device_t, bus_size_t, uint64_t var); +#endif + /* * ITS */ -#define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service" -#define GIC_V3_ITS_COMPSTR "arm,gic-v3-its" - -DECLARE_CLASS(gic_v3_its_driver); /* LPI chunk owned by ITS device */ struct lpi_chunk { u_int lpi_base; u_int lpi_free; /* First free LPI in set */ + u_int lpi_num; /* Total number of LPIs in chunk */ +#ifndef INTRNG u_int *lpi_col_ids; - u_int lpi_num; /* Total number of LPIs in chunk */ u_int lpi_busy; /* Number of busy LPIs in chink */ +#endif }; /* ITS device */ @@ -132,6 +147,7 @@ vm_offset_t itt; size_t itt_size; }; +#ifndef INTRNG TAILQ_HEAD(its_dev_list, its_dev); /* ITS private table description */ @@ -152,6 +168,11 @@ uint64_t cmd_dword[4]; /* ITS command double word */ }; +#define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service" +#define GIC_V3_ITS_COMPSTR "arm,gic-v3-its" + +DECLARE_CLASS(gic_v3_its_driver); + /* ITS commands encoding */ #define ITS_CMD_MOVI (0x01) #define ITS_CMD_SYNC (0x05) @@ -183,6 +204,7 @@ /* Valid command bit */ #define CMD_VALID_SHIFT (63) #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) +#endif /* INTRNG */ /* * ITS command descriptor. @@ -237,13 +259,14 @@ }; }; +#define ITS_TARGET_NONE 0xFBADBEEF + +#ifndef INTRNG #define ITS_CMDQ_SIZE PAGE_SIZE_64K #define ITS_CMDQ_NENTRIES (ITS_CMDQ_SIZE / sizeof(struct its_cmd)) #define ITS_FLAGS_CMDQ_FLUSH (1UL << 0) -#define ITS_TARGET_NONE 0xFBADBEEF - struct gic_v3_its_softc { device_t dev; struct resource * its_res; @@ -255,7 +278,9 @@ uint64_t its_flags; +#ifndef INTRNG struct its_dev_list its_dev_list; +#endif bitstr_t * its_lpi_bitmap; uint32_t its_lpi_maxid; @@ -290,6 +315,7 @@ int lpi_migrate(device_t, uint32_t, u_int); void lpi_unmask_irq(device_t, uint32_t); void lpi_mask_irq(device_t, uint32_t); +#endif /* * GIC Distributor accessors. * Notice that only GIC sofc can be passed. Index: sys/arm64/arm64/gicv3_its.c =================================================================== --- /dev/null +++ sys/arm64/arm64/gicv3_its.c @@ -0,0 +1,1516 @@ +/*- + * Copyright (c) 2015-2016 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Andrew Turner under + * the sponsorship of the FreeBSD Foundation. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_platform.h" + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +#ifdef FDT +#include +#include +#include +#endif +#include +#include + +#include "pcib_if.h" +#include "pic_if.h" +#include "msi_if.h" + +MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", + "ARM GICv3 Interrupt Translation Service"); + +#define LPI_NIRQS (64 * 1024) + +/* The size and alignment of the command circular buffer */ +#define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ +#define ITS_CMDQ_ALIGN (64 * 1024) + +#define LPI_CONFTAB_SIZE LPI_NIRQS +#define LPI_CONFTAB_ALIGN (64 * 1024) +#define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ + +/* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ +#define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) +#define LPI_PENDTAB_ALIGN (64 * 1024) +#define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ + +#define LPI_INT_TRANS_TAB_ALIGN 256 +#define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) + +/* ITS commands encoding */ +#define ITS_CMD_MOVI (0x01) +#define ITS_CMD_SYNC (0x05) +#define ITS_CMD_MAPD (0x08) +#define ITS_CMD_MAPC (0x09) +#define ITS_CMD_MAPTI (0x0a) +#define ITS_CMD_MAPI (0x0b) +#define ITS_CMD_INV (0x0c) +#define ITS_CMD_INVALL (0x0d) +/* Command */ +#define CMD_COMMAND_MASK (0xFFUL) +/* PCI device ID */ +#define CMD_DEVID_SHIFT (32) +#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) +/* Size of IRQ ID bitfield */ +#define CMD_SIZE_MASK (0xFFUL) +/* Virtual LPI ID */ +#define CMD_ID_MASK (0xFFFFFFFFUL) +/* Physical LPI ID */ +#define CMD_PID_SHIFT (32) +#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) +/* Collection */ +#define CMD_COL_MASK (0xFFFFUL) +/* Target (CPU or Re-Distributor) */ +#define CMD_TARGET_SHIFT (16) +#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) +/* Interrupt Translation Table address */ +#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) +/* Valid command bit */ +#define CMD_VALID_SHIFT (63) +#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) + +/* ITS command. Each command is 32 bytes long */ +struct its_cmd { + uint64_t cmd_dword[4]; /* ITS command double word */ +}; + +/* An ITS private table */ +struct its_ptable { + vm_offset_t ptab_vaddr; + unsigned long ptab_size; +}; + +/* ITS collection description. */ +struct its_col { + uint64_t col_target; /* Target Re-Distributor */ + uint64_t col_id; /* Collection ID */ +}; + +struct gicv3_its_irqsrc { + struct intr_irqsrc gi_isrc; + u_int gi_irq; + struct its_dev *gi_its_dev; +}; + +struct gicv3_its_softc { + struct intr_pic *sc_pic; + struct resource *sc_its_res; + + struct its_ptable sc_its_ptab[GITS_BASER_NUM]; + struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ + + /* + * TODO: We should get these from the parent as we only want a + * single copy of each across the interrupt controller. + */ + vm_offset_t sc_conf_base; + vm_offset_t sc_pend_base[MAXCPU]; + + /* Command handling */ + struct mtx sc_its_cmd_lock; + struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ + size_t sc_its_cmd_next_idx; + + u_int sc_next_irq; + struct gicv3_its_irqsrc *sc_irqs; + + struct mtx sc_its_dev_lock; + TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; + +#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 +#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 +#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 + u_int sc_its_flags; +}; + +typedef void (its_quirk_func_t)(device_t); +static its_quirk_func_t its_quirk_cavium_22375; + +static const struct { + const char *desc; + uint32_t iidr; + uint32_t iidr_mask; + its_quirk_func_t *func; +} its_quirks[] = { + { + /* Cavium ThunderX Pass 1.x */ + .desc = "Cavoum ThunderX errata: 22375, 24313", + .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, + GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), + .iidr_mask = ~GITS_IIDR_REVISION_MASK, + .func = its_quirk_cavium_22375, + }, +}; + +static u_int gic_irq_cpu; + +#define gic_its_read_4(sc, reg) \ + bus_read_4((sc)->sc_its_res, (reg)) +#define gic_its_read_8(sc, reg) \ + bus_read_8((sc)->sc_its_res, (reg)) + +#define gic_its_write_4(sc, reg, val) \ + bus_write_4((sc)->sc_its_res, (reg), (val)) +#define gic_its_write_8(sc, reg, val) \ + bus_write_8((sc)->sc_its_res, (reg), (val)) + +static device_attach_t gicv3_its_attach; +static device_detach_t gicv3_its_detach; + +static pic_disable_intr_t gicv3_its_disable_intr; +static pic_enable_intr_t gicv3_its_enable_intr; +static pic_map_intr_t gicv3_its_map_intr; +static pic_setup_intr_t gicv3_its_setup_intr; +static pic_teardown_intr_t gicv3_its_teardown_intr; +static pic_post_filter_t gicv3_its_post_filter; +static pic_post_ithread_t gicv3_its_post_ithread; +static pic_pre_ithread_t gicv3_its_pre_ithread; +static pic_bind_intr_t gicv3_its_bind_intr; +#ifdef SMP +static pic_init_secondary_t gicv3_its_init_secondary; +#endif +static msi_alloc_msi_t gicv3_its_alloc_msi; +static msi_release_msi_t gicv3_its_release_msi; +static msi_alloc_msix_t gicv3_its_alloc_msix; +static msi_release_msix_t gicv3_its_release_msix; +static msi_map_msi_t gicv3_its_map_msi; + +static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); +static void its_cmd_mapc(device_t, struct its_col *, uint8_t); +static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); +static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); +static void its_cmd_inv(device_t, struct its_dev *, uint32_t); +static void its_cmd_invall(device_t, struct its_col *); + +static device_method_t gicv3_its_methods[] = { + /* Device interface */ + DEVMETHOD(device_detach, gicv3_its_detach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), + DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), + DEVMETHOD(pic_map_intr, gicv3_its_map_intr), + DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), + DEVMETHOD(pic_teardown_intr, gicv3_its_teardown_intr), + DEVMETHOD(pic_post_filter, gicv3_its_post_filter), + DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), + DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), +#ifdef SMP + DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), + DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), +#endif + + /* MSI/MSI-X */ + DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), + DEVMETHOD(msi_release_msi, gicv3_its_release_msi), + DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), + DEVMETHOD(msi_release_msix, gicv3_its_release_msix), + DEVMETHOD(msi_map_msi, gicv3_its_map_msi), + + /* End */ + DEVMETHOD_END +}; + +static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, + sizeof(struct gicv3_its_softc)); + +static void +gicv3_its_cmdq_init(struct gicv3_its_softc *sc) +{ + vm_paddr_t cmd_paddr; + uint64_t reg, tmp; + + /* Set up the command circular buffer */ + sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, + M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); + sc->sc_its_cmd_next_idx = 0; + + cmd_paddr = vtophys(sc->sc_its_cmd_base); + + /* Set the base of the command buffer */ + reg = GITS_CBASER_VALID | + (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | + cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | + (ITS_CMDQ_SIZE / 4096 - 1); + gic_its_write_8(sc, GITS_CBASER, reg); + + /* Read back to check for fixed value fields */ + tmp = gic_its_read_8(sc, GITS_CBASER); + + if ((tmp & GITS_CBASER_SHARE_MASK) != + (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { + /* Check if the hardware reported non-shareable */ + if ((tmp & GITS_CBASER_SHARE_MASK) == + (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { + /* If so remove the cache attribute */ + reg &= ~GITS_CBASER_CACHE_MASK; + reg &= ~GITS_CBASER_SHARE_MASK; + /* Set to Non-cacheable, Non-shareable */ + reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; + reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; + + gic_its_write_8(sc, GITS_CBASER, reg); + } + + /* The command queue has to be flushed after each command */ + sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; + } + + /* Get the next command from the start of the buffer */ + gic_its_write_8(sc, GITS_CWRITER, 0x0); +} + +static int +gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) +{ + vm_offset_t table; + vm_paddr_t paddr; + uint64_t cache, reg, share, tmp, type; + size_t esize, its_tbl_size, nidents, nitspages, npages; + int i, page_size; + int devbits; + + if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { + /* + * GITS_TYPER[17:13] of ThunderX reports that device IDs + * are to be 21 bits in length. The entry size of the ITS + * table can be read from GITS_BASERn[52:48] and on ThunderX + * is supposed to be 8 bytes in length (for device table). + * Finally the page size that is to be used by ITS to access + * this table will be set to 64KB. + * + * This gives 0x200000 entries of size 0x8 bytes covered by + * 256 pages each of which 64KB in size. The number of pages + * (minus 1) should then be written to GITS_BASERn[7:0]. In + * that case this value would be 0xFF but on ThunderX the + * maximum value that HW accepts is 0xFD. + * + * Set an arbitrary number of device ID bits to 20 in order + * to limit the number of entries in ITS device table to + * 0x100000 and the table size to 8MB. + */ + devbits = 20; + cache = 0; + } else { + devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); + cache = GITS_BASER_CACHE_WAWB; + } + share = GITS_BASER_SHARE_IS; + page_size = PAGE_SIZE_64K; + + for (i = 0; i < GITS_BASER_NUM; i++) { + reg = gic_its_read_8(sc, GITS_BASER(i)); + /* The type of table */ + type = GITS_BASER_TYPE(reg); + /* The table entry size */ + esize = GITS_BASER_ESIZE(reg); + + switch(type) { + case GITS_BASER_TYPE_DEV: + nidents = (1 << devbits); + its_tbl_size = esize * nidents; + its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); + break; + case GITS_BASER_TYPE_VP: + case GITS_BASER_TYPE_PP: /* Undocumented? */ + case GITS_BASER_TYPE_IC: + its_tbl_size = page_size; + break; + default: + continue; + } + npages = howmany(its_tbl_size, PAGE_SIZE); + + /* Allocate the table */ + table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, + M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, + PAGE_SIZE, 0); + + sc->sc_its_ptab[i].ptab_vaddr = table; + sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; + + paddr = vtophys(table); + + while (1) { + nitspages = howmany(its_tbl_size, page_size); + + /* Clear the fields we will be setting */ + reg &= ~(GITS_BASER_VALID | + GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | + GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | + GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | + GITS_BASER_SIZE_MASK); + /* Set the new values */ + reg |= GITS_BASER_VALID | + (cache << GITS_BASER_CACHE_SHIFT) | + (type << GITS_BASER_TYPE_SHIFT) | + ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | + paddr | (share << GITS_BASER_SHARE_SHIFT) | + (nitspages - 1); + + switch (page_size) { + case PAGE_SIZE: /* 4KB */ + reg |= + GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; + break; + case PAGE_SIZE_16K: /* 16KB */ + reg |= + GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; + break; + case PAGE_SIZE_64K: /* 64KB */ + reg |= + GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; + break; + } + + gic_its_write_8(sc, GITS_BASER(i), reg); + + /* Read back to check */ + tmp = gic_its_read_8(sc, GITS_BASER(i)); + + /* Do the snareability masks line up? */ + if ((tmp & GITS_BASER_SHARE_MASK) != + (reg & GITS_BASER_SHARE_MASK)) { + share = (tmp & GITS_BASER_SHARE_MASK) >> + GITS_BASER_SHARE_SHIFT; + continue; + } + + if ((tmp & GITS_BASER_PSZ_MASK) != + (reg & GITS_BASER_PSZ_MASK)) { + switch (page_size) { + case PAGE_SIZE_16K: + page_size = PAGE_SIZE; + continue; + case PAGE_SIZE_64K: + page_size = PAGE_SIZE_16K; + continue; + } + } + + if (tmp != reg) { + device_printf(dev, "GITS_BASER%d: " + "unable to be updated: %lx != %lx\n", + i, reg, tmp); + return (ENXIO); + } + + /* We should have made all needed changes */ + break; + } + } + + return (0); +} + +static void +gicv3_its_conftable_init(struct gicv3_its_softc *sc) +{ + + sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, + M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, + 0); + + /* Set the default configuration */ + memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, + LPI_CONFTAB_SIZE); + + /* Flush the table to memory */ + cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); +} + +static void +gicv3_its_pendtables_init(struct gicv3_its_softc *sc) +{ + int i; + + for (i = 0; i < mp_ncpus; i++) { + if (CPU_ISSET(i, &all_cpus) == 0) + continue; + + sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( + LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, + 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); + + /* Flush so the ITS can see the memory */ + cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base, + LPI_PENDTAB_SIZE); + } +} + +static int +its_init_cpu(device_t dev, struct gicv3_its_softc *sc) +{ + device_t gicv3; + vm_paddr_t target; + uint64_t xbaser, tmp; + uint32_t ctlr; + u_int cpuid; + + gicv3 = device_get_parent(dev); + cpuid = PCPU_GET(cpuid); + + /* Check if the ITS is enabled on this CPU */ + if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { + return (ENXIO); + } + + /* Disable LPIs */ + ctlr = gic_r_read_4(gicv3, GICR_CTLR); + ctlr &= ~GICR_CTLR_LPI_ENABLE; + gic_r_write_4(gicv3, GICR_CTLR, ctlr); + + /* Make sure changes are observable my the GIC */ + dsb(sy); + + /* + * Set the redistributor base + */ + xbaser = vtophys(sc->sc_conf_base) | + (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | + (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | + (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); + gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); + + /* Check the cache attributes we set */ + tmp = gic_r_read_8(gicv3, GICR_PROPBASER); + + if ((tmp & GICR_PROPBASER_SHARE_MASK) != + (xbaser & GICR_PROPBASER_SHARE_MASK)) { + if ((tmp & GICR_PROPBASER_SHARE_MASK) == + (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { + /* We need to mark as non-cacheable */ + xbaser &= ~(GICR_PROPBASER_SHARE_MASK | + GICR_PROPBASER_CACHE_MASK); + /* Non-cacheable */ + xbaser |= GICR_PROPBASER_CACHE_NIN << + GICR_PROPBASER_CACHE_SHIFT; + /* Non-sareable */ + xbaser |= GICR_PROPBASER_SHARE_NS << + GICR_PROPBASER_SHARE_SHIFT; + gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); + } + sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; + } + + /* + * Set the LPI pending table base + */ + xbaser = vtophys(sc->sc_pend_base[cpuid]) | + (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | + (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); + + gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); + + tmp = gic_r_read_8(gicv3, GICR_PENDBASER); + + if ((tmp & GICR_PENDBASER_SHARE_MASK) == + (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { + /* Clear the cahce and shareability bits */ + xbaser &= ~(GICR_PENDBASER_CACHE_MASK | + GICR_PENDBASER_SHARE_MASK); + /* Mark as non-shareable */ + xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; + /* And non-cacheable */ + xbaser |= GICR_PENDBASER_CACHE_NIN << + GICR_PENDBASER_CACHE_SHIFT; + } + + /* Enable LPIs */ + ctlr = gic_r_read_4(gicv3, GICR_CTLR); + ctlr |= GICR_CTLR_LPI_ENABLE; + gic_r_write_4(gicv3, GICR_CTLR, ctlr); + + /* Make sure the GIC has seen everything */ + dsb(sy); + + if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { + /* This ITS wants the redistributor physical address */ + target = vtophys(gicv3_get_redist_vaddr(dev)); + } else { + /* This ITS wants the unique processor number */ + target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); + } + + sc->sc_its_cols[cpuid]->col_target = target; + sc->sc_its_cols[cpuid]->col_id = cpuid; + + its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); + its_cmd_invall(dev, sc->sc_its_cols[cpuid]); + + return (0); +} + +static int +gicv3_its_attach(device_t dev) +{ + struct gicv3_its_softc *sc; + const char *name; + uint32_t iidr; + int err, i, rid; + + sc = device_get_softc(dev); + + rid = 0; + sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->sc_its_res == NULL) { + device_printf(dev, "Could not allocate memory\n"); + return (ENXIO); + } + + iidr = gic_its_read_4(sc, GITS_IIDR); + for (i = 0; i < nitems(its_quirks); i++) { + if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { + if (bootverbose) { + device_printf(dev, "Applying %s\n", + its_quirks[i].desc); + } + its_quirks[i].func(dev); + break; + } + } + + /* Allocate the private tables */ + err = gicv3_its_table_init(dev, sc); + if (err != 0) + return (err); + + /* Protects access to the device list */ + mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); + + /* Protects access to the ITS command circular buffer. */ + mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); + + /* Allocate the command circular buffer */ + gicv3_its_cmdq_init(sc); + + /* Allocate the per-CPU collections */ + for (int cpu = 0; cpu < mp_ncpus; cpu++) + if (CPU_ISSET(cpu, &all_cpus) != 0) + sc->sc_its_cols[cpu] = malloc( + sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, + M_WAITOK | M_ZERO); + else + sc->sc_its_cols[cpu] = NULL; + + /* Enable the ITS */ + gic_its_write_4(sc, GITS_CTLR, + gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); + + /* Create the LPI configuration table */ + gicv3_its_conftable_init(sc); + + /* And the pending tebles */ + gicv3_its_pendtables_init(sc); + + /* Enable LPIs on this CPU */ + its_init_cpu(dev, sc); + + TAILQ_INIT(&sc->sc_its_dev_list); + + sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * LPI_NIRQS, M_GICV3_ITS, + M_WAITOK | M_ZERO); + name = device_get_nameunit(dev); + for (i = 0; i < LPI_NIRQS; i++) { + sc->sc_irqs[i].gi_irq = i; + err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, + "%s,%u", name, i); + } + + return (0); +} + +static int +gicv3_its_detach(device_t dev) +{ + + return (ENXIO); +} + +static void +its_quirk_cavium_22375(device_t dev) +{ + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; +} + +static void +gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv3_its_softc *sc; + struct gicv3_its_irqsrc *girq; + uint8_t *conf; + + sc = device_get_softc(dev); + girq = (struct gicv3_its_irqsrc *)isrc; + conf = (uint8_t *)sc->sc_conf_base; + + conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; + + if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { + /* Clean D-cache under command. */ + cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); + } else { + /* DSB inner shareable, store */ + dsb(ishst); + } + + its_cmd_inv(dev, girq->gi_its_dev, girq->gi_irq + GIC_FIRST_LPI); +} + +static void +gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv3_its_softc *sc; + struct gicv3_its_irqsrc *girq; + uint8_t *conf; + + sc = device_get_softc(dev); + girq = (struct gicv3_its_irqsrc *)isrc; + conf = (uint8_t *)sc->sc_conf_base; + + conf[girq->gi_irq] |= LPI_CONF_ENABLE; + + if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { + /* Clean D-cache under command. */ + cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); + } else { + /* DSB inner shareable, store */ + dsb(ishst); + } + + its_cmd_inv(dev, girq->gi_its_dev, girq->gi_irq + GIC_FIRST_LPI); +} + +static int +gicv3_its_intr(void *arg, uintptr_t irq) +{ + struct gicv3_its_softc *sc = arg; + struct gicv3_its_irqsrc *girq; + struct trapframe *tf; + + irq -= GIC_FIRST_LPI; + girq = &sc->sc_irqs[irq]; + if (girq == NULL) + panic("%s: Invalid interrupt %ld", __func__, + irq + GIC_FIRST_LPI); + + tf = curthread->td_intr_frame; + intr_isrc_dispatch(&girq->gi_isrc, tf); + return (FILTER_HANDLED); +} + +static void +gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv3_its_irqsrc *girq; + + girq = (struct gicv3_its_irqsrc *)isrc; + gicv3_its_disable_intr(dev, isrc); + gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); +} + +static void +gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + + gicv3_its_enable_intr(dev, isrc); +} + +static void +gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv3_its_irqsrc *girq; + + girq = (struct gicv3_its_irqsrc *)isrc; + gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); +} + +static int +gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gicv3_its_irqsrc *girq; + + girq = (struct gicv3_its_irqsrc *)isrc; + if (CPU_EMPTY(&isrc->isrc_cpu)) { + gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); + CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); + } + + its_cmd_movi(dev, girq); + + return (0); +} + +static int +gicv3_its_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + + /* + * This should never happen, we only call this function to map + * interrupts found before the controller driver is ready. + */ + panic("%s: Unable to map a MSI interrupt", __func__); +} + +static int +gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + + /* Bind the interrupt to a CPU */ + gicv3_its_bind_intr(dev, isrc); + + return (0); +} + + +static int +gicv3_its_teardown_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + + panic("gicv3_its_teardown_intr"); +} + +#ifdef SMP +static void +gicv3_its_init_secondary(device_t dev) +{ + + panic("gicv3_its_init_secondary"); +} +#endif + +static uint32_t +its_get_devid(device_t pci_dev) +{ + uintptr_t id; + + if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) + panic("its_get_devid: Unable to get the MSI DeviceID"); + + return (id); +} + +static struct its_dev * +its_device_get(device_t dev, device_t child, u_int nvecs) +{ + struct gicv3_its_softc *sc; + struct its_dev *its_dev = NULL; + size_t esize; + + sc = device_get_softc(dev); + + mtx_lock_spin(&sc->sc_its_dev_lock); + TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { + if (its_dev->pci_dev == child) + break; + } + mtx_unlock_spin(&sc->sc_its_dev_lock); + + if (its_dev != NULL) + return (its_dev); + + its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); + if (its_dev == NULL) + return (NULL); + + its_dev->pci_dev = child; + its_dev->devid = its_get_devid(child); + + its_dev->lpis.lpi_num = nvecs; + its_dev->lpis.lpi_free = nvecs; + + mtx_lock_spin(&sc->sc_its_dev_lock); + its_dev->lpis.lpi_base = sc->sc_next_irq; + sc->sc_next_irq = its_dev->lpis.lpi_base + nvecs; + mtx_unlock_spin(&sc->sc_its_dev_lock); + + /* Get ITT entry size */ + esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); + + /* + * Allocate ITT for this device. + * PA has to be 256 B aligned. At least two entries for device. + */ + its_dev->itt = (vm_offset_t)contigmalloc( + roundup2(MAX(nvecs, 2) * esize, 256), M_GICV3_ITS, + M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, + LPI_INT_TRANS_TAB_ALIGN, 0); + if (its_dev->itt == 0) { + free(its_dev, M_GICV3_ITS); + return (NULL); + } + + mtx_lock_spin(&sc->sc_its_dev_lock); + TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); + mtx_unlock_spin(&sc->sc_its_dev_lock); + + /* Map device to its ITT */ + its_cmd_mapd(dev, its_dev, 1); + + return (its_dev); +} + +static int +gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, + device_t *pic, struct intr_irqsrc **srcs) +{ + struct gicv3_its_softc *sc; + struct gicv3_its_irqsrc *girq; + struct its_dev *its_dev; + u_int irq; + int i; + + its_dev = its_device_get(dev, child, count); + if (its_dev == NULL) + return (ENXIO); + + KASSERT(its_dev->lpis.lpi_free >= count, + ("gicv3_its_alloc_msi: No free LPIs")); + sc = device_get_softc(dev); + irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - + its_dev->lpis.lpi_free; + for (i = 0; i < count; i++, irq++) { + its_dev->lpis.lpi_free--; + girq = &sc->sc_irqs[irq]; + girq->gi_its_dev = its_dev; + srcs[i] = (struct intr_irqsrc *)girq; + } + *pic = dev; + + return (0); +} + +static int +gicv3_its_release_msi(device_t dev, device_t child, int count, + struct intr_irqsrc **isrc) +{ + + panic("gicv3_its_release_msi"); +} + +static int +gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, + struct intr_irqsrc **isrcp) +{ + struct gicv3_its_softc *sc; + struct gicv3_its_irqsrc *girq; + struct its_dev *its_dev; + u_int nvecs, irq; + + nvecs = pci_msix_count(child); + its_dev = its_device_get(dev, child, nvecs); + if (its_dev == NULL) + return (ENXIO); + + KASSERT(its_dev->lpis.lpi_free > 0, + ("gicv3_its_alloc_msix: No free LPIs")); + sc = device_get_softc(dev); + irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - + its_dev->lpis.lpi_free; + its_dev->lpis.lpi_free--; + girq = &sc->sc_irqs[irq]; + girq->gi_its_dev = its_dev; + + *pic = dev; + *isrcp = (struct intr_irqsrc *)girq; + + return (0); +} + +static int +gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) +{ + + panic("gicv3_its_release_msix"); +} + +static int +gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, + uint64_t *addr, uint32_t *data) +{ + struct gicv3_its_softc *sc; + struct gicv3_its_irqsrc *girq; + + sc = device_get_softc(dev); + girq = (struct gicv3_its_irqsrc *)isrc; + + /* Map the message to the given IRQ */ + its_cmd_mapti(dev, girq); + + *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; + *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; + + return (0); +} + +/* + * Commands handling. + */ + +static __inline void +cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) +{ + /* Command field: DW0 [7:0] */ + cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); + cmd->cmd_dword[0] |= htole64(cmd_type); +} + +static __inline void +cmd_format_devid(struct its_cmd *cmd, uint32_t devid) +{ + /* Device ID field: DW0 [63:32] */ + cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); + cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); +} + +static __inline void +cmd_format_size(struct its_cmd *cmd, uint16_t size) +{ + /* Size field: DW1 [4:0] */ + cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); + cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); +} + +static __inline void +cmd_format_id(struct its_cmd *cmd, uint32_t id) +{ + /* ID field: DW1 [31:0] */ + cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); + cmd->cmd_dword[1] |= htole64(id); +} + +static __inline void +cmd_format_pid(struct its_cmd *cmd, uint32_t pid) +{ + /* Physical ID field: DW1 [63:32] */ + cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); + cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); +} + +static __inline void +cmd_format_col(struct its_cmd *cmd, uint16_t col_id) +{ + /* Collection field: DW2 [16:0] */ + cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); + cmd->cmd_dword[2] |= htole64(col_id); +} + +static __inline void +cmd_format_target(struct its_cmd *cmd, uint64_t target) +{ + /* Target Address field: DW2 [47:16] */ + cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); + cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); +} + +static __inline void +cmd_format_itt(struct its_cmd *cmd, uint64_t itt) +{ + /* ITT Address field: DW2 [47:8] */ + cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); + cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); +} + +static __inline void +cmd_format_valid(struct its_cmd *cmd, uint8_t valid) +{ + /* Valid field: DW2 [63] */ + cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); + cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); +} + +static inline bool +its_cmd_queue_full(struct gicv3_its_softc *sc) +{ + size_t read_idx, next_write_idx; + + /* Get the index of the next command */ + next_write_idx = (sc->sc_its_cmd_next_idx + 1) % + (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); + /* And the index of the current command being read */ + read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); + + /* + * The queue is full when the write offset points + * at the command before the current read offset. + */ + return (next_write_idx == read_idx); +} + +static inline void +its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) +{ + + if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { + /* Clean D-cache under command. */ + cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); + } else { + /* DSB inner shareable, store */ + dsb(ishst); + } + +} + +static inline uint64_t +its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) +{ + uint64_t off; + + off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); + + return (off); +} + +static void +its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, + struct its_cmd *cmd_last) +{ + struct gicv3_its_softc *sc; + uint64_t first, last, read; + size_t us_left; + + sc = device_get_softc(dev); + + /* + * XXX ARM64TODO: This is obviously a significant delay. + * The reason for that is that currently the time frames for + * the command to complete are not known. + */ + us_left = 1000000; + + first = its_cmd_cwriter_offset(sc, cmd_first); + last = its_cmd_cwriter_offset(sc, cmd_last); + + for (;;) { + read = gic_its_read_8(sc, GITS_CREADR); + if (first < last) { + if (read < first || read >= last) + break; + } else if (read < first && read >= last) + break; + + if (us_left-- == 0) { + /* This means timeout */ + device_printf(dev, + "Timeout while waiting for CMD completion.\n"); + return; + } + DELAY(1); + } +} + + +static struct its_cmd * +its_cmd_alloc_locked(device_t dev) +{ + struct gicv3_its_softc *sc; + struct its_cmd *cmd; + size_t us_left; + + sc = device_get_softc(dev); + + /* + * XXX ARM64TODO: This is obviously a significant delay. + * The reason for that is that currently the time frames for + * the command to complete (and therefore free the descriptor) + * are not known. + */ + us_left = 1000000; + + mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); + while (its_cmd_queue_full(sc)) { + if (us_left-- == 0) { + /* Timeout while waiting for free command */ + device_printf(dev, + "Timeout while waiting for free command\n"); + return (NULL); + } + DELAY(1); + } + + cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; + sc->sc_its_cmd_next_idx++; + sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); + + return (cmd); +} + +static uint64_t +its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) +{ + uint64_t target; + uint8_t cmd_type; + u_int size; + boolean_t error; + + error = FALSE; + cmd_type = desc->cmd_type; + target = ITS_TARGET_NONE; + + switch (cmd_type) { + case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ + target = desc->cmd_desc_movi.col->col_target; + cmd_format_command(cmd, ITS_CMD_MOVI); + cmd_format_id(cmd, desc->cmd_desc_movi.id); + cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); + cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); + break; + case ITS_CMD_SYNC: /* Wait for previous commands completion */ + target = desc->cmd_desc_sync.col->col_target; + cmd_format_command(cmd, ITS_CMD_SYNC); + cmd_format_target(cmd, target); + break; + case ITS_CMD_MAPD: /* Assign ITT to device */ + cmd_format_command(cmd, ITS_CMD_MAPD); + cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); + /* + * Size describes number of bits to encode interrupt IDs + * supported by the device minus one. + * When V (valid) bit is zero, this field should be written + * as zero. + */ + if (desc->cmd_desc_mapd.valid != 0) { + size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); + size = MAX(1, size) - 1; + } else + size = 0; + + cmd_format_size(cmd, size); + cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); + cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); + break; + case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ + target = desc->cmd_desc_mapc.col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPC); + cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); + cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); + cmd_format_target(cmd, target); + break; + case ITS_CMD_MAPTI: + target = desc->cmd_desc_mapvi.col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPTI); + cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_mapvi.id); + cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); + cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); + break; + case ITS_CMD_MAPI: + target = desc->cmd_desc_mapi.col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPI); + cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_mapi.pid); + cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); + break; + case ITS_CMD_INV: + target = desc->cmd_desc_inv.col->col_target; + cmd_format_command(cmd, ITS_CMD_INV); + cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_inv.pid); + break; + case ITS_CMD_INVALL: + cmd_format_command(cmd, ITS_CMD_INVALL); + cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); + break; + default: + panic("%s: Invalid command: %x", __func__, cmd_type); + } + + return (target); +} + +static int +its_cmd_send(device_t dev, struct its_cmd_desc *desc) +{ + struct gicv3_its_softc *sc; + struct its_cmd *cmd, *cmd_sync, *cmd_write; + struct its_col col_sync; + struct its_cmd_desc desc_sync; + uint64_t target, cwriter; + + sc = device_get_softc(dev); + mtx_lock_spin(&sc->sc_its_cmd_lock); + cmd = its_cmd_alloc_locked(dev); + if (cmd == NULL) { + device_printf(dev, "could not allocate ITS command\n"); + mtx_unlock_spin(&sc->sc_its_cmd_lock); + return (EBUSY); + } + + target = its_cmd_prepare(cmd, desc); + its_cmd_sync(sc, cmd); + + if (target != ITS_TARGET_NONE) { + cmd_sync = its_cmd_alloc_locked(dev); + if (cmd_sync != NULL) { + desc_sync.cmd_type = ITS_CMD_SYNC; + col_sync.col_target = target; + desc_sync.cmd_desc_sync.col = &col_sync; + its_cmd_prepare(cmd_sync, &desc_sync); + its_cmd_sync(sc, cmd_sync); + } + } + + /* Update GITS_CWRITER */ + cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); + gic_its_write_8(sc, GITS_CWRITER, cwriter); + cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; + mtx_unlock_spin(&sc->sc_its_cmd_lock); + + its_cmd_wait_completion(dev, cmd, cmd_write); + + return (0); +} + +/* Handlers to send commands */ +static void +its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) +{ + struct gicv3_its_softc *sc; + struct its_cmd_desc desc; + struct its_col *col; + + sc = device_get_softc(dev); + col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; + + desc.cmd_type = ITS_CMD_MOVI; + desc.cmd_desc_movi.its_dev = girq->gi_its_dev; + desc.cmd_desc_movi.col = col; + desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; + + its_cmd_send(dev, &desc); +} + +static void +its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPC; + desc.cmd_desc_mapc.col = col; + /* + * Valid bit set - map the collection. + * Valid bit cleared - unmap the collection. + */ + desc.cmd_desc_mapc.valid = valid; + + its_cmd_send(dev, &desc); +} + +static void +its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) +{ + struct gicv3_its_softc *sc; + struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + + sc = device_get_softc(dev); + + col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; + col = sc->sc_its_cols[col_id]; + + desc.cmd_type = ITS_CMD_MAPTI; + desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; + desc.cmd_desc_mapvi.col = col; + /* The ID sent to the device */ + desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; + /* The physical interrupt presented to softeware */ + desc.cmd_desc_mapvi.pid = girq->gi_irq + GIC_FIRST_LPI; + + its_cmd_send(dev, &desc); +} + +#if 0 +static void __unused +its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) +{ + struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + uint32_t id; + + KASSERT(pid >= its_dev->lpis.lpi_base, + ("%s: invalid pid: %d for the ITS device", __func__, pid)); + id = pid - its_dev->lpis.lpi_base; + col_id = its_dev->lpis.lpi_col_ids[id]; + col = sc->its_cols[col_id]; + + desc.cmd_type = ITS_CMD_MAPI; + desc.cmd_desc_mapi.its_dev = its_dev; + desc.cmd_desc_mapi.col = col; + desc.cmd_desc_mapi.pid = pid; + + its_cmd_send(sc, &desc); +} +#endif + +static void +its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPD; + desc.cmd_desc_mapd.its_dev = its_dev; + desc.cmd_desc_mapd.valid = valid; + + its_cmd_send(dev, &desc); +} + +static void +its_cmd_inv(device_t dev, struct its_dev *its_dev, uint32_t pid) +{ + struct gicv3_its_softc *sc; + struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + uint32_t id; + + KASSERT(pid >= its_dev->lpis.lpi_base, + ("%s: invalid pid: %d for the ITS device", __func__, pid)); + + sc = device_get_softc(dev); + id = pid - its_dev->lpis.lpi_base; + //col_id = its_dev->lpis.lpi_col_ids[id]; + col_id = 0; + col = sc->sc_its_cols[col_id]; + + desc.cmd_type = ITS_CMD_INV; + desc.cmd_desc_inv.pid = pid - its_dev->lpis.lpi_base; + desc.cmd_desc_inv.its_dev = its_dev; + desc.cmd_desc_inv.col = col; + + its_cmd_send(dev, &desc); +} + +static void +its_cmd_invall(device_t dev, struct its_col *col) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_INVALL; + desc.cmd_desc_invall.col = col; + + its_cmd_send(dev, &desc); +} + +#ifdef FDT +static device_probe_t gicv3_its_fdt_probe; +static device_attach_t gicv3_its_fdt_attach; + +static device_method_t gicv3_its_fdt_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, gicv3_its_fdt_probe), + DEVMETHOD(device_attach, gicv3_its_fdt_attach), + + /* End */ + DEVMETHOD_END +}; + +#define its_baseclasses its_fdt_baseclasses +DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, + sizeof(struct gicv3_its_softc), gicv3_its_driver); +#undef its_baseclasses +static devclass_t gicv3_its_fdt_devclass; + +EARLY_DRIVER_MODULE(its, gic, gicv3_its_fdt_driver, + gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); + +static int +gicv3_its_fdt_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) + return (ENXIO); + + device_set_desc(dev, "ARM GIC Interrupt Translation Service"); + return (BUS_PROBE_DEFAULT); +} + +static int +gicv3_its_fdt_attach(device_t dev) +{ + struct gicv3_its_softc *sc; + phandle_t xref; + int err; + + err = gicv3_its_attach(dev); + if (err != 0) + return (err); + + sc = device_get_softc(dev); + + /* Register this device as a interrupt controller */ + xref = OF_xref_from_node(ofw_bus_get_node(dev)); + sc->sc_pic = intr_pic_register(dev, xref); + intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, + gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS); + + /* Register this device to handle MSI interrupts */ + intr_msi_register(dev, xref); + + return (0); +} +#endif Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -26,6 +26,7 @@ arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/gic.c optional !intrng +arm64/arm64/gicv3_its.c optional intrng arm64/arm64/gic_acpi.c optional !intrng acpi arm64/arm64/gic_fdt.c optional !intrng fdt arm64/arm64/gic_v3.c standard