Index: head/sys/arm64/arm64/gic_v3.c =================================================================== --- head/sys/arm64/arm64/gic_v3.c (revision 299943) +++ head/sys/arm64/arm64/gic_v3.c (revision 299944) @@ -1,745 +1,1331 @@ /*- - * Copyright (c) 2015 The FreeBSD Foundation + * Copyright (c) 2015-2016 The FreeBSD Foundation * All rights reserved. * + * This software was developed by Andrew Turner under + * the sponsorship of the FreeBSD Foundation. + * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include "opt_platform.h" + #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #include "gic_v3_reg.h" #include "gic_v3_var.h" +#ifdef INTRNG +static pic_disable_intr_t gic_v3_disable_intr; +static pic_enable_intr_t gic_v3_enable_intr; +static pic_map_intr_t gic_v3_map_intr; +static pic_setup_intr_t gic_v3_setup_intr; +static pic_teardown_intr_t gic_v3_teardown_intr; +static pic_post_filter_t gic_v3_post_filter; +static pic_post_ithread_t gic_v3_post_ithread; +static pic_pre_ithread_t gic_v3_pre_ithread; +static pic_bind_intr_t gic_v3_bind_intr; +#ifdef SMP +static pic_init_secondary_t gic_v3_init_secondary; +static pic_ipi_send_t gic_v3_ipi_send; +static pic_ipi_setup_t gic_v3_ipi_setup; +#endif + +static u_int gic_irq_cpu; +#ifdef SMP +static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; +static u_int sgi_first_unused = GIC_FIRST_SGI; +#endif +#else /* Device and PIC methods */ static int gic_v3_bind(device_t, u_int, u_int); static void gic_v3_dispatch(device_t, struct trapframe *); static void gic_v3_eoi(device_t, u_int); static void gic_v3_mask_irq(device_t, u_int); static void gic_v3_unmask_irq(device_t, u_int); #ifdef SMP static void gic_v3_init_secondary(device_t); static void gic_v3_ipi_send(device_t, cpuset_t, u_int); #endif +#endif static device_method_t gic_v3_methods[] = { /* Device interface */ DEVMETHOD(device_detach, gic_v3_detach), +#ifdef INTRNG + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, gic_v3_disable_intr), + DEVMETHOD(pic_enable_intr, gic_v3_enable_intr), + DEVMETHOD(pic_map_intr, gic_v3_map_intr), + DEVMETHOD(pic_setup_intr, gic_v3_setup_intr), + DEVMETHOD(pic_teardown_intr, gic_v3_teardown_intr), + DEVMETHOD(pic_post_filter, gic_v3_post_filter), + DEVMETHOD(pic_post_ithread, gic_v3_post_ithread), + DEVMETHOD(pic_pre_ithread, gic_v3_pre_ithread), +#ifdef SMP + DEVMETHOD(pic_bind_intr, gic_v3_bind_intr), + DEVMETHOD(pic_init_secondary, gic_v3_init_secondary), + DEVMETHOD(pic_ipi_send, gic_v3_ipi_send), + DEVMETHOD(pic_ipi_setup, gic_v3_ipi_setup), +#endif +#else /* PIC interface */ DEVMETHOD(pic_bind, gic_v3_bind), DEVMETHOD(pic_dispatch, gic_v3_dispatch), DEVMETHOD(pic_eoi, gic_v3_eoi), DEVMETHOD(pic_mask, gic_v3_mask_irq), DEVMETHOD(pic_unmask, gic_v3_unmask_irq), #ifdef SMP DEVMETHOD(pic_init_secondary, gic_v3_init_secondary), DEVMETHOD(pic_ipi_send, gic_v3_ipi_send), #endif +#endif + /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(gic, gic_v3_driver, gic_v3_methods, sizeof(struct gic_v3_softc)); /* * Driver-specific definitions. */ MALLOC_DEFINE(M_GIC_V3, "GICv3", GIC_V3_DEVSTR); /* * Helper functions and definitions. */ /* Destination registers, either Distributor or Re-Distributor */ enum gic_v3_xdist { DIST = 0, REDIST, }; /* Helper routines starting with gic_v3_ */ static int gic_v3_dist_init(struct gic_v3_softc *); static int gic_v3_redist_alloc(struct gic_v3_softc *); static int gic_v3_redist_find(struct gic_v3_softc *); static int gic_v3_redist_init(struct gic_v3_softc *); static int gic_v3_cpu_init(struct gic_v3_softc *); static void gic_v3_wait_for_rwp(struct gic_v3_softc *, enum gic_v3_xdist); /* A sequence of init functions for primary (boot) CPU */ typedef int (*gic_v3_initseq_t) (struct gic_v3_softc *); /* Primary CPU initialization sequence */ static gic_v3_initseq_t gic_v3_primary_init[] = { gic_v3_dist_init, gic_v3_redist_alloc, gic_v3_redist_init, gic_v3_cpu_init, NULL }; #ifdef SMP /* Secondary CPU initialization sequence */ static gic_v3_initseq_t gic_v3_secondary_init[] = { gic_v3_redist_init, gic_v3_cpu_init, NULL }; #endif /* * Device interface. */ int gic_v3_attach(device_t dev) { struct gic_v3_softc *sc; gic_v3_initseq_t *init_func; uint32_t typer; int rid; int err; size_t i; +#ifdef INTRNG + u_int irq; + const char *name; +#endif sc = device_get_softc(dev); sc->gic_registered = FALSE; sc->dev = dev; err = 0; /* Initialize mutex */ mtx_init(&sc->gic_mtx, "GICv3 lock", NULL, MTX_SPIN); /* * Allocate array of struct resource. * One entry for Distributor and all remaining for Re-Distributor. */ sc->gic_res = malloc( sizeof(*sc->gic_res) * (sc->gic_redists.nregions + 1), M_GIC_V3, M_WAITOK); /* Now allocate corresponding resources */ for (i = 0, rid = 0; i < (sc->gic_redists.nregions + 1); i++, rid++) { sc->gic_res[rid] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->gic_res[rid] == NULL) return (ENXIO); } /* * Distributor interface */ sc->gic_dist = sc->gic_res[0]; /* * Re-Dristributor interface */ /* Allocate space under region descriptions */ sc->gic_redists.regions = malloc( sizeof(*sc->gic_redists.regions) * sc->gic_redists.nregions, M_GIC_V3, M_WAITOK); /* Fill-up bus_space information for each region. */ for (i = 0, rid = 1; i < sc->gic_redists.nregions; i++, rid++) sc->gic_redists.regions[i] = sc->gic_res[rid]; /* Get the number of supported SPI interrupts */ typer = gic_d_read(sc, 4, GICD_TYPER); sc->gic_nirqs = GICD_TYPER_I_NUM(typer); if (sc->gic_nirqs > GIC_I_NUM_MAX) sc->gic_nirqs = GIC_I_NUM_MAX; +#ifdef INTRNG + sc->gic_irqs = malloc(sizeof(*sc->gic_irqs) * sc->gic_nirqs, + M_GIC_V3, M_WAITOK | M_ZERO); + name = device_get_nameunit(dev); + for (irq = 0; irq < sc->gic_nirqs; irq++) { + struct intr_irqsrc *isrc; + + sc->gic_irqs[irq].gi_irq = irq; + sc->gic_irqs[irq].gi_pol = INTR_POLARITY_CONFORM; + sc->gic_irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; + + isrc = &sc->gic_irqs[irq].gi_isrc; + if (irq <= GIC_LAST_SGI) { + err = intr_isrc_register(isrc, sc->dev, + INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); + } else if (irq <= GIC_LAST_PPI) { + err = intr_isrc_register(isrc, sc->dev, + INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); + } else { + err = intr_isrc_register(isrc, sc->dev, 0, + "%s,s%u", name, irq - GIC_FIRST_SPI); + } + if (err != 0) { + /* XXX call intr_isrc_deregister() */ + free(irqs, M_DEVBUF); + return (err); + } + } +#endif + /* Get the number of supported interrupt identifier bits */ sc->gic_idbits = GICD_TYPER_IDBITS(typer); if (bootverbose) { device_printf(dev, "SPIs: %u, IDs: %u\n", sc->gic_nirqs, (1 << sc->gic_idbits) - 1); } /* Train init sequence for boot CPU */ for (init_func = gic_v3_primary_init; *init_func != NULL; init_func++) { err = (*init_func)(sc); if (err != 0) return (err); } /* * Full success. * Now register PIC to the interrupts handling layer. */ +#ifndef INTRNG arm_register_root_pic(dev, sc->gic_nirqs); sc->gic_registered = TRUE; +#endif return (0); } int gic_v3_detach(device_t dev) { struct gic_v3_softc *sc; size_t i; int rid; sc = device_get_softc(dev); if (device_is_attached(dev)) { /* * XXX: We should probably deregister PIC */ if (sc->gic_registered) panic("Trying to detach registered PIC"); } for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); for (i = 0; i < mp_ncpus; i++) free(sc->gic_redists.pcpu[i], M_GIC_V3); free(sc->gic_res, M_GIC_V3); free(sc->gic_redists.regions, M_GIC_V3); return (0); } +#ifdef INTRNG +int +arm_gic_v3_intr(void *arg) +{ + struct gic_v3_softc *sc = arg; + struct gic_v3_irqsrc *gi; + uint64_t active_irq; + struct trapframe *tf; + bool first; + + first = true; + + while (1) { + if (CPU_MATCH_ERRATA_CAVIUM_THUNDER_1_1) { + /* + * Hardware: Cavium ThunderX + * Chip revision: Pass 1.0 (early version) + * Pass 1.1 (production) + * ERRATUM: 22978, 23154 + */ + __asm __volatile( + "nop;nop;nop;nop;nop;nop;nop;nop; \n" + "mrs %0, ICC_IAR1_EL1 \n" + "nop;nop;nop;nop; \n" + "dsb sy \n" + : "=&r" (active_irq)); + } else { + active_irq = gic_icc_read(IAR1); + } + + if (__predict_false(active_irq >= sc->gic_nirqs)) + return (FILTER_HANDLED); + + tf = curthread->td_intr_frame; + gi = &sc->gic_irqs[active_irq]; + if (active_irq <= GIC_LAST_SGI) { + /* Call EOI for all IPI before dispatch. */ + gic_icc_write(EOIR1, (uint64_t)active_irq); +#ifdef SMP + intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); +#else + device_printf(sc->dev, "SGI %u on UP system detected\n", + active_irq - GIC_FIRST_SGI); +#endif + } else if (active_irq >= GIC_FIRST_PPI && + active_irq <= GIC_LAST_SPI) { + if (gi->gi_pol == INTR_TRIGGER_EDGE) + gic_icc_write(EOIR1, gi->gi_irq); + + if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { + if (gi->gi_pol != INTR_TRIGGER_EDGE) + gic_icc_write(EOIR1, gi->gi_irq); + gic_v3_disable_intr(sc->dev, &gi->gi_isrc); + device_printf(sc->dev, + "Stray irq %lu disabled\n", active_irq); + } + } + } +} + +#ifdef FDT +static int +gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, + enum intr_polarity *polp, enum intr_trigger *trigp) +{ + u_int irq; + + if (ncells < 3) + return (EINVAL); + + /* + * The 1st cell is the interrupt type: + * 0 = SPI + * 1 = PPI + * The 2nd cell contains the interrupt number: + * [0 - 987] for SPI + * [0 - 15] for PPI + * The 3rd cell is the flags, encoded as follows: + * bits[3:0] trigger type and level flags + * 1 = edge triggered + * 2 = edge triggered (PPI only) + * 4 = level-sensitive + * 8 = level-sensitive (PPI only) + */ + switch (cells[0]) { + case 0: + irq = GIC_FIRST_SPI + cells[1]; + /* SPI irq is checked later. */ + break; + case 1: + irq = GIC_FIRST_PPI + cells[1]; + if (irq > GIC_LAST_PPI) { + device_printf(dev, "unsupported PPI interrupt " + "number %u\n", cells[1]); + return (EINVAL); + } + break; + default: + device_printf(dev, "unsupported interrupt type " + "configuration %u\n", cells[0]); + return (EINVAL); + } + + switch (cells[2] & 0xf) { + case 1: + *trigp = INTR_TRIGGER_EDGE; + *polp = INTR_POLARITY_HIGH; + break; + case 2: + *trigp = INTR_TRIGGER_EDGE; + *polp = INTR_POLARITY_LOW; + break; + case 4: + *trigp = INTR_TRIGGER_LEVEL; + *polp = INTR_POLARITY_HIGH; + break; + case 8: + *trigp = INTR_TRIGGER_LEVEL; + *polp = INTR_POLARITY_LOW; + break; + default: + device_printf(dev, "unsupported trigger/polarity " + "configuration 0x%02x\n", cells[2]); + return (EINVAL); + } + + /* Check the interrupt is valid */ + if (irq >= GIC_FIRST_SPI && *polp != INTR_POLARITY_HIGH) + return (EINVAL); + + *irqp = irq; + return (0); +} +#endif + +static int +do_gic_v3_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, + enum intr_polarity *polp, enum intr_trigger *trigp) +{ + struct gic_v3_softc *sc; + enum intr_polarity pol; + enum intr_trigger trig; +#ifdef FDT + struct intr_map_data_fdt *daf; +#endif + u_int irq; + + sc = device_get_softc(dev); + + switch (data->type) { +#ifdef FDT + case INTR_MAP_DATA_FDT: + daf = (struct intr_map_data_fdt *)data; + if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol, + &trig) != 0) + return (EINVAL); + break; +#endif + default: + return (EINVAL); + } + + if (irq >= sc->gic_nirqs) + return (EINVAL); + switch (pol) { + case INTR_POLARITY_CONFORM: + case INTR_POLARITY_LOW: + case INTR_POLARITY_HIGH: + break; + default: + return (EINVAL); + } + switch (trig) { + case INTR_TRIGGER_CONFORM: + case INTR_TRIGGER_EDGE: + case INTR_TRIGGER_LEVEL: + break; + default: + return (EINVAL); + } + + *irqp = irq; + if (polp != NULL) + *polp = pol; + if (trigp != NULL) + *trigp = trig; + return (0); +} + +static int +gic_v3_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct gic_v3_softc *sc; + int error; + u_int irq; + + error = do_gic_v3_map_intr(dev, data, &irq, NULL, NULL); + if (error == 0) { + sc = device_get_softc(dev); + *isrcp = GIC_INTR_ISRC(sc, irq); + } + return (error); +} + +static int +gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + struct gic_v3_softc *sc = device_get_softc(dev); + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + enum intr_trigger trig; + enum intr_polarity pol; + uint32_t reg; + u_int irq; + int error; + + if (data == NULL) + return (ENOTSUP); + + error = do_gic_v3_map_intr(dev, data, &irq, &pol, &trig); + if (error != 0) + return (error); + + if (gi->gi_irq != irq || pol == INTR_POLARITY_CONFORM || + trig == INTR_TRIGGER_CONFORM) + return (EINVAL); + + /* Compare config if this is not first setup. */ + if (isrc->isrc_handlers != 0) { + if (pol != gi->gi_pol || trig != gi->gi_trig) + return (EINVAL); + else + return (0); + } + + gi->gi_pol = pol; + gi->gi_trig = trig; + + /* + * XXX - In case that per CPU interrupt is going to be enabled in time + * when SMP is already started, we need some IPI call which + * enables it on others CPUs. Further, it's more complicated as + * pic_enable_source() and pic_disable_source() should act on + * per CPU basis only. Thus, it should be solved here somehow. + */ + if (isrc->isrc_flags & INTR_ISRCF_PPI) + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) { + mtx_lock_spin(&sc->gic_mtx); + + /* Set the trigger and polarity */ + if (irq <= GIC_LAST_PPI) + reg = gic_r_read(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICFGR(irq)); + else + reg = gic_d_read(sc, 4, GICD_ICFGR(irq)); + if (trig == INTR_TRIGGER_LEVEL) + reg &= ~(2 << ((irq % 16) * 2)); + else + reg |= 2 << ((irq % 16) * 2); + + if (irq <= GIC_LAST_PPI) { + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg); + gic_v3_wait_for_rwp(sc, REDIST); + } else { + gic_d_write(sc, 4, GICD_ICFGR(irq), reg); + gic_v3_wait_for_rwp(sc, DIST); + } + + mtx_unlock_spin(&sc->gic_mtx); + + gic_v3_bind_intr(dev, isrc); + } + + return (0); +} + +static int +gic_v3_teardown_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + + panic("gic_v3_teardown_intr"); +} + +static void +gic_v3_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gic_v3_softc *sc; + struct gic_v3_irqsrc *gi; + u_int irq; + + sc = device_get_softc(dev); + gi = (struct gic_v3_irqsrc *)isrc; + irq = gi->gi_irq; + + if (irq <= GIC_LAST_PPI) { + /* SGIs and PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), + GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { + /* SPIs in distributor */ + gic_d_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("gic_v3_disable_intr"); +} + +static void +gic_v3_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gic_v3_softc *sc; + struct gic_v3_irqsrc *gi; + u_int irq; + + sc = device_get_softc(dev); + gi = (struct gic_v3_irqsrc *)isrc; + irq = gi->gi_irq; + + if (irq <= GIC_LAST_PPI) { + /* SGIs and PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), + GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { + /* SPIs in distributor */ + gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("gic_v3_enable_intr"); +} + +static void +gic_v3_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + + gic_v3_disable_intr(dev, isrc); + gic_icc_write(EOIR1, gi->gi_irq); +} + +static void +gic_v3_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + + gic_v3_enable_intr(dev, isrc); +} + +static void +gic_v3_post_filter(device_t dev, struct intr_irqsrc *isrc) +{ + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + + if (gi->gi_pol == INTR_TRIGGER_EDGE) + return; + + gic_icc_write(EOIR1, gi->gi_irq); +} + +static int +gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct gic_v3_softc *sc; + struct gic_v3_irqsrc *gi; + int cpu; + + gi = (struct gic_v3_irqsrc *)isrc; + if (gi->gi_irq <= GIC_LAST_PPI) + return (EINVAL); + + KASSERT(gi->gi_irq >= GIC_FIRST_SPI && gi->gi_irq <= GIC_LAST_SPI, + ("%s: Attempting to bind an invalid IRQ", __func__)); + + sc = device_get_softc(dev); + + if (CPU_EMPTY(&isrc->isrc_cpu)) { + gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); + CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); + gic_d_write(sc, 4, GICD_IROUTER(gi->gi_irq), + CPU_AFFINITY(gic_irq_cpu)); + } else { + /* + * We can only bind to a single CPU so select + * the first CPU found. + */ + cpu = CPU_FFS(&isrc->isrc_cpu) - 1; + gic_d_write(sc, 4, GICD_IROUTER(gi->gi_irq), CPU_AFFINITY(cpu)); + } + + return (0); +} + +#ifdef SMP +static void +gic_v3_init_secondary(device_t dev) +{ + struct gic_v3_softc *sc; + gic_v3_initseq_t *init_func; + struct intr_irqsrc *isrc; + u_int cpu, irq; + int err; + + sc = device_get_softc(dev); + cpu = PCPU_GET(cpuid); + + /* Train init sequence for boot CPU */ + for (init_func = gic_v3_secondary_init; *init_func != NULL; + init_func++) { + err = (*init_func)(sc); + if (err != 0) { + device_printf(dev, + "Could not initialize GIC for CPU%u\n", cpu); + return; + } + } + + /* Unmask attached SGI interrupts. */ + for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { + isrc = GIC_INTR_ISRC(sc, irq); + if (intr_isrc_init_on_cpu(isrc, cpu)) + gic_v3_enable_intr(dev, isrc); + } + + /* Unmask attached PPI interrupts. */ + for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { + isrc = GIC_INTR_ISRC(sc, irq); + if (intr_isrc_init_on_cpu(isrc, cpu)) + gic_v3_enable_intr(dev, isrc); + } +} + +static void +gic_v3_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; + uint64_t aff, val, irq; + int i; + +#define GIC_AFF_MASK (CPU_AFF3_MASK | CPU_AFF2_MASK | CPU_AFF1_MASK) +#define GIC_AFFINITY(i) (CPU_AFFINITY(i) & GIC_AFF_MASK) + aff = GIC_AFFINITY(0); + irq = gi->gi_irq; + val = 0; + + /* Iterate through all CPUs in set */ + for (i = 0; i < mp_ncpus; i++) { + /* Move to the next affinity group */ + if (aff != GIC_AFFINITY(i)) { + /* Send the IPI */ + if (val != 0) { + gic_icc_write(SGI1R, val); + val = 0; + } + aff = GIC_AFFINITY(i); + } + + /* Send the IPI to this cpu */ + if (CPU_ISSET(i, &cpus)) { +#define ICC_SGI1R_AFFINITY(aff) \ + (((uint64_t)CPU_AFF3(aff) << ICC_SGI1R_EL1_AFF3_SHIFT) | \ + ((uint64_t)CPU_AFF2(aff) << ICC_SGI1R_EL1_AFF2_SHIFT) | \ + ((uint64_t)CPU_AFF1(aff) << ICC_SGI1R_EL1_AFF1_SHIFT)) + /* Set the affinity when the first at this level */ + if (val == 0) + val = ICC_SGI1R_AFFINITY(aff) | + irq << ICC_SGI1R_EL1_SGIID_SHIFT; + /* Set the bit to send the IPI to te CPU */ + val |= 1 << CPU_AFF0(CPU_AFFINITY(i)); + } + } + + /* Send the IPI to the last cpu affinity group */ + if (val != 0) + gic_icc_write(SGI1R, val); +#undef GIC_AFF_MASK +#undef GIC_AFFINITY +} + +static int +gic_v3_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct intr_irqsrc *isrc; + struct gic_v3_softc *sc = device_get_softc(dev); + + if (sgi_first_unused > GIC_LAST_SGI) + return (ENOSPC); + + isrc = GIC_INTR_ISRC(sc, sgi_first_unused); + sgi_to_ipi[sgi_first_unused++] = ipi; + + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + *isrcp = isrc; + return (0); +} +#endif /* SMP */ +#else /* INTRNG */ /* * PIC interface. */ static int gic_v3_bind(device_t dev, u_int irq, u_int cpuid) { uint64_t aff; struct gic_v3_softc *sc; sc = device_get_softc(dev); if (irq <= GIC_LAST_PPI) { /* Can't bind PPI to another CPU but it's not an error */ return (0); } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { aff = CPU_AFFINITY(cpuid); gic_d_write(sc, 4, GICD_IROUTER(irq), aff); return (0); } else if (irq >= GIC_FIRST_LPI) return (lpi_migrate(dev, irq, cpuid)); return (EINVAL); } static void gic_v3_dispatch(device_t dev, struct trapframe *frame) { uint64_t active_irq; while (1) { if (CPU_MATCH_ERRATA_CAVIUM_THUNDER_1_1) { /* * Hardware: Cavium ThunderX * Chip revision: Pass 1.0 (early version) * Pass 1.1 (production) * ERRATUM: 22978, 23154 */ __asm __volatile( "nop;nop;nop;nop;nop;nop;nop;nop; \n" "mrs %0, ICC_IAR1_EL1 \n" "nop;nop;nop;nop; \n" "dsb sy \n" : "=&r" (active_irq)); } else { active_irq = gic_icc_read(IAR1); } if (__predict_false(active_irq == ICC_IAR1_EL1_SPUR)) break; if (__predict_true((active_irq >= GIC_FIRST_PPI && active_irq <= GIC_LAST_SPI) || active_irq >= GIC_FIRST_LPI)) { arm_dispatch_intr(active_irq, frame); continue; } if (active_irq <= GIC_LAST_SGI) { gic_icc_write(EOIR1, (uint64_t)active_irq); arm_dispatch_intr(active_irq, frame); continue; } } } static void gic_v3_eoi(device_t dev, u_int irq) { gic_icc_write(EOIR1, (uint64_t)irq); } static void gic_v3_mask_irq(device_t dev, u_int irq) { struct gic_v3_softc *sc; sc = device_get_softc(dev); if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_r_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, DIST); } else if (irq >= GIC_FIRST_LPI) { /* LPIs */ lpi_mask_irq(dev, irq); } else panic("%s: Unsupported IRQ number %u", __func__, irq); } static void gic_v3_unmask_irq(device_t dev, u_int irq) { struct gic_v3_softc *sc; sc = device_get_softc(dev); if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, DIST); } else if (irq >= GIC_FIRST_LPI) { /* LPIs */ lpi_unmask_irq(dev, irq); } else panic("%s: Unsupported IRQ number %u", __func__, irq); } #ifdef SMP static void gic_v3_init_secondary(device_t dev) { struct gic_v3_softc *sc; gic_v3_initseq_t *init_func; int err; sc = device_get_softc(dev); /* Train init sequence for boot CPU */ for (init_func = gic_v3_secondary_init; *init_func != NULL; init_func++) { err = (*init_func)(sc); if (err != 0) { device_printf(dev, "Could not initialize GIC for CPU%u\n", PCPU_GET(cpuid)); return; } } /* * Try to initialize ITS. * If there is no driver attached this routine will fail but that * does not mean failure here as only LPIs will not be functional * on the current CPU. */ if (its_init_cpu(NULL) != 0) { device_printf(dev, "Could not initialize ITS for CPU%u. " "No LPIs will arrive on this CPU\n", PCPU_GET(cpuid)); } /* * ARM64TODO: Unmask timer PPIs. To be removed when appropriate * mechanism is implemented. * Activate the timer interrupts: virtual (27), secure (29), * and non-secure (30). Use hardcoded values here as there * should be no defines for them. */ gic_v3_unmask_irq(dev, 27); gic_v3_unmask_irq(dev, 29); gic_v3_unmask_irq(dev, 30); } static void gic_v3_ipi_send(device_t dev, cpuset_t cpuset, u_int ipi) { u_int cpu; uint64_t aff, tlist; uint64_t val; uint64_t aff_mask; /* Set affinity mask to match level 3, 2 and 1 */ aff_mask = CPU_AFF1_MASK | CPU_AFF2_MASK | CPU_AFF3_MASK; /* Iterate through all CPUs in set */ while (!CPU_EMPTY(&cpuset)) { aff = tlist = 0; for (cpu = 0; cpu < mp_ncpus; cpu++) { /* Compose target list for single AFF3:AFF2:AFF1 set */ if (CPU_ISSET(cpu, &cpuset)) { if (!tlist) { /* * Save affinity of the first CPU to * send IPI to for later comparison. */ aff = CPU_AFFINITY(cpu); tlist |= (1UL << CPU_AFF0(aff)); CPU_CLR(cpu, &cpuset); } /* Check for same Affinity level 3, 2 and 1 */ if ((aff & aff_mask) == (CPU_AFFINITY(cpu) & aff_mask)) { tlist |= (1UL << CPU_AFF0(CPU_AFFINITY(cpu))); /* Clear CPU in cpuset from target list */ CPU_CLR(cpu, &cpuset); } } } if (tlist) { KASSERT((tlist & ~ICC_SGI1R_EL1_TL_MASK) == 0, ("Target list too long for GICv3 IPI")); /* Send SGI to CPUs in target list */ val = tlist; val |= (uint64_t)CPU_AFF3(aff) << ICC_SGI1R_EL1_AFF3_SHIFT; val |= (uint64_t)CPU_AFF2(aff) << ICC_SGI1R_EL1_AFF2_SHIFT; val |= (uint64_t)CPU_AFF1(aff) << ICC_SGI1R_EL1_AFF1_SHIFT; val |= (uint64_t)(ipi & ICC_SGI1R_EL1_SGIID_MASK) << ICC_SGI1R_EL1_SGIID_SHIFT; gic_icc_write(SGI1R, val); } } } #endif +#endif /* !INTRNG */ /* * Helper routines */ static void gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) { struct resource *res; u_int cpuid; size_t us_left = 1000000; cpuid = PCPU_GET(cpuid); switch (xdist) { case DIST: res = sc->gic_dist; break; case REDIST: res = sc->gic_redists.pcpu[cpuid]; break; default: KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__)); return; } while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) { DELAY(1); if (us_left-- == 0) panic("GICD Register write pending for too long"); } } /* CPU interface. */ static __inline void gic_v3_cpu_priority(uint64_t mask) { /* Set prority mask */ gic_icc_write(PMR, mask & ICC_PMR_EL1_PRIO_MASK); } static int gic_v3_cpu_enable_sre(struct gic_v3_softc *sc) { uint64_t sre; u_int cpuid; cpuid = PCPU_GET(cpuid); /* * Set the SRE bit to enable access to GIC CPU interface * via system registers. */ sre = READ_SPECIALREG(icc_sre_el1); sre |= ICC_SRE_EL1_SRE; WRITE_SPECIALREG(icc_sre_el1, sre); isb(); /* * Now ensure that the bit is set. */ sre = READ_SPECIALREG(icc_sre_el1); if ((sre & ICC_SRE_EL1_SRE) == 0) { /* We are done. This was disabled in EL2 */ device_printf(sc->dev, "ERROR: CPU%u cannot enable CPU interface " "via system registers\n", cpuid); return (ENXIO); } else if (bootverbose) { device_printf(sc->dev, "CPU%u enabled CPU interface via system registers\n", cpuid); } return (0); } static int gic_v3_cpu_init(struct gic_v3_softc *sc) { int err; /* Enable access to CPU interface via system registers */ err = gic_v3_cpu_enable_sre(sc); if (err != 0) return (err); /* Priority mask to minimum - accept all interrupts */ gic_v3_cpu_priority(GIC_PRIORITY_MIN); /* Disable EOI mode */ gic_icc_clear(CTLR, ICC_CTLR_EL1_EOIMODE); /* Enable group 1 (insecure) interrups */ gic_icc_set(IGRPEN1, ICC_IGRPEN0_EL1_EN); return (0); } /* Distributor */ static int gic_v3_dist_init(struct gic_v3_softc *sc) { uint64_t aff; u_int i; /* * 1. Disable the Distributor */ gic_d_write(sc, 4, GICD_CTLR, 0); gic_v3_wait_for_rwp(sc, DIST); /* * 2. Configure the Distributor */ /* Set all global interrupts to be level triggered, active low. */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ICFGRn) gic_d_write(sc, 4, GICD_ICFGR(i), 0x00000000); /* Set priority to all shared interrupts */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_IPRIORITYn) { /* Set highest priority */ gic_d_write(sc, 4, GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); } /* * Disable all interrupts. Leave PPI and SGIs as they are enabled in * Re-Distributor registers. */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ISENABLERn) gic_d_write(sc, 4, GICD_ICENABLER(i), 0xFFFFFFFF); gic_v3_wait_for_rwp(sc, DIST); /* * 3. Enable Distributor */ /* Enable Distributor with ARE, Group 1 */ gic_d_write(sc, 4, GICD_CTLR, GICD_CTLR_ARE_NS | GICD_CTLR_G1A | GICD_CTLR_G1); /* * 4. Route all interrupts to boot CPU. */ aff = CPU_AFFINITY(0); for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i++) gic_d_write(sc, 4, GICD_IROUTER(i), aff); return (0); } /* Re-Distributor */ static int gic_v3_redist_alloc(struct gic_v3_softc *sc) { u_int cpuid; /* Allocate struct resource for all CPU's Re-Distributor registers */ for (cpuid = 0; cpuid < mp_ncpus; cpuid++) if (CPU_ISSET(cpuid, &all_cpus) != 0) sc->gic_redists.pcpu[cpuid] = malloc(sizeof(*sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); else sc->gic_redists.pcpu[cpuid] = NULL; return (0); } static int gic_v3_redist_find(struct gic_v3_softc *sc) { struct resource r_res; bus_space_handle_t r_bsh; uint64_t aff; uint64_t typer; uint32_t pidr2; u_int cpuid; size_t i; cpuid = PCPU_GET(cpuid); aff = CPU_AFFINITY(cpuid); /* Affinity in format for comparison with typer */ aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) | (CPU_AFF1(aff) << 8) | CPU_AFF0(aff); if (bootverbose) { device_printf(sc->dev, "Start searching for Re-Distributor\n"); } /* Iterate through Re-Distributor regions */ for (i = 0; i < sc->gic_redists.nregions; i++) { /* Take a copy of the region's resource */ r_res = *sc->gic_redists.regions[i]; r_bsh = rman_get_bushandle(&r_res); pidr2 = bus_read_4(&r_res, GICR_PIDR2); switch (pidr2 & GICR_PIDR2_ARCH_MASK) { case GICR_PIDR2_ARCH_GICv3: /* fall through */ case GICR_PIDR2_ARCH_GICv4: break; default: device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); return (ENODEV); } do { typer = bus_read_8(&r_res, GICR_TYPER); if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) { KASSERT(sc->gic_redists.pcpu[cpuid] != NULL, ("Invalid pointer to per-CPU redistributor")); /* Copy res contents to its final destination */ *sc->gic_redists.pcpu[cpuid] = r_res; if (bootverbose) { device_printf(sc->dev, "CPU%u Re-Distributor has been found\n", cpuid); } return (0); } r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); if ((typer & GICR_TYPER_VLPIS) != 0) { r_bsh += (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE); } rman_set_bushandle(&r_res, r_bsh); } while ((typer & GICR_TYPER_LAST) == 0); } device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); return (ENXIO); } static int gic_v3_redist_wake(struct gic_v3_softc *sc) { uint32_t waker; size_t us_left = 1000000; waker = gic_r_read(sc, 4, GICR_WAKER); /* Wake up Re-Distributor for this CPU */ waker &= ~GICR_WAKER_PS; gic_r_write(sc, 4, GICR_WAKER, waker); /* * When clearing ProcessorSleep bit it is required to wait for * ChildrenAsleep to become zero following the processor power-on. */ while ((gic_r_read(sc, 4, GICR_WAKER) & GICR_WAKER_CA) != 0) { DELAY(1); if (us_left-- == 0) { panic("Could not wake Re-Distributor for CPU%u", PCPU_GET(cpuid)); } } if (bootverbose) { device_printf(sc->dev, "CPU%u Re-Distributor woke up\n", PCPU_GET(cpuid)); } return (0); } static int gic_v3_redist_init(struct gic_v3_softc *sc) { int err; size_t i; err = gic_v3_redist_find(sc); if (err != 0) return (err); err = gic_v3_redist_wake(sc); if (err != 0) return (err); /* Disable SPIs */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ICENABLER0, GICR_I_ENABLER_PPI_MASK); /* Enable SGIs */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ISENABLER0, GICR_I_ENABLER_SGI_MASK); /* Set priority for SGIs and PPIs */ for (i = 0; i <= GIC_LAST_PPI; i += GICR_I_PER_IPRIORITYn) { gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); } gic_v3_wait_for_rwp(sc, REDIST); return (0); } Index: head/sys/arm64/arm64/gic_v3_fdt.c =================================================================== --- head/sys/arm64/arm64/gic_v3_fdt.c (revision 299943) +++ head/sys/arm64/arm64/gic_v3_fdt.c (revision 299944) @@ -1,312 +1,330 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include +#include #include #include #include #include -#include "pic_if.h" - #include "gic_v3_reg.h" #include "gic_v3_var.h" /* * FDT glue. */ static int gic_v3_fdt_probe(device_t); static int gic_v3_fdt_attach(device_t); static struct resource *gic_v3_ofw_bus_alloc_res(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static const struct ofw_bus_devinfo *gic_v3_ofw_get_devinfo(device_t, device_t); static device_method_t gic_v3_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gic_v3_fdt_probe), DEVMETHOD(device_attach, gic_v3_fdt_attach), /* Bus interface */ DEVMETHOD(bus_alloc_resource, gic_v3_ofw_bus_alloc_res), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, gic_v3_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gic, gic_v3_fdt_driver, gic_v3_fdt_methods, sizeof(struct gic_v3_softc), gic_v3_driver); static devclass_t gic_v3_fdt_devclass; EARLY_DRIVER_MODULE(gic_v3, simplebus, gic_v3_fdt_driver, gic_v3_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, gic_v3_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); /* * Helper functions declarations. */ static int gic_v3_ofw_bus_attach(device_t); /* * Device interface. */ static int gic_v3_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "arm,gic-v3")) return (ENXIO); device_set_desc(dev, GIC_V3_DEVSTR); return (BUS_PROBE_DEFAULT); } static int gic_v3_fdt_attach(device_t dev) { struct gic_v3_softc *sc; pcell_t redist_regions; +#ifdef INTRNG + intptr_t xref; +#endif int err; sc = device_get_softc(dev); sc->dev = dev; /* * Recover number of the Re-Distributor regions. */ if (OF_getencprop(ofw_bus_get_node(dev), "#redistributor-regions", &redist_regions, sizeof(redist_regions)) <= 0) sc->gic_redists.nregions = 1; else sc->gic_redists.nregions = redist_regions; err = gic_v3_attach(dev); - if (err) + if (err != 0) goto error; + +#ifdef INTRNG + xref = OF_xref_from_node(ofw_bus_get_node(dev)); + if (intr_pic_register(dev, xref) != 0) { + device_printf(dev, "could not register PIC\n"); + goto error; + } + + if (intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc, + GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { + goto error; + } +#endif + /* * Try to register ITS to this GIC. * GIC will act as a bus in that case. * Failure here will not affect main GIC functionality. */ if (gic_v3_ofw_bus_attach(dev) != 0) { if (bootverbose) { device_printf(dev, "Failed to attach ITS to this GIC\n"); } } return (err); error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } /* Failure so free resources */ gic_v3_detach(dev); return (err); } /* OFW bus interface */ struct gic_v3_ofw_devinfo { struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; static const struct ofw_bus_devinfo * gic_v3_ofw_get_devinfo(device_t bus __unused, device_t child) { struct gic_v3_ofw_devinfo *di; di = device_get_ivars(child); return (&di->di_dinfo); } static struct resource * gic_v3_ofw_bus_alloc_res(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct gic_v3_ofw_devinfo *di; struct resource_list_entry *rle; int ranges_len; if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type != SYS_RES_MEMORY) return (NULL); /* Find defaults for this rid */ rle = resource_list_find(&di->di_rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; end = rle->end; count = rle->count; } /* * XXX: No ranges remap! * Absolute address is expected. */ if (ofw_bus_has_prop(bus, "ranges")) { ranges_len = OF_getproplen(ofw_bus_get_node(bus), "ranges"); if (ranges_len != 0) { if (bootverbose) { device_printf(child, "Ranges remap not supported\n"); } return (NULL); } } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } /* Helper functions */ /* * Bus capability support for GICv3. * Collects and configures device informations and finally * adds ITS device as a child of GICv3 in Newbus hierarchy. */ static int gic_v3_ofw_bus_attach(device_t dev) { struct gic_v3_ofw_devinfo *di; device_t child; phandle_t parent, node; pcell_t addr_cells, size_cells; parent = ofw_bus_get_node(dev); if (parent > 0) { addr_cells = 2; OF_getencprop(parent, "#address-cells", &addr_cells, sizeof(addr_cells)); size_cells = 2; OF_getencprop(parent, "#size-cells", &size_cells, sizeof(size_cells)); /* Iterate through all GIC subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) { if (bootverbose) { device_printf(dev, "Could not set up devinfo for ITS\n"); } free(di, M_GIC_V3); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, &di->di_rl); /* Should not have any interrupts, so don't add any */ /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (!child) { if (bootverbose) { device_printf(dev, "Could not add child: %s\n", di->di_dinfo.obd_name); } resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_GIC_V3); continue; } device_set_ivars(child, di); } } return (bus_generic_attach(dev)); } +#ifndef INTRNG static int gic_v3_its_fdt_probe(device_t dev); static device_method_t gic_v3_its_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gic_v3_its_fdt_probe), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(its, gic_v3_its_fdt_driver, gic_v3_its_fdt_methods, sizeof(struct gic_v3_its_softc), gic_v3_its_driver); static devclass_t gic_v3_its_fdt_devclass; EARLY_DRIVER_MODULE(its, gic, gic_v3_its_fdt_driver, gic_v3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); static int gic_v3_its_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, GIC_V3_ITS_COMPSTR)) return (ENXIO); device_set_desc(dev, GIC_V3_ITS_DEVSTR); return (BUS_PROBE_DEFAULT); } +#endif Index: head/sys/arm64/arm64/gic_v3_var.h =================================================================== --- head/sys/arm64/arm64/gic_v3_var.h (revision 299943) +++ head/sys/arm64/arm64/gic_v3_var.h (revision 299944) @@ -1,326 +1,344 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_V3_VAR_H_ #define _GIC_V3_VAR_H_ #define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" DECLARE_CLASS(gic_v3_driver); #define LPI_FLAGS_CONF_FLUSH (1UL << 0) #define LPI_CONFTAB_SIZE PAGE_SIZE_64K /* 1 bit per LPI + 1 KB more for the obligatory PPI, SGI, SPI stuff */ #define LPI_PENDTAB_SIZE ((LPI_CONFTAB_SIZE / 8) + 0x400) +#ifdef INTRNG +struct gic_v3_irqsrc { + struct intr_irqsrc gi_isrc; + uint32_t gi_irq; + enum intr_polarity gi_pol; + enum intr_trigger gi_trig; +}; +#endif + struct redist_lpis { vm_offset_t conf_base; vm_offset_t pend_base[MAXCPU]; uint64_t flags; }; struct gic_redists { /* * Re-Distributor region description. * We will have few of those depending * on the #redistributor-regions property in FDT. */ struct resource ** regions; /* Number of Re-Distributor regions */ u_int nregions; /* Per-CPU Re-Distributor handler */ struct resource * pcpu[MAXCPU]; /* LPIs data */ struct redist_lpis lpis; }; struct gic_v3_softc { device_t dev; struct resource ** gic_res; struct mtx gic_mtx; /* Distributor */ struct resource * gic_dist; /* Re-Distributors */ struct gic_redists gic_redists; u_int gic_nirqs; u_int gic_idbits; boolean_t gic_registered; + +#ifdef INTRNG + struct gic_v3_irqsrc *gic_irqs; +#endif }; +#ifdef INTRNG +#define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) +#endif + MALLOC_DECLARE(M_GIC_V3); /* Device methods */ int gic_v3_attach(device_t dev); int gic_v3_detach(device_t dev); +int arm_gic_v3_intr(void *); /* * ITS */ #define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service" #define GIC_V3_ITS_COMPSTR "arm,gic-v3-its" DECLARE_CLASS(gic_v3_its_driver); /* LPI chunk owned by ITS device */ struct lpi_chunk { u_int lpi_base; u_int lpi_num; u_int lpi_free; /* First free LPI in set */ u_int *lpi_col_ids; }; /* ITS device */ struct its_dev { TAILQ_ENTRY(its_dev) entry; /* PCI device */ device_t pci_dev; /* Device ID (i.e. PCI device ID) */ uint32_t devid; /* List of assigned LPIs */ struct lpi_chunk lpis; /* Virtual address of ITT */ vm_offset_t itt; }; TAILQ_HEAD(its_dev_list, its_dev); /* ITS private table description */ struct its_ptab { vm_offset_t ptab_vaddr; /* Virtual Address of table */ size_t ptab_pgsz; /* Page size */ size_t ptab_npages; /* Number of pages */ }; /* ITS collection description. */ struct its_col { uint64_t col_target; /* Target Re-Distributor */ uint64_t col_id; /* Collection ID */ }; /* ITS command. Each command is 32 bytes long */ struct its_cmd { uint64_t cmd_dword[4]; /* ITS command double word */ }; /* ITS commands encoding */ #define ITS_CMD_MOVI (0x01) #define ITS_CMD_SYNC (0x05) #define ITS_CMD_MAPD (0x08) #define ITS_CMD_MAPC (0x09) #define ITS_CMD_MAPVI (0x0a) #define ITS_CMD_MAPI (0x0b) #define ITS_CMD_INV (0x0c) #define ITS_CMD_INVALL (0x0d) /* Command */ #define CMD_COMMAND_MASK (0xFFUL) /* PCI device ID */ #define CMD_DEVID_SHIFT (32) #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) /* Size of IRQ ID bitfield */ #define CMD_SIZE_MASK (0xFFUL) /* Virtual LPI ID */ #define CMD_ID_MASK (0xFFFFFFFFUL) /* Physical LPI ID */ #define CMD_PID_SHIFT (32) #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) /* Collection */ #define CMD_COL_MASK (0xFFFFUL) /* Target (CPU or Re-Distributor) */ #define CMD_TARGET_SHIFT (16) #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) /* Interrupt Translation Table address */ #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) /* Valid command bit */ #define CMD_VALID_SHIFT (63) #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) /* * ITS command descriptor. * Idea for command description passing taken from Linux. */ struct its_cmd_desc { uint8_t cmd_type; union { struct { struct its_dev *its_dev; struct its_col *col; uint32_t id; } cmd_desc_movi; struct { struct its_col *col; } cmd_desc_sync; struct { struct its_col *col; uint8_t valid; } cmd_desc_mapc; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; uint32_t id; } cmd_desc_mapvi; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_mapi; struct { struct its_dev *its_dev; uint8_t valid; } cmd_desc_mapd; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_inv; struct { struct its_col *col; } cmd_desc_invall; }; }; #define ITS_CMDQ_SIZE PAGE_SIZE_64K #define ITS_CMDQ_NENTRIES (ITS_CMDQ_SIZE / sizeof(struct its_cmd)) #define ITS_FLAGS_CMDQ_FLUSH (1UL << 0) #define ITS_TARGET_NONE 0xFBADBEEF struct gic_v3_its_softc { device_t dev; struct resource * its_res; struct its_cmd * its_cmdq_base; /* ITS command queue base */ struct its_cmd * its_cmdq_write; /* ITS command queue write ptr */ struct its_ptab its_ptabs[GITS_BASER_NUM];/* ITS private tables */ struct its_col * its_cols[MAXCPU];/* Per-CPU collections */ uint64_t its_flags; struct its_dev_list its_dev_list; bitstr_t * its_lpi_bitmap; uint32_t its_lpi_maxid; struct mtx its_dev_lock; struct mtx its_cmd_lock; uint32_t its_socket; /* Socket number ITS is attached to */ }; /* Stuff that is specific to the vendor's implementation */ typedef uint32_t (*its_devbits_func_t)(device_t); struct its_quirks { uint64_t cpuid; uint64_t cpuid_mask; its_devbits_func_t devbits_func; }; extern devclass_t gic_v3_its_devclass; int gic_v3_its_detach(device_t); int gic_v3_its_alloc_msix(device_t, device_t, int *); int gic_v3_its_alloc_msi(device_t, device_t, int, int *); int gic_v3_its_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); int its_init_cpu(struct gic_v3_its_softc *); int lpi_migrate(device_t, uint32_t, u_int); void lpi_unmask_irq(device_t, uint32_t); void lpi_mask_irq(device_t, uint32_t); /* * GIC Distributor accessors. * Notice that only GIC sofc can be passed. */ #define gic_d_read(sc, len, reg) \ ({ \ bus_read_##len(sc->gic_dist, reg); \ }) #define gic_d_write(sc, len, reg, val) \ ({ \ bus_write_##len(sc->gic_dist, reg, val);\ }) /* GIC Re-Distributor accessors (per-CPU) */ #define gic_r_read(sc, len, reg) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_read_##len( \ sc->gic_redists.pcpu[cpu], \ reg); \ }) #define gic_r_write(sc, len, reg, val) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_write_##len( \ sc->gic_redists.pcpu[cpu], \ reg, val); \ }) #define PCI_DEVID_GENERIC(pci_dev) \ ({ \ ((pci_get_domain(pci_dev) << PCI_RID_DOMAIN_SHIFT) | \ (pci_get_bus(pci_dev) << PCI_RID_BUS_SHIFT) | \ (pci_get_slot(pci_dev) << PCI_RID_SLOT_SHIFT) | \ (pci_get_function(pci_dev) << PCI_RID_FUNC_SHIFT)); \ }) /* * Request number of maximum MSI-X vectors for this device. * Device can ask for less vectors than maximum supported but not more. */ #define PCI_MSIX_NUM(pci_dev) \ ({ \ struct pci_devinfo *dinfo; \ pcicfgregs *cfg; \ \ dinfo = device_get_ivars(pci_dev); \ cfg = &dinfo->cfg; \ \ cfg->msix.msix_msgnum; \ }) #endif /* _GIC_V3_VAR_H_ */ Index: head/sys/conf/files.arm64 =================================================================== --- head/sys/conf/files.arm64 (revision 299943) +++ head/sys/conf/files.arm64 (revision 299944) @@ -1,103 +1,103 @@ # $FreeBSD$ arm/arm/generic_timer.c standard arm/arm/gic.c optional intrng arm/arm/pmu.c standard arm64/acpica/acpi_machdep.c optional acpi arm64/acpica/OsdEnvironment.c optional acpi arm64/acpica/acpi_wakeup.c optional acpi arm64/acpica/pci_cfgreg.c optional acpi pci arm64/arm64/autoconf.c standard arm64/arm64/bcopy.c standard arm64/arm64/bus_machdep.c standard arm64/arm64/bus_space_asm.S standard arm64/arm64/busdma_bounce.c standard arm64/arm64/busdma_machdep.c standard arm64/arm64/bzero.S standard arm64/arm64/clock.c standard arm64/arm64/copyinout.S standard arm64/arm64/copystr.c standard arm64/arm64/cpufunc_asm.S standard arm64/arm64/db_disasm.c optional ddb arm64/arm64/db_interface.c optional ddb arm64/arm64/db_trace.c optional ddb arm64/arm64/debug_monitor.c optional kdb arm64/arm64/disassem.c optional ddb arm64/arm64/dump_machdep.c standard arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/gic.c optional !intrng arm64/arm64/gic_acpi.c optional !intrng acpi arm64/arm64/gic_fdt.c optional !intrng fdt -arm64/arm64/gic_v3.c optional !intrng -arm64/arm64/gic_v3_fdt.c optional !intrng fdt +arm64/arm64/gic_v3.c standard +arm64/arm64/gic_v3_fdt.c optional fdt arm64/arm64/gic_v3_its.c optional !intrng arm64/arm64/identcpu.c standard arm64/arm64/intr_machdep.c optional !intrng arm64/arm64/in_cksum.c optional inet | inet6 arm64/arm64/locore.S standard no-obj arm64/arm64/machdep.c standard arm64/arm64/mem.c standard arm64/arm64/minidump_machdep.c standard arm64/arm64/mp_machdep.c optional smp arm64/arm64/nexus.c standard arm64/arm64/ofw_machdep.c optional fdt arm64/arm64/pic_if.m optional !intrng arm64/arm64/pmap.c standard arm64/arm64/stack_machdep.c optional ddb | stack arm64/arm64/support.S standard arm64/arm64/swtch.S standard arm64/arm64/sys_machdep.c standard arm64/arm64/trap.c standard arm64/arm64/uio_machdep.c standard arm64/arm64/uma_machdep.c standard arm64/arm64/unwind.c optional ddb | kdtrace_hooks | stack arm64/arm64/vfp.c standard arm64/arm64/vm_machdep.c standard arm64/cavium/thunder_pcie_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_pem.c optional soc_cavm_thunderx pci arm64/cavium/thunder_pcie_pem_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_common.c optional soc_cavm_thunderx pci arm64/cloudabi64/cloudabi64_sysvec.c optional compat_cloudabi64 crypto/blowfish/bf_enc.c optional crypto | ipsec crypto/des/des_enc.c optional crypto | ipsec | netsmb dev/acpica/acpi_if.m optional acpi dev/ahci/ahci_generic.c optional ahci fdt dev/hwpmc/hwpmc_arm64.c optional hwpmc dev/hwpmc/hwpmc_arm64_md.c optional hwpmc dev/mmc/host/dwmmc.c optional dwmmc dev/mmc/host/dwmmc_hisi.c optional dwmmc soc_hisi_hi6220 dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofwpci.c optional fdt pci dev/pci/pci_host_generic.c optional pci fdt dev/psci/psci.c optional psci dev/psci/psci_arm64.S optional psci dev/uart/uart_cpu_fdt.c optional uart fdt dev/uart/uart_dev_pl011.c optional uart pl011 dev/usb/controller/dwc_otg_hisi.c optional dwcotg soc_hisi_hi6220 dev/vnic/mrml_bridge.c optional vnic fdt dev/vnic/nic_main.c optional vnic pci dev/vnic/nicvf_main.c optional vnic pci pci_iov dev/vnic/nicvf_queues.c optional vnic pci pci_iov dev/vnic/thunder_bgx_fdt.c optional vnic fdt dev/vnic/thunder_bgx.c optional vnic pci dev/vnic/thunder_mdio_fdt.c optional vnic fdt dev/vnic/thunder_mdio.c optional vnic dev/vnic/lmac_if.m optional vnic kern/kern_clocksource.c standard kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_intr.c optional intrng libkern/bcmp.c standard libkern/ffs.c standard libkern/ffsl.c standard libkern/ffsll.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S optional zfs | dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/aarch64/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}"