Index: head/sys/arm64/arm64/gic_v3.c =================================================================== --- head/sys/arm64/arm64/gic_v3.c (nonexistent) +++ head/sys/arm64/arm64/gic_v3.c (revision 282867) @@ -0,0 +1,580 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "pic_if.h" + +#include "gic_v3_reg.h" +#include "gic_v3_var.h" + +/* Device and PIC methods */ +static void gic_v3_dispatch(device_t, struct trapframe *); +static void gic_v3_eoi(device_t, u_int); +static void gic_v3_mask_irq(device_t, u_int); +static void gic_v3_unmask_irq(device_t, u_int); + +static device_method_t gic_v3_methods[] = { + /* Device interface */ + DEVMETHOD(device_detach, gic_v3_detach), + + /* PIC interface */ + DEVMETHOD(pic_dispatch, gic_v3_dispatch), + DEVMETHOD(pic_eoi, gic_v3_eoi), + DEVMETHOD(pic_mask, gic_v3_mask_irq), + DEVMETHOD(pic_unmask, gic_v3_unmask_irq), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_0(gic_v3, gic_v3_driver, gic_v3_methods, + sizeof(struct gic_v3_softc)); + +/* + * Driver-specific definitions. + */ +MALLOC_DEFINE(M_GIC_V3, "GICv3", GIC_V3_DEVSTR); + +/* + * Helper functions and definitions. + */ +/* Destination registers, either Distributor or Re-Distributor */ +enum gic_v3_xdist { + DIST = 0, + REDIST, +}; + +/* Helper routines starting with gic_v3_ */ +static int gic_v3_dist_init(struct gic_v3_softc *); +static int gic_v3_redist_find(struct gic_v3_softc *); +static int gic_v3_redist_init(struct gic_v3_softc *); +static int gic_v3_cpu_init(struct gic_v3_softc *); +static void gic_v3_wait_for_rwp(struct gic_v3_softc *, enum gic_v3_xdist); + +/* A sequence of init functions for primary (boot) CPU */ +typedef int (*gic_v3_initseq_t) (struct gic_v3_softc *); +/* Primary CPU initialization sequence */ +static gic_v3_initseq_t gic_v3_primary_init[] = { + gic_v3_dist_init, + gic_v3_redist_init, + gic_v3_cpu_init, + NULL +}; + +/* + * Device interface. + */ +int +gic_v3_attach(device_t dev) +{ + struct gic_v3_softc *sc; + gic_v3_initseq_t *init_func; + uint32_t typer; + int rid; + int err; + size_t i; + + sc = device_get_softc(dev); + sc->gic_registered = FALSE; + sc->dev = dev; + err = 0; + + /* Initialize mutex */ + mtx_init(&sc->gic_mtx, "GICv3 lock", NULL, MTX_SPIN); + + /* + * Allocate array of struct resource. + * One entry for Distributor and all remaining for Re-Distributor. + */ + sc->gic_res = malloc( + sizeof(sc->gic_res) * (sc->gic_redists.nregions + 1), + M_GIC_V3, M_WAITOK); + + /* Now allocate corresponding resources */ + for (i = 0, rid = 0; i < (sc->gic_redists.nregions + 1); i++, rid++) { + sc->gic_res[rid] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + if (sc->gic_res[rid] == NULL) + return (ENXIO); + } + + /* + * Distributor interface + */ + sc->gic_dist = sc->gic_res[0]; + + /* + * Re-Dristributor interface + */ + /* Allocate space under region descriptions */ + sc->gic_redists.regions = malloc( + sizeof(*sc->gic_redists.regions) * sc->gic_redists.nregions, + M_GIC_V3, M_WAITOK); + + /* Fill-up bus_space information for each region. */ + for (i = 0, rid = 1; i < sc->gic_redists.nregions; i++, rid++) + sc->gic_redists.regions[i] = sc->gic_res[rid]; + + /* Get the number of supported SPI interrupts */ + typer = gic_d_read(sc, 4, GICD_TYPER); + sc->gic_nirqs = GICD_TYPER_I_NUM(typer); + if (sc->gic_nirqs > GIC_I_NUM_MAX) + sc->gic_nirqs = GIC_I_NUM_MAX; + + /* Get the number of supported interrupt identifier bits */ + sc->gic_idbits = GICD_TYPER_IDBITS(typer); + + if (bootverbose) { + device_printf(dev, "SPIs: %u, IDs: %u\n", + sc->gic_nirqs, (1 << sc->gic_idbits) - 1); + } + + /* Train init sequence for boot CPU */ + for (init_func = gic_v3_primary_init; *init_func != NULL; init_func++) { + err = (*init_func)(sc); + if (err != 0) + return (err); + } + /* + * Full success. + * Now register PIC to the interrupts handling layer. + */ + arm_register_root_pic(dev, sc->gic_nirqs); + sc->gic_registered = TRUE; + + return (0); +} + +int +gic_v3_detach(device_t dev) +{ + struct gic_v3_softc *sc; + size_t i; + int rid; + + sc = device_get_softc(dev); + + if (device_is_attached(dev)) { + /* + * XXX: We should probably deregister PIC + */ + if (sc->gic_registered) + panic("Trying to detach registered PIC"); + } + for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); + + for (i = 0; i < MAXCPU; i++) + free(sc->gic_redists.pcpu[i], M_GIC_V3); + + free(sc->gic_res, M_GIC_V3); + free(sc->gic_redists.regions, M_GIC_V3); + + return (0); +} + +/* + * PIC interface. + */ +static void +gic_v3_dispatch(device_t dev, struct trapframe *frame) +{ + uint64_t active_irq; + + while (1) { + active_irq = gic_icc_read(IAR1); + + if (__predict_false(active_irq == ICC_IAR1_EL1_SPUR)) + break; + + if (__predict_true((active_irq >= GIC_FIRST_PPI && + active_irq <= GIC_LAST_SPI))) { + arm_dispatch_intr(active_irq, frame); + continue; + } + + if (active_irq <= GIC_LAST_SGI || active_irq >= GIC_FIRST_LPI) { + /* + * TODO: Implement proper SGI/LPI handling. + * Mask it if such is received for some reason. + */ + device_printf(dev, + "Received unsupported interrupt type: %s\n", + active_irq >= GIC_FIRST_LPI ? "LPI" : "SGI"); + PIC_MASK(dev, active_irq); + } + } +} + +static void +gic_v3_eoi(device_t dev, u_int irq) +{ + + gic_icc_write(EOIR1, (uint64_t)irq); +} + +static void +gic_v3_mask_irq(device_t dev, u_int irq) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ + gic_r_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("%s: Unsupported IRQ number %u", __func__, irq); +} + +static void +gic_v3_unmask_irq(device_t dev, u_int irq) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ + gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("%s: Unsupported IRQ number %u", __func__, irq); +} + +/* + * Helper routines + */ +static void +gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) +{ + struct resource *res; + u_int cpuid; + size_t us_left = 1000000; + + cpuid = PCPU_GET(cpuid); + + switch (xdist) { + case DIST: + res = sc->gic_dist; + break; + case REDIST: + res = sc->gic_redists.pcpu[cpuid]; + break; + default: + KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__)); + return; + } + + while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) { + DELAY(1); + if (us_left-- == 0) + panic("GICD Register write pending for too long"); + } +} + +/* CPU interface. */ +static __inline void +gic_v3_cpu_priority(uint64_t mask) +{ + + /* Set prority mask */ + gic_icc_write(PMR, mask & ICC_PMR_EL1_PRIO_MASK); +} + +static int +gic_v3_cpu_enable_sre(struct gic_v3_softc *sc) +{ + uint64_t sre; + u_int cpuid; + + cpuid = PCPU_GET(cpuid); + /* + * Set the SRE bit to enable access to GIC CPU interface + * via system registers. + */ + sre = READ_SPECIALREG(icc_sre_el1); + sre |= ICC_SRE_EL1_SRE; + WRITE_SPECIALREG(icc_sre_el1, sre); + isb(); + /* + * Now ensure that the bit is set. + */ + sre = READ_SPECIALREG(icc_sre_el1); + if ((sre & ICC_SRE_EL1_SRE) == 0) { + /* We are done. This was disabled in EL2 */ + device_printf(sc->dev, "ERROR: CPU%u cannot enable CPU interface " + "via system registers\n", cpuid); + return (ENXIO); + } else if (bootverbose) { + device_printf(sc->dev, + "CPU%u enabled CPU interface via system registers\n", + cpuid); + } + + return (0); +} + +static int +gic_v3_cpu_init(struct gic_v3_softc *sc) +{ + int err; + + /* Enable access to CPU interface via system registers */ + err = gic_v3_cpu_enable_sre(sc); + if (err != 0) + return (err); + /* Priority mask to minimum - accept all interrupts */ + gic_v3_cpu_priority(GIC_PRIORITY_MIN); + /* Disable EOI mode */ + gic_icc_clear(CTLR, ICC_CTLR_EL1_EOIMODE); + /* Enable group 1 (insecure) interrups */ + gic_icc_set(IGRPEN1, ICC_IGRPEN0_EL1_EN); + + return (0); +} + +/* Distributor */ +static int +gic_v3_dist_init(struct gic_v3_softc *sc) +{ + uint64_t aff; + u_int i; + + /* + * 1. Disable the Distributor + */ + gic_d_write(sc, 4, GICD_CTLR, 0); + gic_v3_wait_for_rwp(sc, DIST); + + /* + * 2. Configure the Distributor + */ + /* Set all global interrupts to be level triggered, active low. */ + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ICFGRn) + gic_d_write(sc, 4, GICD_ICFGR(i), 0x00000000); + + /* Set priority to all shared interrupts */ + for (i = GIC_FIRST_SPI; + i < sc->gic_nirqs; i += GICD_I_PER_IPRIORITYn) { + /* Set highest priority */ + gic_d_write(sc, 4, GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); + } + + /* + * Disable all interrupts. Leave PPI and SGIs as they are enabled in + * Re-Distributor registers. + */ + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ISENABLERn) + gic_d_write(sc, 4, GICD_ICENABLER(i), 0xFFFFFFFF); + + gic_v3_wait_for_rwp(sc, DIST); + + /* + * 3. Enable Distributor + */ + /* Enable Distributor with ARE, Group 1 */ + gic_d_write(sc, 4, GICD_CTLR, GICD_CTLR_ARE_NS | GICD_CTLR_G1A | + GICD_CTLR_G1); + + /* + * 4. Route all interrupts to boot CPU. + */ + aff = CPU_AFFINITY(PCPU_GET(cpuid)); + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i++) + gic_d_write(sc, 4, GICD_IROUTER(i), aff); + + return (0); +} + +/* Re-Distributor */ +static int +gic_v3_redist_find(struct gic_v3_softc *sc) +{ + struct resource r_res; + bus_space_handle_t r_bsh; + uint64_t aff; + uint64_t typer; + uint32_t pidr2; + u_int cpuid; + size_t i; + + cpuid = PCPU_GET(cpuid); + + /* Allocate struct resource for this CPU's Re-Distributor registers */ + sc->gic_redists.pcpu[cpuid] = + malloc(sizeof(*sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); + + aff = CPU_AFFINITY(cpuid); + /* Affinity in format for comparison with typer */ + aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) | + (CPU_AFF1(aff) << 8) | CPU_AFF0(aff); + + if (bootverbose) { + device_printf(sc->dev, + "Start searching for Re-Distributor\n"); + } + /* Iterate through Re-Distributor regions */ + for (i = 0; i < sc->gic_redists.nregions; i++) { + /* Take a copy of the region's resource */ + r_res = *sc->gic_redists.regions[i]; + r_bsh = rman_get_bushandle(&r_res); + + pidr2 = bus_read_4(&r_res, GICR_PIDR2); + switch (pidr2 & GICR_PIDR2_ARCH_MASK) { + case GICR_PIDR2_ARCH_GICv3: /* fall through */ + case GICR_PIDR2_ARCH_GICv4: + break; + default: + device_printf(sc->dev, + "No Re-Distributor found for CPU%u\n", cpuid); + free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); + return (ENODEV); + } + + do { + typer = bus_read_8(&r_res, GICR_TYPER); + if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) { + KASSERT(sc->gic_redists.pcpu[cpuid] != NULL, + ("Invalid pointer to per-CPU redistributor")); + /* Copy res contents to its final destination */ + *sc->gic_redists.pcpu[cpuid] = r_res; + if (bootverbose) { + device_printf(sc->dev, + "CPU%u Re-Distributor has been found\n", + cpuid); + } + return (0); + } + + r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); + if ((typer & GICR_TYPER_VLPIS) != 0) { + r_bsh += + (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE); + } + + rman_set_bushandle(&r_res, r_bsh); + } while ((typer & GICR_TYPER_LAST) == 0); + } + + free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); + device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); + return (ENXIO); +} + +static int +gic_v3_redist_wake(struct gic_v3_softc *sc) +{ + uint32_t waker; + size_t us_left = 1000000; + + waker = gic_r_read(sc, 4, GICR_WAKER); + /* Wake up Re-Distributor for this CPU */ + waker &= ~GICR_WAKER_PS; + gic_r_write(sc, 4, GICR_WAKER, waker); + /* + * When clearing ProcessorSleep bit it is required to wait for + * ChildrenAsleep to become zero following the processor power-on. + */ + while ((gic_r_read(sc, 4, GICR_WAKER) & GICR_WAKER_CA) != 0) { + DELAY(1); + if (us_left-- == 0) { + panic("Could not wake Re-Distributor for CPU%u", + PCPU_GET(cpuid)); + } + } + + if (bootverbose) { + device_printf(sc->dev, "CPU%u Re-Distributor woke up\n", + PCPU_GET(cpuid)); + } + + return (0); +} + +static int +gic_v3_redist_init(struct gic_v3_softc *sc) +{ + int err; + size_t i; + + err = gic_v3_redist_find(sc); + if (err != 0) + return (err); + + err = gic_v3_redist_wake(sc); + if (err != 0) + return (err); + + /* Disable SPIs */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ICENABLER0, + GICR_I_ENABLER_PPI_MASK); + /* Enable SGIs */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ISENABLER0, + GICR_I_ENABLER_SGI_MASK); + + /* Set priority for SGIs and PPIs */ + for (i = 0; i <= GIC_LAST_PPI; i += GICR_I_PER_IPRIORITYn) { + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_IPRIORITYR(i), + GIC_PRIORITY_MAX); + } + + gic_v3_wait_for_rwp(sc, REDIST); + + return (0); +} Property changes on: head/sys/arm64/arm64/gic_v3.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/arm64/arm64/gic_v3_fdt.c =================================================================== --- head/sys/arm64/arm64/gic_v3_fdt.c (nonexistent) +++ head/sys/arm64/arm64/gic_v3_fdt.c (revision 282867) @@ -0,0 +1,124 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "pic_if.h" + +#include "gic_v3_reg.h" +#include "gic_v3_var.h" + +/* + * FDT glue. + */ +static int gic_v3_fdt_probe(device_t); +static int gic_v3_fdt_attach(device_t); + +static device_method_t gic_v3_fdt_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, gic_v3_fdt_probe), + DEVMETHOD(device_attach, gic_v3_fdt_attach), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_1(gic_v3, gic_v3_fdt_driver, gic_v3_fdt_methods, + sizeof(struct gic_v3_softc), gic_v3_driver); + +static devclass_t gic_v3_fdt_devclass; + +EARLY_DRIVER_MODULE(gic_v3, simplebus, gic_v3_fdt_driver, gic_v3_fdt_devclass, + 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); +EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, gic_v3_fdt_devclass, + 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); + +/* + * Device interface. + */ +static int +gic_v3_fdt_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "arm,gic-v3")) + return (ENXIO); + + device_set_desc(dev, GIC_V3_DEVSTR); + return (BUS_PROBE_DEFAULT); +} + +static int +gic_v3_fdt_attach(device_t dev) +{ + struct gic_v3_softc *sc; + pcell_t redist_regions; + int err; + + sc = device_get_softc(dev); + sc->dev = dev; + + /* + * Recover number of the Re-Distributor regions. + */ + if (OF_getencprop(ofw_bus_get_node(dev), "#redistributor-regions", + &redist_regions, sizeof(redist_regions)) <= 0) + sc->gic_redists.nregions = 1; + else + sc->gic_redists.nregions = redist_regions; + + err = gic_v3_attach(dev); + if (err) + goto error; + + return (err); + +error: + if (bootverbose) { + device_printf(dev, + "Failed to attach. Error %d\n", err); + } + /* Failure so free resources */ + gic_v3_detach(dev); + + return (err); +} Property changes on: head/sys/arm64/arm64/gic_v3_fdt.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/arm64/arm64/gic_v3_reg.h =================================================================== --- head/sys/arm64/arm64/gic_v3_reg.h (nonexistent) +++ head/sys/arm64/arm64/gic_v3_reg.h (revision 282867) @@ -0,0 +1,185 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GIC_V3_REG_H_ +#define _GIC_V3_REG_H_ + +/* + * Maximum number of interrupts + * supported by GIC (including SGIs, PPIs and SPIs) + */ +#define GIC_I_NUM_MAX (1020) +/* + * Priority MAX/MIN values + */ +#define GIC_PRIORITY_MAX (0x00UL) +/* Upper value is determined by LPI max priority */ +#define GIC_PRIORITY_MIN (0xFCUL) + +/* Numbers for software generated interrupts */ +#define GIC_FIRST_SGI (0) +#define GIC_LAST_SGI (15) +/* Numbers for private peripheral interrupts */ +#define GIC_FIRST_PPI (16) +#define GIC_LAST_PPI (31) +/* Numbers for spared peripheral interrupts */ +#define GIC_FIRST_SPI (32) +#define GIC_LAST_SPI (1019) +/* Numbers for local peripheral interrupts */ +#define GIC_FIRST_LPI (8192) + +/* + * Registers (v2/v3) + */ +#define GICD_CTLR (0x0000) +#define GICD_CTLR_G1 (1 << 0) +#define GICD_CTLR_G1A (1 << 1) +#define GICD_CTLR_ARE_NS (1 << 4) +#define GICD_CTLR_RWP (1 << 31) + +#define GICD_TYPER (0x0004) +#define GICD_TYPER_IDBITS(n) ((((n) >> 19) & 0x1F) + 1) +#define GICD_TYPER_I_NUM(n) ((((n) & 0xF1) + 1) * 32) + +#define GICD_ISENABLER(n) (0x0100 + (((n) >> 5) * 4)) +#define GICD_I_PER_ISENABLERn (32) + +#define GICD_ICENABLER(n) (0x0180 + (((n) >> 5) * 4)) +#define GICD_IPRIORITYR(n) (0x0400 + (((n) >> 2) * 4)) +#define GICD_I_PER_IPRIORITYn (4) + +#define GICD_I_MASK(n) (1 << ((n) % 32)) + +#define GICD_ICFGR(n) (0x0C00 + (((n) >> 4) * 4)) +/* First bit is a polarity bit (0 - low, 1 - high) */ +#define GICD_ICFGR_POL_LOW (0 << 0) +#define GICD_ICFGR_POL_HIGH (1 << 0) +#define GICD_ICFGR_POL_MASK (0x1) +/* Second bit is a trigger bit (0 - level, 1 - edge) */ +#define GICD_ICFGR_TRIG_LVL (0 << 1) +#define GICD_ICFGR_TRIG_EDGE (1 << 1) +#define GICD_ICFGR_TRIG_MASK (0x2) + +#define GICD_I_PER_ICFGRn (16) + +/* + * Registers (v3) + */ +#define GICD_IROUTER(n) (0x6000 + ((n) * 8)) +#define GICD_PIDR2 (0xFFE8) + +#define GICR_PIDR2_ARCH_MASK (0xF0) +#define GICR_PIDR2_ARCH_GICv3 (0x30) +#define GICR_PIDR2_ARCH_GICv4 (0x40) + +/* Redistributor registers */ +#define GICR_PIDR2 GICD_PIDR2 + +#define GICR_TYPER (0x0008) +#define GICR_TYPER_VLPIS (1 << 1) +#define GICR_TYPER_LAST (1 << 4) +#define GICR_TYPER_AFF_SHIFT (32) + +#define GICR_WAKER (0x0014) +#define GICR_WAKER_PS (1 << 1) /* Processor sleep */ +#define GICR_WAKER_CA (1 << 2) /* Children asleep */ + +/* Re-distributor registers for SGIs and PPIs */ +#define GICR_RD_BASE_SIZE PAGE_SIZE_64K +#define GICR_SGI_BASE_SIZE PAGE_SIZE_64K +#define GICR_VLPI_BASE_SIZE PAGE_SIZE_64K +#define GICR_RESERVED_SIZE PAGE_SIZE_64K + +#define GICR_ISENABLER0 (0x0100) +#define GICR_ICENABLER0 (0x0180) +#define GICR_I_ENABLER_SGI_MASK (0x0000FFFF) +#define GICR_I_ENABLER_PPI_MASK (0xFFFF0000) + +#define GICR_I_PER_IPRIORITYn (GICD_I_PER_IPRIORITYn) + +/* + * CPU interface + */ + +/* + * Registers list (ICC_xyz_EL1): + * + * PMR - Priority Mask Register + * * interrupts of priority higher than specified + * in this mask will be signalled to the CPU. + * (0xff - lowest possible prio., 0x00 - highest prio.) + * + * CTLR - Control Register + * * controls behavior of the CPU interface and displays + * implemented features. + * + * IGRPEN1 - Interrupt Group 1 Enable Register + * + * IAR1 - Interrupt Acknowledge Register Group 1 + * * contains number of the highest priority pending + * interrupt from the Group 1. + * + * EOIR1 - End of Interrupt Register Group 1 + * * Writes inform CPU interface about completed Group 1 + * interrupts processing. + */ + +#define gic_icc_write(reg, val) \ +do { \ + WRITE_SPECIALREG(ICC_ ##reg ##_EL1, val); \ + isb(); \ +} while (0) + +#define gic_icc_read(reg) \ +({ \ + uint64_t val; \ + \ + val = READ_SPECIALREG(ICC_ ##reg ##_EL1); \ + (val); \ +}) + +#define gic_icc_set(reg, mask) \ +do { \ + uint64_t val; \ + val = gic_icc_read(reg); \ + val |= (mask); \ + gic_icc_write(reg, val); \ +} while (0) + +#define gic_icc_clear(reg, mask) \ +do { \ + uint64_t val; \ + val = gic_icc_read(reg); \ + val &= ~(mask); \ + gic_icc_write(reg, val); \ +} while (0) + +#endif /* _GIC_V3_REG_H_ */ Property changes on: head/sys/arm64/arm64/gic_v3_reg.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/arm64/arm64/gic_v3_var.h =================================================================== --- head/sys/arm64/arm64/gic_v3_var.h (nonexistent) +++ head/sys/arm64/arm64/gic_v3_var.h (revision 282867) @@ -0,0 +1,106 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _GIC_V3_VAR_H_ +#define _GIC_V3_VAR_H_ + +#define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" + +DECLARE_CLASS(gic_v3_driver); + +struct gic_redists { + /* + * Re-Distributor region description. + * We will have few of those depending + * on the #redistributor-regions property in FDT. + */ + struct resource ** regions; + /* Number of Re-Distributor regions */ + u_int nregions; + /* Per-CPU Re-Distributor handler */ + struct resource * pcpu[MAXCPU]; +}; + +struct gic_v3_softc { + device_t dev; + struct resource ** gic_res; + struct mtx gic_mtx; + /* Distributor */ + struct resource * gic_dist; + /* Re-Distributors */ + struct gic_redists gic_redists; + + u_int gic_nirqs; + u_int gic_idbits; + + boolean_t gic_registered; +}; + +MALLOC_DECLARE(M_GIC_V3); + +/* Device methods */ +int gic_v3_attach(device_t dev); +int gic_v3_detach(device_t dev); + +/* + * GIC Distributor accessors. + * Notice that only GIC sofc can be passed. + */ +#define gic_d_read(sc, len, reg) \ +({ \ + bus_read_##len(sc->gic_dist, reg); \ +}) + +#define gic_d_write(sc, len, reg, val) \ +({ \ + bus_write_##len(sc->gic_dist, reg, val);\ +}) + +/* GIC Re-Distributor accessors (per-CPU) */ +#define gic_r_read(sc, len, reg) \ +({ \ + u_int cpu = PCPU_GET(cpuid); \ + \ + bus_read_##len( \ + sc->gic_redists.pcpu[cpu], \ + reg); \ +}) + +#define gic_r_write(sc, len, reg, val) \ +({ \ + u_int cpu = PCPU_GET(cpuid); \ + \ + bus_write_##len( \ + sc->gic_redists.pcpu[cpu], \ + reg, val); \ +}) + +#endif /* _GIC_V3_VAR_H_ */ Property changes on: head/sys/arm64/arm64/gic_v3_var.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/arm64/arm64/locore.S =================================================================== --- head/sys/arm64/arm64/locore.S (revision 282866) +++ head/sys/arm64/arm64/locore.S (revision 282867) @@ -1,544 +1,558 @@ /*- * Copyright (c) 2012-2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "assym.s" #include #include #include #include #include #include #define VIRT_BITS 39 .globl kernbase .set kernbase, KERNBASE #define DEVICE_MEM 0 #define NORMAL_UNCACHED 1 #define NORMAL_MEM 2 /* * We assume: * MMU on with an identity map, or off * D-Cache: off * I-Cache: on or off * We are loaded at a 2MiB aligned address */ #define INIT_STACK_SIZE (PAGE_SIZE * 4) .text .globl _start _start: /* Drop to EL1 */ bl drop_to_el1 /* * Disable the MMU. We may have entered the kernel with it on and * will need to update the tables later. If this has been set up * with anything other than a VA == PA map then this will fail, * but in this case the code to find where we are running from * would have also failed. */ dsb sy mrs x2, sctlr_el1 bic x2, x2, SCTLR_M msr sctlr_el1, x2 isb /* Get the virt -> phys offset */ bl get_virt_delta /* * At this point: * x29 = PA - VA * x28 = Our physical load address */ /* Create the page tables */ bl create_pagetables /* * At this point: * x27 = TTBR0 table * x26 = TTBR1 table */ /* Enable the mmu */ bl start_mmu /* Jump to the virtual address space */ ldr x15, .Lvirtdone br x15 virtdone: /* Set up the stack */ adr x25, initstack_end mov sp, x25 sub sp, sp, #PCB_SIZE /* Zero the BSS */ ldr x15, .Lbss ldr x14, .Lend 1: str xzr, [x15], #8 cmp x15, x14 b.lo 1b /* Backup the module pointer */ mov x1, x0 /* Make the page table base a virtual address */ sub x26, x26, x29 sub sp, sp, #(64 * 4) mov x0, sp /* Degate the delda so it is VA -> PA */ neg x29, x29 str x1, [x0] /* modulep */ str x26, [x0, 8] /* kern_l1pt */ str x29, [x0, 16] /* kern_delta */ str x25, [x0, 24] /* kern_stack */ /* trace back starts here */ mov fp, #0 /* Branch to C code */ bl initarm bl mi_startup /* We should not get here */ brk 0 .align 3 .Lvirtdone: .quad virtdone .Lbss: .quad __bss_start .Lend: .quad _end /* * If we are started in EL2, configure the required hypervisor * registers and drop to EL1. */ drop_to_el1: mrs x1, CurrentEL lsr x1, x1, #2 cmp x1, #0x2 b.eq 1f ret 1: /* Configure the Hypervisor */ mov x2, #(HCR_RW) msr hcr_el2, x2 /* Load the Virtualization Process ID Register */ mrs x2, midr_el1 msr vpidr_el2, x2 /* Load the Virtualization Multiprocess ID Register */ mrs x2, mpidr_el1 msr vmpidr_el2, x2 /* Set the bits that need to be 1 in sctlr_el1 */ ldr x2, .Lsctlr_res1 msr sctlr_el1, x2 /* Don't trap to EL2 for exceptions */ mov x2, #CPTR_RES1 msr cptr_el2, x2 /* Don't trap to EL2 for CP15 traps */ msr hstr_el2, xzr /* Hypervisor trap functions */ adr x2, hyp_vectors msr vbar_el2, x2 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) msr spsr_el2, x2 + /* Configure GICv3 CPU interface */ + mrs x2, id_aa64pfr0_el1 + /* Extract GIC bits from the register */ + ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS + /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ + cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) + b.ne 2f + + mrs x2, icc_sre_el2 + orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ + msr icc_sre_el2, x2 + isb +2: + /* Set the address to return to our return address */ msr elr_el2, x30 eret .align 3 .Lsctlr_res1: .quad SCTLR_RES1 #define VECT_EMPTY \ .align 7; \ 1: b 1b .align 11 hyp_vectors: VECT_EMPTY /* Synchronous EL2t */ VECT_EMPTY /* IRQ EL2t */ VECT_EMPTY /* FIQ EL2t */ VECT_EMPTY /* Error EL2t */ VECT_EMPTY /* Synchronous EL2h */ VECT_EMPTY /* IRQ EL2h */ VECT_EMPTY /* FIQ EL2h */ VECT_EMPTY /* Error EL2h */ VECT_EMPTY /* Synchronous 64-bit EL1 */ VECT_EMPTY /* IRQ 64-bit EL1 */ VECT_EMPTY /* FIQ 64-bit EL1 */ VECT_EMPTY /* Error 64-bit EL1 */ VECT_EMPTY /* Synchronous 32-bit EL1 */ VECT_EMPTY /* IRQ 32-bit EL1 */ VECT_EMPTY /* FIQ 32-bit EL1 */ VECT_EMPTY /* Error 32-bit EL1 */ /* * Get the delta between the physical address we were loaded to and the * virtual address we expect to run from. This is used when building the * initial page table. */ get_virt_delta: /* Load the physical address of virt_map */ adr x29, virt_map /* Load the virtual address of virt_map stored in virt_map */ ldr x28, [x29] /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ sub x29, x29, x28 /* Find the load address for the kernel */ mov x28, #(KERNBASE) add x28, x28, x29 ret .align 3 virt_map: .quad virt_map /* * This builds the page tables containing the identity map, and the kernel * virtual map. * * It relys on: * We were loaded to an address that is on a 2MiB boundary * All the memory must not cross a 1GiB boundaty * x28 contains the physical address we were loaded from * * TODO: This is out of date. * There are at least 5 pages before that address for the page tables * The pages used are: * - The identity (PA = VA) table (TTBR0) * - The Kernel L1 table (TTBR1)(not yet) * - The PA != VA L2 table to jump into (not yet) * - The FDT L2 table (not yet) */ create_pagetables: /* Save the Link register */ mov x5, x30 /* Clean the page table */ adr x6, pagetable mov x26, x6 adr x27, pagetable_end 1: stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 cmp x6, x27 b.lo 1b /* * Build the TTBR1 maps. */ /* Find the size of the kernel */ mov x6, #(KERNBASE) ldr x7, .Lend /* Find the end - begin */ sub x8, x7, x6 /* Get the number of l2 pages to allocate, rounded down */ lsr x10, x8, #(L2_SHIFT) /* Add 4 MiB for any rounding above and the module data */ add x10, x10, #2 /* Create the kernel space L2 table */ mov x6, x26 mov x7, #NORMAL_MEM mov x8, #(KERNBASE & L2_BLOCK_MASK) mov x9, x28 bl build_block_pagetable /* Move to the l1 table */ add x26, x26, #PAGE_SIZE /* Link the l1 -> l2 table */ mov x9, x6 mov x6, x26 bl link_l1_pagetable /* * Build the TTBR0 maps. */ add x27, x26, #PAGE_SIZE #if defined(SOCDEV_PA) && defined(SOCDEV_VA) /* Create a table for the UART */ mov x6, x27 /* The initial page table */ mov x7, #DEVICE_MEM mov x8, #(SOCDEV_VA) /* VA start */ mov x9, #(SOCDEV_PA) /* PA start */ bl build_section_pagetable #endif /* Create the VA = PA map */ mov x6, x27 /* The initial page table */ mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ mov x9, x27 mov x8, x9 /* VA start (== PA start) */ bl build_section_pagetable /* Restore the Link register */ mov x30, x5 ret /* * Builds a 1 GiB page table entry * x6 = L1 table * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) * x11, x12 and x13 are trashed */ build_section_pagetable: /* * Build the L1 table entry. */ /* Find the table index */ lsr x11, x8, #L1_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ lsl x12, x7, #2 orr x12, x12, #L1_BLOCK orr x12, x12, #(ATTR_AF) /* Only use the output address bits */ lsr x9, x9, #L1_SHIFT orr x12, x12, x9, lsl #L1_SHIFT /* Store the entry */ str x12, [x6, x11, lsl #3] ret /* * Builds an L1 -> L2 table descriptor * * This is a link for a 1GiB block of memory with up to 2MiB regions mapped * within it by build_block_pagetable. * * x6 = L1 table * x8 = Virtual Address * x9 = L2 PA (trashed) * x11, x12 and x13 are trashed */ link_l1_pagetable: /* * Link an L1 -> L2 table entry. */ /* Find the table index */ lsr x11, x8, #L1_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ mov x12, #L1_TABLE /* Only use the output address bits */ lsr x9, x9, #12 orr x12, x12, x9, lsl #12 /* Store the entry */ str x12, [x6, x11, lsl #3] ret /* * Builds count 2 MiB page table entry * x6 = L2 table * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count (TODO) * x11, x12 and x13 are trashed */ build_block_pagetable: /* * Build the L2 table entry. */ /* Find the table index */ lsr x11, x8, #L2_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L2 block entry */ lsl x12, x7, #2 orr x12, x12, #L2_BLOCK orr x12, x12, #(ATTR_AF) /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT /* Set the physical address for this virtual address */ 1: orr x12, x12, x9, lsl #L2_SHIFT /* Store the entry */ str x12, [x6, x11, lsl #3] /* Clear the address bits */ and x12, x12, #ATTR_MASK_L sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b 2: ret start_mmu: dsb sy /* Load the exception vectors */ ldr x2, =exception_vectors msr vbar_el1, x2 /* Load ttbr0 and ttbr1 */ msr ttbr0_el1, x27 msr ttbr1_el1, x26 isb /* Clear the Monitor Debug System control register */ msr mdscr_el1, xzr /* Invalidate the TLB */ tlbi vmalle1is ldr x2, mair msr mair_el1, x2 /* Setup TCR according to PARange bits from ID_AA64MMFR0_EL1 */ ldr x2, tcr mrs x3, id_aa64mmfr0_el1 bfi x2, x3, #32, #3 msr tcr_el1, x2 /* Setup SCTLR */ ldr x2, sctlr_set ldr x3, sctlr_clear mrs x1, sctlr_el1 bic x1, x1, x3 /* Clear the required bits */ orr x1, x1, x2 /* Set the required bits */ msr sctlr_el1, x1 isb ret .align 3 mair: /* Device Normal, no cache Normal, write-back */ .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) tcr: .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K) sctlr_set: /* Bits to set */ .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ SCTLR_I | SCTLR_SED | SCTLR_C | SCTLR_M) sctlr_clear: /* Bits to clear */ .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ SCTLR_THEE | SCTLR_CP15BEN | SCTLR_SA0 | SCTLR_SA | SCTLR_A) .globl abort abort: b abort //.section .init_pagetable .align 12 /* 4KiB aligned */ /* * 3 initial tables (in the following order): * L2 for kernel (High addresses) * L1 for kernel * L1 for user (Low addresses) */ pagetable: .space PAGE_SIZE pagetable_l1_ttbr1: .space PAGE_SIZE pagetable_l1_ttbr0: .space PAGE_SIZE pagetable_end: el2_pagetable: .space PAGE_SIZE .globl init_pt_va init_pt_va: .quad pagetable /* XXX: Keep page tables VA */ .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: ENTRY(sigcode) mov x0, sp add x0, x0, #SF_UC 1: mov x8, #SYS_sigreturn svc 0 /* sigreturn failed, exit */ mov x8, #SYS_exit svc 0 b 1b END(sigcode) /* This may be copied to the stack, keep it 16-byte aligned */ .align 3 esigcode: .data .align 3 .global szsigcode szsigcode: .quad esigcode - sigcode Index: head/sys/arm64/include/armreg.h =================================================================== --- head/sys/arm64/include/armreg.h (revision 282866) +++ head/sys/arm64/include/armreg.h (revision 282867) @@ -1,194 +1,215 @@ /*- * Copyright (c) 2013, 2014 Andrew Turner * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ARMREG_H_ #define _MACHINE_ARMREG_H_ #define READ_SPECIALREG(reg) \ ({ uint64_t val; \ __asm __volatile("mrs %0, " __STRING(reg) : "=&r" (val)); \ val; \ }) #define WRITE_SPECIALREG(reg, val) \ __asm __volatile("msr " __STRING(reg) ", %0" : : "r"((uint64_t)val)) /* CPACR_EL1 */ #define CPACR_FPEN_MASK (0x3 << 20) #define CPACR_FPEN_TRAP_ALL1 (0x0 << 20) /* Traps from EL0 and EL1 */ #define CPACR_FPEN_TRAP_EL0 (0x1 << 20) /* Traps from EL0 */ #define CPACR_FPEN_TRAP_ALL2 (0x2 << 20) /* Traps from EL0 and EL1 */ #define CPACR_FPEN_TRAP_NONE (0x3 << 20) /* No traps */ #define CPACR_TTA (0x1 << 28) /* CTR_EL0 - Cache Type Register */ #define CTR_DLINE_SHIFT 16 #define CTR_DLINE_MASK (0xf << CTR_DLINE_SHIFT) #define CTR_DLINE_SIZE(reg) (((reg) & CTR_DLINE_MASK) >> CTR_DLINE_SHIFT) #define CTR_ILINE_SHIFT 0 #define CTR_ILINE_MASK (0xf << CTR_ILINE_SHIFT) #define CTR_ILINE_SIZE(reg) (((reg) & CTR_ILINE_MASK) >> CTR_ILINE_SHIFT) /* ESR_ELx */ #define ESR_ELx_ISS_MASK 0x00ffffff #define ISS_INSN_FnV (0x01 << 10) #define ISS_INSN_EA (0x01 << 9) #define ISS_INSN_S1PTW (0x01 << 7) #define ISS_INSN_IFSC_MASK (0x1f << 0) #define ISS_DATA_ISV (0x01 << 24) #define ISS_DATA_SAS_MASK (0x03 << 22) #define ISS_DATA_SSE (0x01 << 21) #define ISS_DATA_SRT_MASK (0x1f << 16) #define ISS_DATA_SF (0x01 << 15) #define ISS_DATA_AR (0x01 << 14) #define ISS_DATA_FnV (0x01 << 10) #define ISS_DATa_EA (0x01 << 9) #define ISS_DATa_CM (0x01 << 8) #define ISS_INSN_S1PTW (0x01 << 7) #define ISS_DATa_WnR (0x01 << 6) #define ISS_DATA_DFSC_MASK (0x1f << 0) #define ESR_ELx_IL (0x01 << 25) #define ESR_ELx_EC_SHIFT 26 #define ESR_ELx_EC_MASK (0x3f << 26) #define ESR_ELx_EXCEPTION(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) #define EXCP_UNKNOWN 0x00 /* Unkwn exception */ #define EXCP_FP_SIMD 0x07 /* VFP/SIMD trap */ #define EXCP_ILL_STATE 0x0e /* Illegal execution state */ #define EXCP_SVC 0x15 /* SVC trap */ #define EXCP_MSR 0x18 /* MSR/MRS trap */ #define EXCP_INSN_ABORT_L 0x20 /* Instruction abort, from lower EL */ #define EXCP_INSN_ABORT 0x21 /* Instruction abort, from same EL */ #define EXCP_PC_ALIGN 0x22 /* PC alignment fault */ #define EXCP_DATA_ABORT_L 0x24 /* Data abort, from lower EL */ #define EXCP_DATA_ABORT 0x25 /* Data abort, from same EL */ #define EXCP_SP_ALIGN 0x26 /* SP slignment fault */ #define EXCP_TRAP_FP 0x2c /* Trapped FP exception */ #define EXCP_SERROR 0x2f /* SError interrupt */ #define EXCP_SOFTSTP_EL1 0x33 /* Software Step, from same EL */ #define EXCP_WATCHPT_EL1 0x35 /* Watchpoint, from same EL */ #define EXCP_BRK 0x3c /* Breakpoint */ +/* ICC_CTLR_EL1 */ +#define ICC_CTLR_EL1_EOIMODE (1U << 1) + +/* ICC_IAR1_EL1 */ +#define ICC_IAR1_EL1_SPUR (0x03ff) + +/* ICC_IGRPEN0_EL1 */ +#define ICC_IGRPEN0_EL1_EN (1U << 0) + +/* ICC_PMR_EL1 */ +#define ICC_PMR_EL1_PRIO_MASK (0xFFUL) + +/* ICC_SRE_EL1 */ +#define ICC_SRE_EL1_SRE (1U << 0) + +/* ICC_SRE_EL2 */ +#define ICC_SRE_EL2_EN (1U << 3) + /* ID_AA64PFR0_EL1 */ #define ID_AA64PFR0_EL0_MASK (0xf << 0) #define ID_AA64PFR0_EL1_MASK (0xf << 4) #define ID_AA64PFR0_EL2_MASK (0xf << 8) #define ID_AA64PFR0_EL3_MASK (0xf << 12) #define ID_AA64PFR0_FP_MASK (0xf << 16) #define ID_AA64PFR0_FP_IMPL (0x0 << 16) /* Floating-point implemented */ #define ID_AA64PFR0_FP_NONE (0xf << 16) /* Floating-point not implemented */ #define ID_AA64PFR0_ADV_SIMD_MASK (0xf << 20) -#define ID_AA64PFR0_GIC_MASK (0xf << 24) +#define ID_AA64PFR0_GIC_SHIFT (24) +#define ID_AA64PFR0_GIC_BITS (0x4) /* Number of bits in GIC field */ +#define ID_AA64PFR0_GIC_MASK (0xf << ID_AA64PFR0_GIC_SHIFT) +#define ID_AA64PFR0_GIC_CPUIF_EN (0x1 << ID_AA64PFR0_GIC_SHIFT) /* MAIR_EL1 - Memory Attribute Indirection Register */ #define MAIR_ATTR_MASK(idx) (0xff << ((n)* 8)) #define MAIR_ATTR(attr, idx) ((attr) << ((idx) * 8)) /* SCTLR_EL1 - System Control Register */ #define SCTLR_RES0 0xc8222400 /* Reserved, write 0 */ #define SCTLR_RES1 0x30d00800 /* Reserved, write 1 */ #define SCTLR_M 0x00000001 #define SCTLR_A 0x00000002 #define SCTLR_C 0x00000004 #define SCTLR_SA 0x00000008 #define SCTLR_SA0 0x00000010 #define SCTLR_CP15BEN 0x00000020 #define SCTLR_THEE 0x00000040 #define SCTLR_ITD 0x00000080 #define SCTLR_SED 0x00000100 #define SCTLR_UMA 0x00000200 #define SCTLR_I 0x00001000 #define SCTLR_DZE 0x00004000 #define SCTLR_UCT 0x00008000 #define SCTLR_nTWI 0x00010000 #define SCTLR_nTWE 0x00040000 #define SCTLR_WXN 0x00080000 #define SCTLR_EOE 0x01000000 #define SCTLR_EE 0x02000000 #define SCTLR_UCI 0x04000000 /* SPSR_EL1 */ /* * When the exception is taken in AArch64: * M[4] is 0 for AArch64 mode * M[3:2] is the exception level * M[1] is unused * M[0] is the SP select: * 0: always SP0 * 1: current ELs SP */ #define PSR_M_EL0t 0x00000000 #define PSR_M_EL1t 0x00000004 #define PSR_M_EL1h 0x00000005 #define PSR_M_EL2t 0x00000008 #define PSR_M_EL2h 0x00000009 #define PSR_M_MASK 0x0000001f #define PSR_F 0x00000040 #define PSR_I 0x00000080 #define PSR_A 0x00000100 #define PSR_D 0x00000200 #define PSR_IL 0x00100000 #define PSR_SS 0x00200000 #define PSR_V 0x10000000 #define PSR_C 0x20000000 #define PSR_Z 0x40000000 #define PSR_N 0x80000000 /* TCR_EL1 - Translation Control Register */ #define TCR_ASID_16 (1 << 36) #define TCR_IPS_SHIFT 32 #define TCR_IPS_32BIT (0 << TCR_IPS_SHIFT) #define TCR_IPS_36BIT (1 << TCR_IPS_SHIFT) #define TCR_IPS_40BIT (2 << TCR_IPS_SHIFT) #define TCR_IPS_42BIT (3 << TCR_IPS_SHIFT) #define TCR_IPS_44BIT (4 << TCR_IPS_SHIFT) #define TCR_IPS_48BIT (5 << TCR_IPS_SHIFT) #define TCR_TG1_SHIFT 30 #define TCR_TG1_16K (1 << TCR_TG1_SHIFT) #define TCR_TG1_4K (2 << TCR_TG1_SHIFT) #define TCR_TG1_64K (3 << TCR_TG1_SHIFT) #define TCR_T1SZ_SHIFT 16 #define TCR_T0SZ_SHIFT 0 #define TCR_TxSZ(x) (((x) << TCR_T1SZ_SHIFT) | ((x) << TCR_T0SZ_SHIFT)) /* Saved Program Status Register */ #define DBG_SPSR_SS (0x1 << 21) /* Monitor Debug System Control Register */ #define DBG_MDSCR_SS (0x1 << 0) #define DBG_MDSCR_KDE (0x1 << 13) #define DBG_MDSCR_MDE (0x1 << 15) #endif /* !_MACHINE_ARMREG_H_ */ Index: head/sys/conf/files.arm64 =================================================================== --- head/sys/conf/files.arm64 (revision 282866) +++ head/sys/conf/files.arm64 (revision 282867) @@ -1,55 +1,57 @@ # $FreeBSD$ arm/arm/devmap.c standard arm/arm/generic_timer.c standard arm64/arm64/autoconf.c standard arm64/arm64/bcopy.c standard arm64/arm64/bus_machdep.c standard arm64/arm64/bus_space_asm.S standard arm64/arm64/busdma_bounce.c standard arm64/arm64/busdma_machdep.c standard arm64/arm64/clock.c standard arm64/arm64/copyinout.S standard arm64/arm64/copystr.c standard arm64/arm64/cpufunc_asm.S standard arm64/arm64/db_disasm.c optional ddb arm64/arm64/db_interface.c optional ddb arm64/arm64/db_trace.c optional ddb arm64/arm64/debug_monitor.c optional kdb arm64/arm64/dump_machdep.c standard arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/gic.c standard +arm64/arm64/gic_v3.c standard +arm64/arm64/gic_v3_fdt.c optional fdt arm64/arm64/identcpu.c standard arm64/arm64/intr_machdep.c standard arm64/arm64/in_cksum.c optional inet | inet6 arm64/arm64/locore.S standard no-obj arm64/arm64/machdep.c standard arm64/arm64/mem.c standard arm64/arm64/minidump_machdep.c standard arm64/arm64/nexus.c standard arm64/arm64/pic_if.m standard arm64/arm64/pmap.c standard arm64/arm64/stack_machdep.c standard arm64/arm64/support.S standard arm64/arm64/swtch.S standard arm64/arm64/sys_machdep.c standard arm64/arm64/trap.c standard arm64/arm64/uio_machdep.c standard arm64/arm64/vfp.c standard arm64/arm64/vm_machdep.c standard dev/fdt/fdt_arm64.c optional fdt dev/ofw/ofw_cpu.c optional fdt dev/psci/psci.c optional psci dev/psci/psci_arm64.S optional psci dev/uart/uart_cpu_fdt.c optional uart fdt dev/uart/uart_dev_pl011.c optional uart pl011 kern/kern_clocksource.c standard kern/subr_dummy_vdso_tc.c standard libkern/bcmp.c standard libkern/ffs.c standard libkern/ffsl.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard