Index: head/sys/arm/allwinner/aw_mp.c =================================================================== --- head/sys/arm/allwinner/aw_mp.c (revision 306755) +++ head/sys/arm/allwinner/aw_mp.c (revision 306756) @@ -1,286 +1,288 @@ /*- * Copyright (c) 2014 Ganbold Tsagaankhuu * Copyright (c) 2016 Emmanuel Vadot * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Register for all dual-core SoC */ #define A20_CPUCFG_BASE 0x01c25c00 /* Register for all quad-core SoC */ #define CPUCFG_BASE 0x01f01c00 #define CPUCFG_SIZE 0x400 #define PRCM_BASE 0x01f01400 #define PRCM_SIZE 0x800 /* Register for multi-cluster SoC */ #define CPUXCFG_BASE 0x01700000 #define CPUXCFG_SIZE 0x400 #define CPU_OFFSET 0x40 #define CPU_OFFSET_CTL 0x04 #define CPU_OFFSET_STATUS 0x08 #define CPU_RST_CTL(cpuid) ((cpuid + 1) * CPU_OFFSET) #define CPU_CTL(cpuid) (((cpuid + 1) * CPU_OFFSET) + CPU_OFFSET_CTL) #define CPU_STATUS(cpuid) (((cpuid + 1) * CPU_OFFSET) + CPU_OFFSET_STATUS) #define CPU_RESET (1 << 0) #define CPU_CORE_RESET (1 << 1) #define CPUCFG_GENCTL 0x184 #define CPUCFG_P_REG0 0x1a4 #define A20_CPU1_PWR_CLAMP 0x1b0 #define CPU_PWR_CLAMP_REG 0x140 #define CPU_PWR_CLAMP(cpu) ((cpu * 4) + CPU_PWR_CLAMP_REG) #define CPU_PWR_CLAMP_STEPS 8 #define A20_CPU1_PWROFF_REG 0x1b4 #define CPU_PWROFF 0x100 #define CPUCFG_DBGCTL0 0x1e0 #define CPUCFG_DBGCTL1 0x1e4 #define CPUS_CL_RST(cl) (0x30 + (cluster) * 0x4) #define CPUX_CL_CTRL0(cl) (0x0 + (cluster) * 0x10) #define CPUX_CL_CTRL1(cl) (0x4 + (cluster) * 0x10) #define CPUX_CL_CPU_STATUS(cl) (0x30 + (cluster) * 0x4) #define CPUX_CL_RST(cl) (0x80 + (cluster) * 0x4) #define PRCM_CL_PWROFF(cl) (0x100 + (cluster) * 0x4) #define PRCM_CL_PWR_CLAMP(cl, cpu) (0x140 + (cluster) * 0x4 + (cpu) * 0x4) void aw_mp_setmaxid(platform_t plat) { int ncpu; uint32_t reg; if (mp_ncpus != 0) return; reg = cp15_l2ctlr_get(); ncpu = CPUV7_L2CTLR_NPROC(reg); mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void aw_mp_start_ap(platform_t plat) { bus_space_handle_t cpucfg; bus_space_handle_t prcm; int i, j, soc_family; uint32_t val; soc_family = allwinner_soc_family(); if (soc_family == ALLWINNERSOC_SUN7I) { if (bus_space_map(fdtbus_bs_tag, A20_CPUCFG_BASE, CPUCFG_SIZE, 0, &cpucfg) != 0) panic("Couldn't map the CPUCFG\n"); } else { if (bus_space_map(fdtbus_bs_tag, CPUCFG_BASE, CPUCFG_SIZE, 0, &cpucfg) != 0) panic("Couldn't map the CPUCFG\n"); if (bus_space_map(fdtbus_bs_tag, PRCM_BASE, PRCM_SIZE, 0, &prcm) != 0) panic("Couldn't map the PRCM\n"); } dcache_wbinv_poc_all(); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_P_REG0, pmap_kextract((vm_offset_t)mpentry)); /* * Assert nCOREPORESET low and set L1RSTDISABLE low. * Ensure DBGPWRDUP is set to LOW to prevent any external * debug access to the processor. */ for (i = 1; i < mp_ncpus; i++) bus_space_write_4(fdtbus_bs_tag, cpucfg, CPU_RST_CTL(i), 0); /* Set L1RSTDISABLE low */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_GENCTL); for (i = 1; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_GENCTL, val); /* Set DBGPWRDUP low */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1); for (i = 1; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1, val); /* Release power clamp */ for (i = 1; i < mp_ncpus; i++) for (j = 0; j <= CPU_PWR_CLAMP_STEPS; j++) { if (soc_family != ALLWINNERSOC_SUN7I) { bus_space_write_4(fdtbus_bs_tag, prcm, CPU_PWR_CLAMP(i), 0xff >> j); } else { bus_space_write_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWR_CLAMP, 0xff >> j); } } DELAY(10000); /* Clear power-off gating */ if (soc_family != ALLWINNERSOC_SUN7I) { val = bus_space_read_4(fdtbus_bs_tag, prcm, CPU_PWROFF); for (i = 0; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, prcm, CPU_PWROFF, val); } else { val = bus_space_read_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWROFF_REG); val &= ~(1 << 0); bus_space_write_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWROFF_REG, val); } DELAY(1000); /* De-assert cpu core reset */ for (i = 1; i < mp_ncpus; i++) bus_space_write_4(fdtbus_bs_tag, cpucfg, CPU_RST_CTL(i), CPU_RESET | CPU_CORE_RESET); /* Assert DBGPWRDUP signal */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1); for (i = 1; i < mp_ncpus; i++) val |= (1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1, val); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, cpucfg, CPUCFG_SIZE); if (soc_family != ALLWINNERSOC_SUN7I) bus_space_unmap(fdtbus_bs_tag, prcm, PRCM_SIZE); } static void aw_mc_mp_start_cpu(bus_space_handle_t cpuscfg, bus_space_handle_t cpuxcfg, bus_space_handle_t prcm, int cluster, int cpu) { uint32_t val; int i; /* Assert core reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster), val); /* Assert power-on reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster), val); /* Disable automatic L1 cache invalidate at reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_CTRL0(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_CTRL0(cluster), val); /* Release power clamp */ for (i = 0; i <= CPU_PWR_CLAMP_STEPS; i++) bus_space_write_4(fdtbus_bs_tag, prcm, PRCM_CL_PWR_CLAMP(cluster, cpu), 0xff >> i); while (bus_space_read_4(fdtbus_bs_tag, prcm, PRCM_CL_PWR_CLAMP(cluster, cpu)) != 0) ; /* Clear power-off gating */ val = bus_space_read_4(fdtbus_bs_tag, prcm, PRCM_CL_PWROFF(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, prcm, PRCM_CL_PWROFF(cluster), val); /* De-assert power-on reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster)); val |= (1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster), val); /* De-assert core reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster)); val |= (1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster), val); } static void aw_mc_mp_start_ap(bus_space_handle_t cpuscfg, bus_space_handle_t cpuxcfg, bus_space_handle_t prcm) { int cluster, cpu; KASSERT(mp_ncpus <= 4, ("multiple clusters not yet supported")); dcache_wbinv_poc_all(); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUCFG_P_REG0, pmap_kextract((vm_offset_t)mpentry)); cluster = 0; for (cpu = 1; cpu < mp_ncpus; cpu++) aw_mc_mp_start_cpu(cpuscfg, cpuxcfg, prcm, cluster, cpu); } void a83t_mp_start_ap(platform_t plat) { bus_space_handle_t cpuscfg, cpuxcfg, prcm; if (bus_space_map(fdtbus_bs_tag, CPUCFG_BASE, CPUCFG_SIZE, 0, &cpuscfg) != 0) panic("Couldn't map the CPUCFG\n"); if (bus_space_map(fdtbus_bs_tag, CPUXCFG_BASE, CPUXCFG_SIZE, 0, &cpuxcfg) != 0) panic("Couldn't map the CPUXCFG\n"); if (bus_space_map(fdtbus_bs_tag, PRCM_BASE, PRCM_SIZE, 0, &prcm) != 0) panic("Couldn't map the PRCM\n"); aw_mc_mp_start_ap(cpuscfg, cpuxcfg, prcm); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, cpuxcfg, CPUXCFG_SIZE); bus_space_unmap(fdtbus_bs_tag, cpuscfg, CPUCFG_SIZE); bus_space_unmap(fdtbus_bs_tag, prcm, PRCM_SIZE); } Index: head/sys/arm/altera/socfpga/socfpga_mp.c =================================================================== --- head/sys/arm/altera/socfpga/socfpga_mp.c (revision 306755) +++ head/sys/arm/altera/socfpga/socfpga_mp.c (revision 306756) @@ -1,159 +1,160 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define SCU_PHYSBASE 0xFFFEC000 #define SCU_SIZE 0x100 #define SCU_CONTROL_REG 0x00 #define SCU_CONTROL_ENABLE (1 << 0) #define SCU_CONFIG_REG 0x04 #define SCU_CONFIG_REG_NCPU_MASK 0x03 #define SCU_CPUPOWER_REG 0x08 #define SCU_INV_TAGS_REG 0x0c #define SCU_DIAG_CONTROL 0x30 #define SCU_DIAG_DISABLE_MIGBIT (1 << 0) #define SCU_FILTER_START_REG 0x40 #define SCU_FILTER_END_REG 0x44 #define SCU_SECURE_ACCESS_REG 0x50 #define SCU_NONSECURE_ACCESS_REG 0x54 #define RSTMGR_PHYSBASE 0xFFD05000 #define RSTMGR_SIZE 0x100 #define MPUMODRST 0x10 #define MPUMODRST_CPU1 (1 << 1) #define RAM_PHYSBASE 0x0 #define RAM_SIZE 0x1000 extern char *mpentry_addr; static void socfpga_trampoline(void); static void socfpga_trampoline(void) { __asm __volatile( "ldr pc, 1f\n" ".globl mpentry_addr\n" "mpentry_addr:\n" "1: .space 4\n"); } void platform_mp_setmaxid(void) { int hwcpu, ncpu; /* If we've already set this don't bother to do it again. */ if (mp_ncpus != 0) return; hwcpu = 2; ncpu = hwcpu; TUNABLE_INT_FETCH("hw.ncpu", &ncpu); if (ncpu < 1 || ncpu > hwcpu) ncpu = hwcpu; mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void platform_mp_start_ap(void) { bus_space_handle_t scu, rst, ram; int reg; if (bus_space_map(fdtbus_bs_tag, SCU_PHYSBASE, SCU_SIZE, 0, &scu) != 0) panic("Couldn't map the SCU\n"); if (bus_space_map(fdtbus_bs_tag, RSTMGR_PHYSBASE, RSTMGR_SIZE, 0, &rst) != 0) panic("Couldn't map the reset manager (RSTMGR)\n"); if (bus_space_map(fdtbus_bs_tag, RAM_PHYSBASE, RAM_SIZE, 0, &ram) != 0) panic("Couldn't map the first physram page\n"); /* Invalidate SCU cache tags */ bus_space_write_4(fdtbus_bs_tag, scu, SCU_INV_TAGS_REG, 0x0000ffff); /* * Erratum ARM/MP: 764369 (problems with cache maintenance). * Setting the "disable-migratory bit" in the undocumented SCU * Diagnostic Control Register helps work around the problem. */ reg = bus_space_read_4(fdtbus_bs_tag, scu, SCU_DIAG_CONTROL); reg |= (SCU_DIAG_DISABLE_MIGBIT); bus_space_write_4(fdtbus_bs_tag, scu, SCU_DIAG_CONTROL, reg); /* Put CPU1 to reset state */ bus_space_write_4(fdtbus_bs_tag, rst, MPUMODRST, MPUMODRST_CPU1); /* Enable the SCU, then clean the cache on this core */ reg = bus_space_read_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG); reg |= (SCU_CONTROL_ENABLE); bus_space_write_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG, reg); /* Set up trampoline code */ mpentry_addr = (char *)pmap_kextract((vm_offset_t)mpentry); bus_space_write_region_4(fdtbus_bs_tag, ram, 0, (uint32_t *)&socfpga_trampoline, 8); dcache_wbinv_poc_all(); /* Put CPU1 out from reset */ bus_space_write_4(fdtbus_bs_tag, rst, MPUMODRST, 0); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, scu, SCU_SIZE); bus_space_unmap(fdtbus_bs_tag, rst, RSTMGR_SIZE); bus_space_unmap(fdtbus_bs_tag, ram, RAM_SIZE); } Index: head/sys/arm/amlogic/aml8726/aml8726_mp.c =================================================================== --- head/sys/arm/amlogic/aml8726/aml8726_mp.c (revision 306755) +++ head/sys/arm/amlogic/aml8726/aml8726_mp.c (revision 306756) @@ -1,627 +1,628 @@ /*- * Copyright 2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Amlogic aml8726 multiprocessor support. * * Some processors require powering on which involves poking registers * on the aobus and cbus ... it's expected that these locations are set * in stone. * * Locking is not used as these routines should only be called by the BP * during startup and should complete prior to the APs being released (the * issue being to ensure that a register such as AML_SOC_CPU_CLK_CNTL_REG * is not concurrently modified). */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const char *scu_compatible[] = { "arm,cortex-a5-scu", "arm,cortex-a9-scu", NULL }; static const char *scu_errata_764369[] = { "arm,cortex-a9-scu", NULL }; static const char *cpucfg_compatible[] = { "amlogic,aml8726-cpuconfig", NULL }; static struct { boolean_t errata_764369; u_long scu_size; struct resource scu_res; u_long cpucfg_size; struct resource cpucfg_res; struct resource aobus_res; struct resource cbus_res; } aml8726_smp; #define AML_SCU_CONTROL_REG 0 #define AML_SCU_CONTROL_ENABLE 1 #define AML_SCU_CONFIG_REG 4 #define AML_SCU_CONFIG_NCPU_MASK 0x3 #define AML_SCU_CPU_PWR_STATUS_REG 8 #define AML_SCU_CPU_PWR_STATUS_CPU3_MASK (3 << 24) #define AML_SCU_CPU_PWR_STATUS_CPU2_MASK (3 << 16) #define AML_SCU_CPU_PWR_STATUS_CPU1_MASK (3 << 8) #define AML_SCU_CPU_PWR_STATUS_CPU0_MASK 3 #define AML_SCU_INV_TAGS_REG 12 #define AML_SCU_DIAG_CONTROL_REG 48 #define AML_SCU_DIAG_CONTROL_DISABLE_MIGBIT 1 #define AML_CPUCONF_CONTROL_REG 0 #define AML_CPUCONF_CPU1_ADDR_REG 4 #define AML_CPUCONF_CPU2_ADDR_REG 8 #define AML_CPUCONF_CPU3_ADDR_REG 12 /* aobus */ #define AML_M8_CPU_PWR_CNTL0_REG 0xe0 #define AML_M8B_CPU_PWR_CNTL0_MODE_CPU3_MASK (3 << 22) #define AML_M8B_CPU_PWR_CNTL0_MODE_CPU2_MASK (3 << 20) #define AML_M8B_CPU_PWR_CNTL0_MODE_CPU1_MASK (3 << 18) #define AML_M8_CPU_PWR_CNTL0_ISO_CPU3 (1 << 3) #define AML_M8_CPU_PWR_CNTL0_ISO_CPU2 (1 << 2) #define AML_M8_CPU_PWR_CNTL0_ISO_CPU1 (1 << 1) #define AML_M8_CPU_PWR_CNTL1_REG 0xe4 #define AML_M8B_CPU_PWR_CNTL1_PWR_CPU3 (1 << 19) #define AML_M8B_CPU_PWR_CNTL1_PWR_CPU2 (1 << 18) #define AML_M8B_CPU_PWR_CNTL1_PWR_CPU1 (1 << 17) #define AML_M8_CPU_PWR_CNTL1_MODE_CPU3_MASK (3 << 8) #define AML_M8_CPU_PWR_CNTL1_MODE_CPU2_MASK (3 << 6) #define AML_M8_CPU_PWR_CNTL1_MODE_CPU1_MASK (3 << 4) #define AML_M8B_CPU_PWR_MEM_PD0_REG 0xf4 #define AML_M8B_CPU_PWR_MEM_PD0_CPU3 (0xf << 20) #define AML_M8B_CPU_PWR_MEM_PD0_CPU2 (0xf << 24) #define AML_M8B_CPU_PWR_MEM_PD0_CPU1 (0xf << 28) /* cbus */ #define AML_SOC_CPU_CLK_CNTL_REG 0x419c #define AML_M8_CPU_CLK_CNTL_RESET_CPU3 (1 << 27) #define AML_M8_CPU_CLK_CNTL_RESET_CPU2 (1 << 26) #define AML_M8_CPU_CLK_CNTL_RESET_CPU1 (1 << 25) #define SCU_WRITE_4(reg, value) bus_write_4(&aml8726_smp.scu_res, \ (reg), (value)) #define SCU_READ_4(reg) bus_read_4(&aml8726_smp.scu_res, (reg)) #define SCU_BARRIER(reg) bus_barrier(&aml8726_smp.scu_res, \ (reg), 4, (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define CPUCONF_WRITE_4(reg, value) bus_write_4(&aml8726_smp.cpucfg_res, \ (reg), (value)) #define CPUCONF_READ_4(reg) bus_read_4(&aml8726_smp.cpucfg_res, \ (reg)) #define CPUCONF_BARRIER(reg) bus_barrier(&aml8726_smp.cpucfg_res, \ (reg), 4, (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define AOBUS_WRITE_4(reg, value) bus_write_4(&aml8726_smp.aobus_res, \ (reg), (value)) #define AOBUS_READ_4(reg) bus_read_4(&aml8726_smp.aobus_res, \ (reg)) #define AOBUS_BARRIER(reg) bus_barrier(&aml8726_smp.aobus_res, \ (reg), 4, (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define CBUS_WRITE_4(reg, value) bus_write_4(&aml8726_smp.cbus_res, \ (reg), (value)) #define CBUS_READ_4(reg) bus_read_4(&aml8726_smp.cbus_res, \ (reg)) #define CBUS_BARRIER(reg) bus_barrier(&aml8726_smp.cbus_res, \ (reg), 4, (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) static phandle_t find_node_for_device(const char *device, const char **compatible) { int i; phandle_t node; /* * Try to access the node directly i.e. through /aliases/. */ if ((node = OF_finddevice(device)) != 0) for (i = 0; compatible[i]; i++) if (fdt_is_compatible_strict(node, compatible[i])) return node; /* * Find the node the long way. */ for (i = 0; compatible[i]; i++) { if ((node = OF_finddevice("/soc")) == 0) return (0); if ((node = fdt_find_compatible(node, compatible[i], 1)) != 0) return node; } return (0); } static int alloc_resource_for_node(phandle_t node, struct resource *res, u_long *size) { int err; u_long pbase, psize; u_long start; if ((err = fdt_get_range(OF_parent(node), 0, &pbase, &psize)) != 0 || (err = fdt_regsize(node, &start, size)) != 0) return (err); start += pbase; memset(res, 0, sizeof(*res)); res->r_bustag = fdtbus_bs_tag; err = bus_space_map(res->r_bustag, start, *size, 0, &res->r_bushandle); return (err); } static void power_on_cpu(int cpu) { uint32_t scpsr; uint32_t value; if (cpu <= 0) return; /* * Power on the CPU if the intricate details are known, otherwise * just hope for the best (it may have already be powered on by * the hardware / firmware). */ switch (aml8726_soc_hw_rev) { case AML_SOC_HW_REV_M8: case AML_SOC_HW_REV_M8B: /* * Set the SCU power status for the CPU to normal mode. */ scpsr = SCU_READ_4(AML_SCU_CPU_PWR_STATUS_REG); scpsr &= ~(AML_SCU_CPU_PWR_STATUS_CPU1_MASK << ((cpu - 1) * 8)); SCU_WRITE_4(AML_SCU_CPU_PWR_STATUS_REG, scpsr); SCU_BARRIER(AML_SCU_CPU_PWR_STATUS_REG); if (aml8726_soc_hw_rev == AML_SOC_HW_REV_M8B) { /* * Reset may cause the current power status from the * actual CPU to be written to the SCU (over-writing * the value we've just written) so set it to normal * mode as well. */ value = AOBUS_READ_4(AML_M8_CPU_PWR_CNTL0_REG); value &= ~(AML_M8B_CPU_PWR_CNTL0_MODE_CPU1_MASK << ((cpu - 1) * 2)); AOBUS_WRITE_4(AML_M8_CPU_PWR_CNTL0_REG, value); AOBUS_BARRIER(AML_M8_CPU_PWR_CNTL0_REG); } DELAY(5); /* * Assert reset. */ value = CBUS_READ_4(AML_SOC_CPU_CLK_CNTL_REG); value |= AML_M8_CPU_CLK_CNTL_RESET_CPU1 << (cpu - 1); CBUS_WRITE_4(AML_SOC_CPU_CLK_CNTL_REG, value); CBUS_BARRIER(AML_SOC_CPU_CLK_CNTL_REG); if (aml8726_soc_hw_rev == AML_SOC_HW_REV_M8B) { /* * Release RAM pull-down. */ value = AOBUS_READ_4(AML_M8B_CPU_PWR_MEM_PD0_REG); value &= ~((uint32_t)AML_M8B_CPU_PWR_MEM_PD0_CPU1 >> ((cpu - 1) * 4)); AOBUS_WRITE_4(AML_M8B_CPU_PWR_MEM_PD0_REG, value); AOBUS_BARRIER(AML_M8B_CPU_PWR_MEM_PD0_REG); } /* * Power on CPU. */ value = AOBUS_READ_4(AML_M8_CPU_PWR_CNTL1_REG); value &= ~(AML_M8_CPU_PWR_CNTL1_MODE_CPU1_MASK << ((cpu - 1) * 2)); AOBUS_WRITE_4(AML_M8_CPU_PWR_CNTL1_REG, value); AOBUS_BARRIER(AML_M8_CPU_PWR_CNTL1_REG); DELAY(10); if (aml8726_soc_hw_rev == AML_SOC_HW_REV_M8B) { /* * Wait for power on confirmation. */ for ( ; ; ) { value = AOBUS_READ_4(AML_M8_CPU_PWR_CNTL1_REG); value &= AML_M8B_CPU_PWR_CNTL1_PWR_CPU1 << (cpu - 1); if (value) break; DELAY(10); } } /* * Release peripheral clamp. */ value = AOBUS_READ_4(AML_M8_CPU_PWR_CNTL0_REG); value &= ~(AML_M8_CPU_PWR_CNTL0_ISO_CPU1 << (cpu - 1)); AOBUS_WRITE_4(AML_M8_CPU_PWR_CNTL0_REG, value); AOBUS_BARRIER(AML_M8_CPU_PWR_CNTL0_REG); /* * Release reset. */ value = CBUS_READ_4(AML_SOC_CPU_CLK_CNTL_REG); value &= ~(AML_M8_CPU_CLK_CNTL_RESET_CPU1 << (cpu - 1)); CBUS_WRITE_4(AML_SOC_CPU_CLK_CNTL_REG, value); CBUS_BARRIER(AML_SOC_CPU_CLK_CNTL_REG); if (aml8726_soc_hw_rev == AML_SOC_HW_REV_M8B) { /* * The Amlogic Linux platform code sets the SCU power * status for the CPU again for some reason so we * follow suit (perhaps in case the reset caused * a stale power status from the actual CPU to be * written to the SCU). */ SCU_WRITE_4(AML_SCU_CPU_PWR_STATUS_REG, scpsr); SCU_BARRIER(AML_SCU_CPU_PWR_STATUS_REG); } break; default: break; } } void platform_mp_setmaxid(void) { int err; int i; int ncpu; phandle_t cpucfg_node; phandle_t scu_node; uint32_t value; if (mp_ncpus != 0) return; ncpu = 1; /* * Is the hardware necessary for SMP present? */ if ((scu_node = find_node_for_device("scu", scu_compatible)) == 0) goto moveon; if ((cpucfg_node = find_node_for_device("cpuconfig", cpucfg_compatible)) == 0) goto moveon; if (alloc_resource_for_node(scu_node, &aml8726_smp.scu_res, &aml8726_smp.scu_size) != 0) panic("Could not allocate resource for SCU"); if (alloc_resource_for_node(cpucfg_node, &aml8726_smp.cpucfg_res, &aml8726_smp.cpucfg_size) != 0) panic("Could not allocate resource for CPUCONFIG"); /* * Strictly speaking the aobus and cbus may not be required in * order to start an AP (it depends on the processor), however * always mapping them in simplifies the code. */ aml8726_smp.aobus_res.r_bustag = fdtbus_bs_tag; err = bus_space_map(aml8726_smp.aobus_res.r_bustag, AML_SOC_AOBUS_BASE_ADDR, 0x100000, 0, &aml8726_smp.aobus_res.r_bushandle); if (err) panic("Could not allocate resource for AOBUS"); aml8726_smp.cbus_res.r_bustag = fdtbus_bs_tag; err = bus_space_map(aml8726_smp.cbus_res.r_bustag, AML_SOC_CBUS_BASE_ADDR, 0x100000, 0, &aml8726_smp.cbus_res.r_bushandle); if (err) panic("Could not allocate resource for CBUS"); aml8726_smp.errata_764369 = false; for (i = 0; scu_errata_764369[i]; i++) if (fdt_is_compatible_strict(scu_node, scu_errata_764369[i])) { aml8726_smp.errata_764369 = true; break; } /* * Read the number of CPUs present. */ value = SCU_READ_4(AML_SCU_CONFIG_REG); ncpu = (value & AML_SCU_CONFIG_NCPU_MASK) + 1; moveon: mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void platform_mp_start_ap(void) { int i; uint32_t reg; uint32_t value; vm_paddr_t paddr; if (mp_ncpus < 2) return; /* * Invalidate SCU cache tags. The 0x0000ffff constant invalidates * all ways on all cores 0-3. Per the ARM docs, it's harmless to * write to the bits for cores that are not present. */ SCU_WRITE_4(AML_SCU_INV_TAGS_REG, 0x0000ffff); if (aml8726_smp.errata_764369) { /* * Erratum ARM/MP: 764369 (problems with cache maintenance). * Setting the "disable-migratory bit" in the undocumented SCU * Diagnostic Control Register helps work around the problem. */ value = SCU_READ_4(AML_SCU_DIAG_CONTROL_REG); value |= AML_SCU_DIAG_CONTROL_DISABLE_MIGBIT; SCU_WRITE_4(AML_SCU_DIAG_CONTROL_REG, value); } /* * Enable the SCU, then clean the cache on this core. After these * two operations the cache tag ram in the SCU is coherent with * the contents of the cache on this core. The other cores aren't * running yet so their caches can't contain valid data yet, however * we've initialized their SCU tag ram above, so they will be * coherent from startup. */ value = SCU_READ_4(AML_SCU_CONTROL_REG); value |= AML_SCU_CONTROL_ENABLE; SCU_WRITE_4(AML_SCU_CONTROL_REG, value); SCU_BARRIER(AML_SCU_CONTROL_REG); dcache_wbinv_poc_all(); /* Set the boot address and power on each AP. */ paddr = pmap_kextract((vm_offset_t)mpentry); for (i = 1; i < mp_ncpus; i++) { reg = AML_CPUCONF_CPU1_ADDR_REG + ((i - 1) * 4); CPUCONF_WRITE_4(reg, paddr); CPUCONF_BARRIER(reg); power_on_cpu(i); } /* * Enable the APs. * * The Amlogic Linux platform code sets the lsb for some reason * in addition to the enable bit for each AP so we follow suit * (the lsb may be the enable bit for the BP, though in that case * it should already be set since it's currently running). */ value = CPUCONF_READ_4(AML_CPUCONF_CONTROL_REG); value |= 1; for (i = 1; i < mp_ncpus; i++) value |= (1 << i); CPUCONF_WRITE_4(AML_CPUCONF_CONTROL_REG, value); CPUCONF_BARRIER(AML_CPUCONF_CONTROL_REG); /* Wakeup the now enabled APs */ - armv7_sev(); + dsb(); + sev(); /* * Free the resources which are not needed after startup. */ bus_space_unmap(aml8726_smp.scu_res.r_bustag, aml8726_smp.scu_res.r_bushandle, aml8726_smp.scu_size); bus_space_unmap(aml8726_smp.cpucfg_res.r_bustag, aml8726_smp.cpucfg_res.r_bushandle, aml8726_smp.cpucfg_size); bus_space_unmap(aml8726_smp.aobus_res.r_bustag, aml8726_smp.aobus_res.r_bushandle, 0x100000); bus_space_unmap(aml8726_smp.cbus_res.r_bustag, aml8726_smp.cbus_res.r_bushandle, 0x100000); memset(&aml8726_smp, 0, sizeof(aml8726_smp)); } /* * Stub drivers for cosmetic purposes. */ struct aml8726_scu_softc { device_t dev; }; static int aml8726_scu_probe(device_t dev) { int i; for (i = 0; scu_compatible[i]; i++) if (ofw_bus_is_compatible(dev, scu_compatible[i])) break; if (!scu_compatible[i]) return (ENXIO); device_set_desc(dev, "ARM Snoop Control Unit"); return (BUS_PROBE_DEFAULT); } static int aml8726_scu_attach(device_t dev) { struct aml8726_scu_softc *sc = device_get_softc(dev); sc->dev = dev; return (0); } static int aml8726_scu_detach(device_t dev) { return (0); } static device_method_t aml8726_scu_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_scu_probe), DEVMETHOD(device_attach, aml8726_scu_attach), DEVMETHOD(device_detach, aml8726_scu_detach), DEVMETHOD_END }; static driver_t aml8726_scu_driver = { "scu", aml8726_scu_methods, sizeof(struct aml8726_scu_softc), }; static devclass_t aml8726_scu_devclass; EARLY_DRIVER_MODULE(scu, simplebus, aml8726_scu_driver, aml8726_scu_devclass, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE); struct aml8726_cpucfg_softc { device_t dev; }; static int aml8726_cpucfg_probe(device_t dev) { int i; for (i = 0; cpucfg_compatible[i]; i++) if (ofw_bus_is_compatible(dev, cpucfg_compatible[i])) break; if (!cpucfg_compatible[i]) return (ENXIO); device_set_desc(dev, "Amlogic CPU Config"); return (BUS_PROBE_DEFAULT); } static int aml8726_cpucfg_attach(device_t dev) { struct aml8726_cpucfg_softc *sc = device_get_softc(dev); sc->dev = dev; return (0); } static int aml8726_cpucfg_detach(device_t dev) { return (0); } static device_method_t aml8726_cpucfg_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_cpucfg_probe), DEVMETHOD(device_attach, aml8726_cpucfg_attach), DEVMETHOD(device_detach, aml8726_cpucfg_detach), DEVMETHOD_END }; static driver_t aml8726_cpucfg_driver = { "cpuconfig", aml8726_cpucfg_methods, sizeof(struct aml8726_cpucfg_softc), }; static devclass_t aml8726_cpucfg_devclass; EARLY_DRIVER_MODULE(cpuconfig, simplebus, aml8726_cpucfg_driver, aml8726_cpucfg_devclass, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE); Index: head/sys/arm/arm/cpufunc_asm_armv7.S =================================================================== --- head/sys/arm/arm/cpufunc_asm_armv7.S (revision 306755) +++ head/sys/arm/arm/cpufunc_asm_armv7.S (revision 306756) @@ -1,174 +1,167 @@ /*- * Copyright (c) 2010 Per Odlund * Copyright (C) 2011 MARVELL INTERNATIONAL LTD. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of MARVELL nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include .cpu cortex-a8 .Lcoherency_level: .word _C_LABEL(arm_cache_loc) .Lcache_type: .word _C_LABEL(arm_cache_type) .Larmv7_dcache_line_size: .word _C_LABEL(arm_dcache_min_line_size) .Larmv7_icache_line_size: .word _C_LABEL(arm_icache_min_line_size) .Larmv7_idcache_line_size: .word _C_LABEL(arm_idcache_min_line_size) .Lway_mask: .word 0x3ff .Lmax_index: .word 0x7fff .Lpage_mask: .word 0xfff #define PT_NOS (1 << 5) #define PT_S (1 << 1) #define PT_INNER_NC 0 #define PT_INNER_WT (1 << 0) #define PT_INNER_WB ((1 << 0) | (1 << 6)) #define PT_INNER_WBWA (1 << 6) #define PT_OUTER_NC 0 #define PT_OUTER_WT (2 << 3) #define PT_OUTER_WB (3 << 3) #define PT_OUTER_WBWA (1 << 3) #ifdef SMP #define PT_ATTR (PT_S|PT_INNER_WBWA|PT_OUTER_WBWA|PT_NOS) #else #define PT_ATTR (PT_INNER_WBWA|PT_OUTER_WBWA) #endif ENTRY(armv7_setttb) dsb orr r0, r0, #PT_ATTR mcr CP15_TTBR0(r0) isb #ifdef SMP mcr CP15_TLBIALLIS #else mcr CP15_TLBIALL #endif dsb isb RET END(armv7_setttb) #ifdef ELF_TRAMPOLINE /* Based on algorithm from ARM Architecture Reference Manual */ ENTRY(armv7_dcache_wbinv_all) stmdb sp!, {r4, r5, r6, r7, r8, r9} /* Get cache level */ ldr r0, .Lcoherency_level ldr r3, [r0] cmp r3, #0 beq Finished /* For each cache level */ mov r8, #0 Loop1: /* Get cache type for given level */ mov r2, r8, lsl #2 add r2, r2, r2 ldr r0, .Lcache_type ldr r1, [r0, r2] /* Get line size */ and r2, r1, #7 add r2, r2, #4 /* Get number of ways */ ldr r4, .Lway_mask ands r4, r4, r1, lsr #3 clz r5, r4 /* Get max index */ ldr r7, .Lmax_index ands r7, r7, r1, lsr #13 Loop2: mov r9, r4 Loop3: mov r6, r8, lsl #1 orr r6, r6, r9, lsl r5 orr r6, r6, r7, lsl r2 /* Clean and invalidate data cache by way/index */ mcr CP15_DCCISW(r6) subs r9, r9, #1 bge Loop3 subs r7, r7, #1 bge Loop2 Skip: add r8, r8, #1 cmp r3, r8 bne Loop1 Finished: dsb ldmia sp!, {r4, r5, r6, r7, r8, r9} RET END(armv7_dcache_wbinv_all) ENTRY(armv7_idcache_wbinv_all) stmdb sp!, {lr} bl armv7_dcache_wbinv_all #ifdef SMP mcr CP15_ICIALLUIS #else mcr CP15_ICIALLU #endif dsb isb ldmia sp!, {lr} RET END(armv7_idcache_wbinv_all) #endif ENTRY(armv7_cpu_sleep) dsb /* data synchronization barrier */ wfi /* wait for interrupt */ RET END(armv7_cpu_sleep) ENTRY(armv7_drain_writebuf) dsb RET END(armv7_drain_writebuf) - -ENTRY(armv7_sev) - dsb - sev - nop - RET -END(armv7_sev) Index: head/sys/arm/arm/mp_machdep.c =================================================================== --- head/sys/arm/arm/mp_machdep.c (revision 306755) +++ head/sys/arm/arm/mp_machdep.c (revision 306756) @@ -1,542 +1,541 @@ /*- * Copyright (c) 2011 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include "opt_smp.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #ifdef CPU_MV_PJ4B #include #include #endif extern struct pcpu __pcpu[]; /* used to hold the AP's until we are ready to release them */ struct mtx ap_boot_mtx; struct pcb stoppcbs[MAXCPU]; /* # of Applications processors */ volatile int mp_naps; /* Set to 1 once we're ready to let the APs out of the pen. */ volatile int aps_ready = 0; #ifndef INTRNG static int ipi_handler(void *arg); #endif void set_stackptrs(int cpu); /* Temporary variables for init_secondary() */ void *dpcpu[MAXCPU - 1]; /* Determine if we running MP machine */ int cpu_mp_probe(void) { KASSERT(mp_ncpus != 0, ("cpu_mp_probe: mp_ncpus is unset")); CPU_SETOF(0, &all_cpus); return (mp_ncpus > 1); } /* Start Application Processor via platform specific function */ static int check_ap(void) { uint32_t ms; for (ms = 0; ms < 2000; ++ms) { if ((mp_naps + 1) == mp_ncpus) return (0); /* success */ else DELAY(1000); } return (-2); } extern unsigned char _end[]; /* Initialize and fire up non-boot processors */ void cpu_mp_start(void) { int error, i; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* Reserve memory for application processors */ for(i = 0; i < (mp_ncpus - 1); i++) dpcpu[i] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, M_WAITOK | M_ZERO); dcache_wbinv_poc_all(); /* Initialize boot code and start up processors */ platform_mp_start_ap(); /* Check if ap's started properly */ error = check_ap(); if (error) printf("WARNING: Some AP's failed to start\n"); else for (i = 1; i < mp_ncpus; i++) CPU_SET(i, &all_cpus); } /* Introduce rest of cores to the world */ void cpu_mp_announce(void) { } extern vm_paddr_t pmap_pa; void init_secondary(int cpu) { struct pcpu *pc; uint32_t loop_counter; #ifndef INTRNG int start = 0, end = 0; #endif uint32_t actlr_mask, actlr_set; pmap_set_tex(); cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); reinit_mmu(pmap_kern_ttb, actlr_mask, actlr_set); cpu_setup(); /* Provide stack pointers for other processor modes. */ set_stackptrs(cpu); enable_interrupts(PSR_A); pc = &__pcpu[cpu]; /* * pcpu_init() updates queue, so it should not be executed in parallel * on several cores */ while(mp_naps < (cpu - 1)) ; pcpu_init(pc, cpu, sizeof(struct pcpu)); dpcpu_init(dpcpu[cpu - 1], cpu); #if __ARM_ARCH >= 6 && defined(DDB) dbg_monitor_init_secondary(); #endif /* Signal our startup to BSP */ atomic_add_rel_32(&mp_naps, 1); /* Spin until the BSP releases the APs */ while (!atomic_load_acq_int(&aps_ready)) { #if __ARM_ARCH >= 7 __asm __volatile("wfe"); #endif } /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pc->pc_curthread = pc->pc_idlethread; pc->pc_curpcb = pc->pc_idlethread->td_pcb; set_curthread(pc->pc_idlethread); #ifdef VFP vfp_init(); #endif /* Configure the interrupt controller */ intr_pic_init_secondary(); mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); } mtx_unlock_spin(&ap_boot_mtx); #ifndef INTRNG /* Enable ipi */ #ifdef IPI_IRQ_START start = IPI_IRQ_START; #ifdef IPI_IRQ_END end = IPI_IRQ_END; #else end = IPI_IRQ_START; #endif #endif for (int i = start; i <= end; i++) arm_unmask_irq(i); #endif /* INTRNG */ enable_interrupts(PSR_I); loop_counter = 0; while (smp_started == 0) { DELAY(100); loop_counter++; if (loop_counter == 1000) CTR0(KTR_SMP, "AP still wait for smp_started"); } /* Start per-CPU event timers. */ cpu_initclocks_ap(); CTR0(KTR_SMP, "go into scheduler"); /* Enter the scheduler */ sched_throw(NULL); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ } #ifdef INTRNG static void ipi_rendezvous(void *dummy __unused) { CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); } static void ipi_ast(void *dummy __unused) { CTR0(KTR_SMP, "IPI_AST"); } static void ipi_stop(void *dummy __unused) { u_int cpu; /* * IPI_STOP_HARD is mapped to IPI_STOP. */ CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); cpu = PCPU_GET(cpuid); savectx(&stoppcbs[cpu]); /* * CPUs are stopped when entering the debugger and at * system shutdown, both events which can precede a * panic dump. For the dump to be correct, all caches * must be flushed and invalidated, but on ARM there's * no way to broadcast a wbinv_all to other cores. * Instead, we have each core do the local wbinv_all as * part of stopping the core. The core requesting the * stop will do the l2 cache flush after all other cores * have done their l1 flushes and stopped. */ dcache_wbinv_poc_all(); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); #ifdef DDB dbg_resume_dbreg(); #endif CTR0(KTR_SMP, "IPI_STOP (restart)"); } static void ipi_preempt(void *arg) { struct trapframe *oldframe; struct thread *td; critical_enter(); td = curthread; td->td_intr_nesting_level++; oldframe = td->td_intr_frame; td->td_intr_frame = (struct trapframe *)arg; CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(td); td->td_intr_frame = oldframe; td->td_intr_nesting_level--; critical_exit(); } static void ipi_hardclock(void *arg) { struct trapframe *oldframe; struct thread *td; critical_enter(); td = curthread; td->td_intr_nesting_level++; oldframe = td->td_intr_frame; td->td_intr_frame = (struct trapframe *)arg; CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); td->td_intr_frame = oldframe; td->td_intr_nesting_level--; critical_exit(); } #else static int ipi_handler(void *arg) { u_int cpu, ipi; cpu = PCPU_GET(cpuid); ipi = pic_ipi_read((int)arg); while ((ipi != 0x3ff)) { switch (ipi) { case IPI_RENDEZVOUS: CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); break; case IPI_AST: CTR0(KTR_SMP, "IPI_AST"); break; case IPI_STOP: /* * IPI_STOP_HARD is mapped to IPI_STOP so it is not * necessary to add it in the switch. */ CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); savectx(&stoppcbs[cpu]); /* * CPUs are stopped when entering the debugger and at * system shutdown, both events which can precede a * panic dump. For the dump to be correct, all caches * must be flushed and invalidated, but on ARM there's * no way to broadcast a wbinv_all to other cores. * Instead, we have each core do the local wbinv_all as * part of stopping the core. The core requesting the * stop will do the l2 cache flush after all other cores * have done their l1 flushes and stopped. */ dcache_wbinv_poc_all(); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); #ifdef DDB dbg_resume_dbreg(); #endif CTR0(KTR_SMP, "IPI_STOP (restart)"); break; case IPI_PREEMPT: CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(curthread); break; case IPI_HARDCLOCK: CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); break; default: panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); } pic_ipi_clear(ipi); ipi = pic_ipi_read(-1); } return (FILTER_HANDLED); } #endif static void release_aps(void *dummy __unused) { uint32_t loop_counter; #ifndef INTRNG int start = 0, end = 0; #endif if (mp_ncpus == 1) return; #ifdef INTRNG intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); #else #ifdef IPI_IRQ_START start = IPI_IRQ_START; #ifdef IPI_IRQ_END end = IPI_IRQ_END; #else end = IPI_IRQ_START; #endif #endif for (int i = start; i <= end; i++) { /* * IPI handler */ /* * Use 0xdeadbeef as the argument value for irq 0, * if we used 0, the intr code will give the trap frame * pointer instead. */ arm_setup_irqhandler("ipi", ipi_handler, NULL, (void *)i, i, INTR_TYPE_MISC | INTR_EXCL, NULL); /* Enable ipi */ arm_unmask_irq(i); } #endif atomic_store_rel_int(&aps_ready, 1); /* Wake the other threads up */ -#if __ARM_ARCH >= 7 - armv7_sev(); -#endif + dsb(); + sev(); printf("Release APs\n"); for (loop_counter = 0; loop_counter < 2000; loop_counter++) { if (smp_started) return; DELAY(1000); } printf("AP's not started\n"); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); struct cpu_group * cpu_topo(void) { return (smp_topo_1level(CG_SHARE_L2, mp_ncpus, 0)); } void cpu_mp_setmaxid(void) { platform_mp_setmaxid(); } /* Sending IPI */ void ipi_all_but_self(u_int ipi) { cpuset_t other_cpus; other_cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); #ifdef INTRNG intr_ipi_send(other_cpus, ipi); #else pic_ipi_send(other_cpus, ipi); #endif } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); #ifdef INTRNG intr_ipi_send(cpus, ipi); #else pic_ipi_send(cpus, ipi); #endif } void ipi_selected(cpuset_t cpus, u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); #ifdef INTRNG intr_ipi_send(cpus, ipi); #else pic_ipi_send(cpus, ipi); #endif } Index: head/sys/arm/broadcom/bcm2835/bcm2836_mp.c =================================================================== --- head/sys/arm/broadcom/bcm2835/bcm2836_mp.c (revision 306755) +++ head/sys/arm/broadcom/bcm2835/bcm2836_mp.c (revision 306756) @@ -1,183 +1,184 @@ /*- * Copyright (C) 2015 Daisuke Aoyama * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEBUG #define DPRINTF(fmt, ...) do { \ printf("%s:%u: ", __func__, __LINE__); \ printf(fmt, ##__VA_ARGS__); \ } while (0) #else #define DPRINTF(fmt, ...) #endif #define ARM_LOCAL_BASE 0x40000000 #define ARM_LOCAL_SIZE 0x00001000 /* mailbox registers */ #define MBOXINTRCTRL_CORE(n) (0x00000050 + (0x04 * (n))) #define MBOX0SET_CORE(n) (0x00000080 + (0x10 * (n))) #define MBOX1SET_CORE(n) (0x00000084 + (0x10 * (n))) #define MBOX2SET_CORE(n) (0x00000088 + (0x10 * (n))) #define MBOX3SET_CORE(n) (0x0000008C + (0x10 * (n))) #define MBOX0CLR_CORE(n) (0x000000C0 + (0x10 * (n))) #define MBOX1CLR_CORE(n) (0x000000C4 + (0x10 * (n))) #define MBOX2CLR_CORE(n) (0x000000C8 + (0x10 * (n))) #define MBOX3CLR_CORE(n) (0x000000CC + (0x10 * (n))) static bus_space_handle_t bs_periph; #define BSRD4(addr) \ bus_space_read_4(fdtbus_bs_tag, bs_periph, (addr)) #define BSWR4(addr, val) \ bus_space_write_4(fdtbus_bs_tag, bs_periph, (addr), (val)) void bcm2836_mp_setmaxid(platform_t plat) { DPRINTF("platform_mp_setmaxid\n"); if (mp_ncpus != 0) return; mp_ncpus = 4; mp_maxid = mp_ncpus - 1; DPRINTF("mp_maxid=%d\n", mp_maxid); } void bcm2836_mp_start_ap(platform_t plat) { uint32_t val; int i, retry; DPRINTF("platform_mp_start_ap\n"); /* initialize */ if (bus_space_map(fdtbus_bs_tag, ARM_LOCAL_BASE, ARM_LOCAL_SIZE, 0, &bs_periph) != 0) panic("can't map local peripheral\n"); for (i = 0; i < mp_ncpus; i++) { /* clear mailbox 0/3 */ BSWR4(MBOX0CLR_CORE(i), 0xffffffff); BSWR4(MBOX3CLR_CORE(i), 0xffffffff); } wmb(); dcache_wbinv_poc_all(); /* boot secondary CPUs */ for (i = 1; i < mp_ncpus; i++) { /* set entry point to mailbox 3 */ BSWR4(MBOX3SET_CORE(i), (uint32_t)pmap_kextract((vm_offset_t)mpentry)); wmb(); /* wait for bootup */ retry = 1000; do { /* check entry point */ val = BSRD4(MBOX3CLR_CORE(i)); if (val == 0) break; DELAY(100); retry--; if (retry <= 0) { printf("can't start for CPU%d\n", i); break; } } while (1); /* dsb and sev */ - armv7_sev(); + dsb(); + sev(); /* recode AP in CPU map */ CPU_SET(i, &all_cpus); } } #ifndef INTRNG void pic_ipi_send(cpuset_t cpus, u_int ipi) { int i; dsb(); for (i = 0; i < mp_ncpus; i++) { if (CPU_ISSET(i, &cpus)) BSWR4(MBOX0SET_CORE(i), 1 << ipi); } wmb(); } int pic_ipi_read(int i) { uint32_t val; int cpu, ipi; cpu = PCPU_GET(cpuid); dsb(); if (i != -1) { val = BSRD4(MBOX0CLR_CORE(cpu)); if (val == 0) return (0); ipi = ffs(val) - 1; BSWR4(MBOX0CLR_CORE(cpu), 1 << ipi); dsb(); return (ipi); } return (0x3ff); } void pic_ipi_clear(int ipi) { } #endif Index: head/sys/arm/freescale/imx/imx6_mp.c =================================================================== --- head/sys/arm/freescale/imx/imx6_mp.c (revision 306755) +++ head/sys/arm/freescale/imx/imx6_mp.c (revision 306756) @@ -1,157 +1,158 @@ /*- * Copyright (c) 2014 Juergen Weiss * Copyright (c) 2014 Ian Lepore * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define SCU_PHYSBASE 0x00a00000 #define SCU_SIZE 0x00001000 #define SCU_CONTROL_REG 0x00 #define SCU_CONTROL_ENABLE (1 << 0) #define SCU_CONFIG_REG 0x04 #define SCU_CONFIG_REG_NCPU_MASK 0x03 #define SCU_CPUPOWER_REG 0x08 #define SCU_INV_TAGS_REG 0x0c #define SCU_DIAG_CONTROL 0x30 #define SCU_DIAG_DISABLE_MIGBIT (1 << 0) #define SCU_FILTER_START_REG 0x40 #define SCU_FILTER_END_REG 0x44 #define SCU_SECURE_ACCESS_REG 0x50 #define SCU_NONSECURE_ACCESS_REG 0x54 #define SRC_PHYSBASE 0x020d8000 #define SRC_SIZE 0x4000 #define SRC_CONTROL_REG 0x00 #define SRC_CONTROL_C1ENA_SHIFT 22 /* Bit for Core 1 enable */ #define SRC_CONTROL_C1RST_SHIFT 14 /* Bit for Core 1 reset */ #define SRC_GPR0_C1FUNC 0x20 /* Register for Core 1 entry func */ #define SRC_GPR1_C1ARG 0x24 /* Register for Core 1 entry arg */ void platform_mp_setmaxid(void) { bus_space_handle_t scu; int hwcpu, ncpu; uint32_t val; /* If we've already set the global vars don't bother to do it again. */ if (mp_ncpus != 0) return; if (bus_space_map(fdtbus_bs_tag, SCU_PHYSBASE, SCU_SIZE, 0, &scu) != 0) panic("Couldn't map the SCU\n"); val = bus_space_read_4(fdtbus_bs_tag, scu, SCU_CONFIG_REG); hwcpu = (val & SCU_CONFIG_REG_NCPU_MASK) + 1; bus_space_unmap(fdtbus_bs_tag, scu, SCU_SIZE); ncpu = hwcpu; TUNABLE_INT_FETCH("hw.ncpu", &ncpu); if (ncpu < 1 || ncpu > hwcpu) ncpu = hwcpu; mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void platform_mp_start_ap(void) { bus_space_handle_t scu; bus_space_handle_t src; uint32_t val; int i; if (bus_space_map(fdtbus_bs_tag, SCU_PHYSBASE, SCU_SIZE, 0, &scu) != 0) panic("Couldn't map the SCU\n"); if (bus_space_map(fdtbus_bs_tag, SRC_PHYSBASE, SRC_SIZE, 0, &src) != 0) panic("Couldn't map the system reset controller (SRC)\n"); /* * Invalidate SCU cache tags. The 0x0000ffff constant invalidates all * ways on all cores 0-3. Per the ARM docs, it's harmless to write to * the bits for cores that are not present. */ bus_space_write_4(fdtbus_bs_tag, scu, SCU_INV_TAGS_REG, 0x0000ffff); /* * Erratum ARM/MP: 764369 (problems with cache maintenance). * Setting the "disable-migratory bit" in the undocumented SCU * Diagnostic Control Register helps work around the problem. */ val = bus_space_read_4(fdtbus_bs_tag, scu, SCU_DIAG_CONTROL); bus_space_write_4(fdtbus_bs_tag, scu, SCU_DIAG_CONTROL, val | SCU_DIAG_DISABLE_MIGBIT); /* * Enable the SCU, then clean the cache on this core. After these two * operations the cache tag ram in the SCU is coherent with the contents * of the cache on this core. The other cores aren't running yet so * their caches can't contain valid data yet, but we've initialized * their SCU tag ram above, so they will be coherent from startup. */ val = bus_space_read_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG); bus_space_write_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG, val | SCU_CONTROL_ENABLE); dcache_wbinv_poc_all(); /* * For each AP core, set the entry point address and argument registers, * and set the core-enable and core-reset bits in the control register. */ val = bus_space_read_4(fdtbus_bs_tag, src, SRC_CONTROL_REG); for (i=1; i < mp_ncpus; i++) { bus_space_write_4(fdtbus_bs_tag, src, SRC_GPR0_C1FUNC + 8*i, pmap_kextract((vm_offset_t)mpentry)); bus_space_write_4(fdtbus_bs_tag, src, SRC_GPR1_C1ARG + 8*i, 0); val |= ((1 << (SRC_CONTROL_C1ENA_SHIFT - 1 + i )) | ( 1 << (SRC_CONTROL_C1RST_SHIFT - 1 + i))); } bus_space_write_4(fdtbus_bs_tag, src, SRC_CONTROL_REG, val); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, scu, SCU_SIZE); bus_space_unmap(fdtbus_bs_tag, src, SRC_SIZE); } Index: head/sys/arm/include/cpu-v6.h =================================================================== --- head/sys/arm/include/cpu-v6.h (revision 306755) +++ head/sys/arm/include/cpu-v6.h (revision 306756) @@ -1,633 +1,635 @@ /*- * Copyright 2014 Svatopluk Kraus * Copyright 2014 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef MACHINE_CPU_V6_H #define MACHINE_CPU_V6_H /* There are no user serviceable parts here, they may change without notice */ #ifndef _KERNEL #error Only include this file in the kernel #endif #include #include #include #include #if __ARM_ARCH < 6 #error Only include this file for ARMv6 #else #define CPU_ASID_KERNEL 0 void dcache_wbinv_poc_all(void); /* !!! NOT SMP coherent function !!! */ vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t); vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t); #ifdef DEV_PMU #include #define PMU_OVSR_C 0x80000000 /* Cycle Counter */ extern uint32_t ccnt_hi[MAXCPU]; extern int pmu_attched; #endif /* DEV_PMU */ +#define sev() __asm __volatile("sev" : : : "memory") +#define wfe() __asm __volatile("wfe" : : : "memory") /* * Macros to generate CP15 (system control processor) read/write functions. */ #define _FX(s...) #s #define _RF0(fname, aname...) \ static __inline register_t \ fname(void) \ { \ register_t reg; \ __asm __volatile("mrc\t" _FX(aname): "=r" (reg)); \ return(reg); \ } #define _R64F0(fname, aname) \ static __inline uint64_t \ fname(void) \ { \ uint64_t reg; \ __asm __volatile("mrrc\t" _FX(aname): "=r" (reg)); \ return(reg); \ } #define _WF0(fname, aname...) \ static __inline void \ fname(void) \ { \ __asm __volatile("mcr\t" _FX(aname)); \ } #define _WF1(fname, aname...) \ static __inline void \ fname(register_t reg) \ { \ __asm __volatile("mcr\t" _FX(aname):: "r" (reg)); \ } #define _W64F1(fname, aname...) \ static __inline void \ fname(uint64_t reg) \ { \ __asm __volatile("mcrr\t" _FX(aname):: "r" (reg)); \ } /* * Raw CP15 maintenance operations * !!! not for external use !!! */ /* TLB */ _WF0(_CP15_TLBIALL, CP15_TLBIALL) /* Invalidate entire unified TLB */ #if __ARM_ARCH >= 7 && defined SMP _WF0(_CP15_TLBIALLIS, CP15_TLBIALLIS) /* Invalidate entire unified TLB IS */ #endif _WF1(_CP15_TLBIASID, CP15_TLBIASID(%0)) /* Invalidate unified TLB by ASID */ #if __ARM_ARCH >= 7 && defined SMP _WF1(_CP15_TLBIASIDIS, CP15_TLBIASIDIS(%0)) /* Invalidate unified TLB by ASID IS */ #endif _WF1(_CP15_TLBIMVAA, CP15_TLBIMVAA(%0)) /* Invalidate unified TLB by MVA, all ASID */ #if __ARM_ARCH >= 7 && defined SMP _WF1(_CP15_TLBIMVAAIS, CP15_TLBIMVAAIS(%0)) /* Invalidate unified TLB by MVA, all ASID IS */ #endif _WF1(_CP15_TLBIMVA, CP15_TLBIMVA(%0)) /* Invalidate unified TLB by MVA */ _WF1(_CP15_TTB_SET, CP15_TTBR0(%0)) /* Cache and Branch predictor */ _WF0(_CP15_BPIALL, CP15_BPIALL) /* Branch predictor invalidate all */ #if __ARM_ARCH >= 7 && defined SMP _WF0(_CP15_BPIALLIS, CP15_BPIALLIS) /* Branch predictor invalidate all IS */ #endif _WF1(_CP15_BPIMVA, CP15_BPIMVA(%0)) /* Branch predictor invalidate by MVA */ _WF1(_CP15_DCCIMVAC, CP15_DCCIMVAC(%0)) /* Data cache clean and invalidate by MVA PoC */ _WF1(_CP15_DCCISW, CP15_DCCISW(%0)) /* Data cache clean and invalidate by set/way */ _WF1(_CP15_DCCMVAC, CP15_DCCMVAC(%0)) /* Data cache clean by MVA PoC */ #if __ARM_ARCH >= 7 _WF1(_CP15_DCCMVAU, CP15_DCCMVAU(%0)) /* Data cache clean by MVA PoU */ #endif _WF1(_CP15_DCCSW, CP15_DCCSW(%0)) /* Data cache clean by set/way */ _WF1(_CP15_DCIMVAC, CP15_DCIMVAC(%0)) /* Data cache invalidate by MVA PoC */ _WF1(_CP15_DCISW, CP15_DCISW(%0)) /* Data cache invalidate by set/way */ _WF0(_CP15_ICIALLU, CP15_ICIALLU) /* Instruction cache invalidate all PoU */ #if __ARM_ARCH >= 7 && defined SMP _WF0(_CP15_ICIALLUIS, CP15_ICIALLUIS) /* Instruction cache invalidate all PoU IS */ #endif _WF1(_CP15_ICIMVAU, CP15_ICIMVAU(%0)) /* Instruction cache invalidate */ /* * Publicly accessible functions */ /* CP14 Debug Registers */ _RF0(cp14_dbgdidr_get, CP14_DBGDIDR(%0)) _RF0(cp14_dbgprsr_get, CP14_DBGPRSR(%0)) _RF0(cp14_dbgoslsr_get, CP14_DBGOSLSR(%0)) _RF0(cp14_dbgosdlr_get, CP14_DBGOSDLR(%0)) _RF0(cp14_dbgdscrint_get, CP14_DBGDSCRint(%0)) _WF1(cp14_dbgdscr_v6_set, CP14_DBGDSCRext_V6(%0)) _WF1(cp14_dbgdscr_v7_set, CP14_DBGDSCRext_V7(%0)) _WF1(cp14_dbgvcr_set, CP14_DBGVCR(%0)) _WF1(cp14_dbgoslar_set, CP14_DBGOSLAR(%0)) /* Various control registers */ _RF0(cp15_cpacr_get, CP15_CPACR(%0)) _WF1(cp15_cpacr_set, CP15_CPACR(%0)) _RF0(cp15_dfsr_get, CP15_DFSR(%0)) _RF0(cp15_ifsr_get, CP15_IFSR(%0)) _WF1(cp15_prrr_set, CP15_PRRR(%0)) _WF1(cp15_nmrr_set, CP15_NMRR(%0)) _RF0(cp15_ttbr_get, CP15_TTBR0(%0)) _RF0(cp15_dfar_get, CP15_DFAR(%0)) #if __ARM_ARCH >= 7 _RF0(cp15_ifar_get, CP15_IFAR(%0)) _RF0(cp15_l2ctlr_get, CP15_L2CTLR(%0)) #endif _RF0(cp15_actlr_get, CP15_ACTLR(%0)) _WF1(cp15_actlr_set, CP15_ACTLR(%0)) _WF1(cp15_ats1cpr_set, CP15_ATS1CPR(%0)) _WF1(cp15_ats1cpw_set, CP15_ATS1CPW(%0)) _WF1(cp15_ats1cur_set, CP15_ATS1CUR(%0)) _WF1(cp15_ats1cuw_set, CP15_ATS1CUW(%0)) _RF0(cp15_par_get, CP15_PAR(%0)) _RF0(cp15_sctlr_get, CP15_SCTLR(%0)) /*CPU id registers */ _RF0(cp15_midr_get, CP15_MIDR(%0)) _RF0(cp15_ctr_get, CP15_CTR(%0)) _RF0(cp15_tcmtr_get, CP15_TCMTR(%0)) _RF0(cp15_tlbtr_get, CP15_TLBTR(%0)) _RF0(cp15_mpidr_get, CP15_MPIDR(%0)) _RF0(cp15_revidr_get, CP15_REVIDR(%0)) _RF0(cp15_ccsidr_get, CP15_CCSIDR(%0)) _RF0(cp15_clidr_get, CP15_CLIDR(%0)) _RF0(cp15_aidr_get, CP15_AIDR(%0)) _WF1(cp15_csselr_set, CP15_CSSELR(%0)) _RF0(cp15_id_pfr0_get, CP15_ID_PFR0(%0)) _RF0(cp15_id_pfr1_get, CP15_ID_PFR1(%0)) _RF0(cp15_id_dfr0_get, CP15_ID_DFR0(%0)) _RF0(cp15_id_afr0_get, CP15_ID_AFR0(%0)) _RF0(cp15_id_mmfr0_get, CP15_ID_MMFR0(%0)) _RF0(cp15_id_mmfr1_get, CP15_ID_MMFR1(%0)) _RF0(cp15_id_mmfr2_get, CP15_ID_MMFR2(%0)) _RF0(cp15_id_mmfr3_get, CP15_ID_MMFR3(%0)) _RF0(cp15_id_isar0_get, CP15_ID_ISAR0(%0)) _RF0(cp15_id_isar1_get, CP15_ID_ISAR1(%0)) _RF0(cp15_id_isar2_get, CP15_ID_ISAR2(%0)) _RF0(cp15_id_isar3_get, CP15_ID_ISAR3(%0)) _RF0(cp15_id_isar4_get, CP15_ID_ISAR4(%0)) _RF0(cp15_id_isar5_get, CP15_ID_ISAR5(%0)) _RF0(cp15_cbar_get, CP15_CBAR(%0)) /* Performance Monitor registers */ #if __ARM_ARCH == 6 && defined(CPU_ARM1176) _RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) _WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) _RF0(cp15_pmcr_get, CP15_PMCR(%0)) _WF1(cp15_pmcr_set, CP15_PMCR(%0)) _RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) _WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) #elif __ARM_ARCH > 6 _RF0(cp15_pmcr_get, CP15_PMCR(%0)) _WF1(cp15_pmcr_set, CP15_PMCR(%0)) _RF0(cp15_pmcnten_get, CP15_PMCNTENSET(%0)) _WF1(cp15_pmcnten_set, CP15_PMCNTENSET(%0)) _WF1(cp15_pmcnten_clr, CP15_PMCNTENCLR(%0)) _RF0(cp15_pmovsr_get, CP15_PMOVSR(%0)) _WF1(cp15_pmovsr_set, CP15_PMOVSR(%0)) _WF1(cp15_pmswinc_set, CP15_PMSWINC(%0)) _RF0(cp15_pmselr_get, CP15_PMSELR(%0)) _WF1(cp15_pmselr_set, CP15_PMSELR(%0)) _RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) _WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) _RF0(cp15_pmxevtyper_get, CP15_PMXEVTYPER(%0)) _WF1(cp15_pmxevtyper_set, CP15_PMXEVTYPER(%0)) _RF0(cp15_pmxevcntr_get, CP15_PMXEVCNTRR(%0)) _WF1(cp15_pmxevcntr_set, CP15_PMXEVCNTRR(%0)) _RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) _WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) _RF0(cp15_pminten_get, CP15_PMINTENSET(%0)) _WF1(cp15_pminten_set, CP15_PMINTENSET(%0)) _WF1(cp15_pminten_clr, CP15_PMINTENCLR(%0)) #endif _RF0(cp15_tpidrurw_get, CP15_TPIDRURW(%0)) _WF1(cp15_tpidrurw_set, CP15_TPIDRURW(%0)) _RF0(cp15_tpidruro_get, CP15_TPIDRURO(%0)) _WF1(cp15_tpidruro_set, CP15_TPIDRURO(%0)) _RF0(cp15_tpidrpwr_get, CP15_TPIDRPRW(%0)) _WF1(cp15_tpidrpwr_set, CP15_TPIDRPRW(%0)) /* Generic Timer registers - only use when you know the hardware is available */ _RF0(cp15_cntfrq_get, CP15_CNTFRQ(%0)) _WF1(cp15_cntfrq_set, CP15_CNTFRQ(%0)) _RF0(cp15_cntkctl_get, CP15_CNTKCTL(%0)) _WF1(cp15_cntkctl_set, CP15_CNTKCTL(%0)) _RF0(cp15_cntp_tval_get, CP15_CNTP_TVAL(%0)) _WF1(cp15_cntp_tval_set, CP15_CNTP_TVAL(%0)) _RF0(cp15_cntp_ctl_get, CP15_CNTP_CTL(%0)) _WF1(cp15_cntp_ctl_set, CP15_CNTP_CTL(%0)) _RF0(cp15_cntv_tval_get, CP15_CNTV_TVAL(%0)) _WF1(cp15_cntv_tval_set, CP15_CNTV_TVAL(%0)) _RF0(cp15_cntv_ctl_get, CP15_CNTV_CTL(%0)) _WF1(cp15_cntv_ctl_set, CP15_CNTV_CTL(%0)) _RF0(cp15_cnthctl_get, CP15_CNTHCTL(%0)) _WF1(cp15_cnthctl_set, CP15_CNTHCTL(%0)) _RF0(cp15_cnthp_tval_get, CP15_CNTHP_TVAL(%0)) _WF1(cp15_cnthp_tval_set, CP15_CNTHP_TVAL(%0)) _RF0(cp15_cnthp_ctl_get, CP15_CNTHP_CTL(%0)) _WF1(cp15_cnthp_ctl_set, CP15_CNTHP_CTL(%0)) _R64F0(cp15_cntpct_get, CP15_CNTPCT(%Q0, %R0)) _R64F0(cp15_cntvct_get, CP15_CNTVCT(%Q0, %R0)) _R64F0(cp15_cntp_cval_get, CP15_CNTP_CVAL(%Q0, %R0)) _W64F1(cp15_cntp_cval_set, CP15_CNTP_CVAL(%Q0, %R0)) _R64F0(cp15_cntv_cval_get, CP15_CNTV_CVAL(%Q0, %R0)) _W64F1(cp15_cntv_cval_set, CP15_CNTV_CVAL(%Q0, %R0)) _R64F0(cp15_cntvoff_get, CP15_CNTVOFF(%Q0, %R0)) _W64F1(cp15_cntvoff_set, CP15_CNTVOFF(%Q0, %R0)) _R64F0(cp15_cnthp_cval_get, CP15_CNTHP_CVAL(%Q0, %R0)) _W64F1(cp15_cnthp_cval_set, CP15_CNTHP_CVAL(%Q0, %R0)) #undef _FX #undef _RF0 #undef _WF0 #undef _WF1 /* * TLB maintenance operations. */ /* Local (i.e. not broadcasting ) operations. */ /* Flush all TLB entries (even global). */ static __inline void tlb_flush_all_local(void) { dsb(); _CP15_TLBIALL(); dsb(); } /* Flush all not global TLB entries. */ static __inline void tlb_flush_all_ng_local(void) { dsb(); _CP15_TLBIASID(CPU_ASID_KERNEL); dsb(); } /* Flush single TLB entry (even global). */ static __inline void tlb_flush_local(vm_offset_t va) { KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); dsb(); _CP15_TLBIMVA(va | CPU_ASID_KERNEL); dsb(); } /* Flush range of TLB entries (even global). */ static __inline void tlb_flush_range_local(vm_offset_t va, vm_size_t size) { vm_offset_t eva = va + size; KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, size)); dsb(); for (; va < eva; va += PAGE_SIZE) _CP15_TLBIMVA(va | CPU_ASID_KERNEL); dsb(); } /* Broadcasting operations. */ #if __ARM_ARCH >= 7 && defined SMP static __inline void tlb_flush_all(void) { dsb(); _CP15_TLBIALLIS(); dsb(); } static __inline void tlb_flush_all_ng(void) { dsb(); _CP15_TLBIASIDIS(CPU_ASID_KERNEL); dsb(); } static __inline void tlb_flush(vm_offset_t va) { KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); dsb(); _CP15_TLBIMVAAIS(va); dsb(); } static __inline void tlb_flush_range(vm_offset_t va, vm_size_t size) { vm_offset_t eva = va + size; KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, size)); dsb(); for (; va < eva; va += PAGE_SIZE) _CP15_TLBIMVAAIS(va); dsb(); } #else /* SMP */ #define tlb_flush_all() tlb_flush_all_local() #define tlb_flush_all_ng() tlb_flush_all_ng_local() #define tlb_flush(va) tlb_flush_local(va) #define tlb_flush_range(va, size) tlb_flush_range_local(va, size) #endif /* SMP */ /* * Cache maintenance operations. */ /* Sync I and D caches to PoU */ static __inline void icache_sync(vm_offset_t va, vm_size_t size) { vm_offset_t eva = va + size; dsb(); va &= ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { #if __ARM_ARCH >= 7 && defined SMP _CP15_DCCMVAU(va); #else _CP15_DCCMVAC(va); #endif } dsb(); #if __ARM_ARCH >= 7 && defined SMP _CP15_ICIALLUIS(); #else _CP15_ICIALLU(); #endif dsb(); isb(); } /* Invalidate I cache */ static __inline void icache_inv_all(void) { #if __ARM_ARCH >= 7 && defined SMP _CP15_ICIALLUIS(); #else _CP15_ICIALLU(); #endif dsb(); isb(); } /* Invalidate branch predictor buffer */ static __inline void bpb_inv_all(void) { #if __ARM_ARCH >= 7 && defined SMP _CP15_BPIALLIS(); #else _CP15_BPIALL(); #endif dsb(); isb(); } /* Write back D-cache to PoU */ static __inline void dcache_wb_pou(vm_offset_t va, vm_size_t size) { vm_offset_t eva = va + size; dsb(); va &= ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { #if __ARM_ARCH >= 7 && defined SMP _CP15_DCCMVAU(va); #else _CP15_DCCMVAC(va); #endif } dsb(); } /* * Invalidate D-cache to PoC * * Caches are invalidated from outermost to innermost as fresh cachelines * flow in this direction. In given range, if there was no dirty cacheline * in any cache before, no stale cacheline should remain in them after this * operation finishes. */ static __inline void dcache_inv_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { vm_offset_t eva = va + size; dsb(); /* invalidate L2 first */ cpu_l2cache_inv_range(pa, size); /* then L1 */ va &= ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { _CP15_DCIMVAC(va); } dsb(); } /* * Discard D-cache lines to PoC, prior to overwrite by DMA engine. * * Normal invalidation does L2 then L1 to ensure that stale data from L2 doesn't * flow into L1 while invalidating. This routine is intended to be used only * when invalidating a buffer before a DMA operation loads new data into memory. * The concern in this case is that dirty lines are not evicted to main memory, * overwriting the DMA data. For that reason, the L1 is done first to ensure * that an evicted L1 line doesn't flow to L2 after the L2 has been cleaned. */ static __inline void dcache_inv_poc_dma(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { vm_offset_t eva = va + size; /* invalidate L1 first */ dsb(); va &= ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { _CP15_DCIMVAC(va); } dsb(); /* then L2 */ cpu_l2cache_inv_range(pa, size); } /* * Write back D-cache to PoC * * Caches are written back from innermost to outermost as dirty cachelines * flow in this direction. In given range, no dirty cacheline should remain * in any cache after this operation finishes. */ static __inline void dcache_wb_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { vm_offset_t eva = va + size; dsb(); va &= ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { _CP15_DCCMVAC(va); } dsb(); cpu_l2cache_wb_range(pa, size); } /* Write back and invalidate D-cache to PoC */ static __inline void dcache_wbinv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size) { vm_offset_t va; vm_offset_t eva = sva + size; dsb(); /* write back L1 first */ va = sva & ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { _CP15_DCCMVAC(va); } dsb(); /* then write back and invalidate L2 */ cpu_l2cache_wbinv_range(pa, size); /* then invalidate L1 */ va = sva & ~cpuinfo.dcache_line_mask; for ( ; va < eva; va += cpuinfo.dcache_line_size) { _CP15_DCIMVAC(va); } dsb(); } /* Set TTB0 register */ static __inline void cp15_ttbr_set(uint32_t reg) { dsb(); _CP15_TTB_SET(reg); dsb(); _CP15_BPIALL(); dsb(); isb(); tlb_flush_all_ng_local(); } /* * Functions for address checking: * * cp15_ats1cpr_check() ... check stage 1 privileged (PL1) read access * cp15_ats1cpw_check() ... check stage 1 privileged (PL1) write access * cp15_ats1cur_check() ... check stage 1 unprivileged (PL0) read access * cp15_ats1cuw_check() ... check stage 1 unprivileged (PL0) write access * * They must be called while interrupts are disabled to get consistent result. */ static __inline int cp15_ats1cpr_check(vm_offset_t addr) { cp15_ats1cpr_set(addr); isb(); return (cp15_par_get() & 0x01 ? EFAULT : 0); } static __inline int cp15_ats1cpw_check(vm_offset_t addr) { cp15_ats1cpw_set(addr); isb(); return (cp15_par_get() & 0x01 ? EFAULT : 0); } static __inline int cp15_ats1cur_check(vm_offset_t addr) { cp15_ats1cur_set(addr); isb(); return (cp15_par_get() & 0x01 ? EFAULT : 0); } static __inline int cp15_ats1cuw_check(vm_offset_t addr) { cp15_ats1cuw_set(addr); isb(); return (cp15_par_get() & 0x01 ? EFAULT : 0); } #endif /* !__ARM_ARCH < 6 */ #endif /* !MACHINE_CPU_V6_H */ Index: head/sys/arm/include/cpufunc.h =================================================================== --- head/sys/arm/include/cpufunc.h (revision 306755) +++ head/sys/arm/include/cpufunc.h (revision 306756) @@ -1,506 +1,505 @@ /* $NetBSD: cpufunc.h,v 1.29 2003/09/06 09:08:35 rearnsha Exp $ */ /*- * Copyright (c) 1997 Mark Brinicombe. * Copyright (c) 1997 Causality Limited * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Causality Limited. * 4. The name of Causality Limited may not be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY CAUSALITY LIMITED ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL CAUSALITY LIMITED BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpufunc.h * * Prototypes for cpu, mmu and tlb related functions. * * $FreeBSD$ */ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ #ifdef _KERNEL #include #include #include static __inline void breakpoint(void) { __asm(".word 0xe7ffffff"); } struct cpu_functions { /* CPU functions */ #if __ARM_ARCH < 6 void (*cf_cpwait) (void); /* MMU functions */ u_int (*cf_control) (u_int bic, u_int eor); void (*cf_setttb) (u_int ttb); /* TLB functions */ void (*cf_tlb_flushID) (void); void (*cf_tlb_flushID_SE) (u_int va); void (*cf_tlb_flushD) (void); void (*cf_tlb_flushD_SE) (u_int va); /* * Cache operations: * * We define the following primitives: * * icache_sync_range Synchronize I-cache range * * dcache_wbinv_all Write-back and Invalidate D-cache * dcache_wbinv_range Write-back and Invalidate D-cache range * dcache_inv_range Invalidate D-cache range * dcache_wb_range Write-back D-cache range * * idcache_wbinv_all Write-back and Invalidate D-cache, * Invalidate I-cache * idcache_wbinv_range Write-back and Invalidate D-cache, * Invalidate I-cache range * * Note that the ARM term for "write-back" is "clean". We use * the term "write-back" since it's a more common way to describe * the operation. * * There are some rules that must be followed: * * ID-cache Invalidate All: * Unlike other functions, this one must never write back. * It is used to intialize the MMU when it is in an unknown * state (such as when it may have lines tagged as valid * that belong to a previous set of mappings). * * I-cache Sync range: * The goal is to synchronize the instruction stream, * so you may beed to write-back dirty D-cache blocks * first. If a range is requested, and you can't * synchronize just a range, you have to hit the whole * thing. * * D-cache Write-Back and Invalidate range: * If you can't WB-Inv a range, you must WB-Inv the * entire D-cache. * * D-cache Invalidate: * If you can't Inv the D-cache, you must Write-Back * and Invalidate. Code that uses this operation * MUST NOT assume that the D-cache will not be written * back to memory. * * D-cache Write-Back: * If you can't Write-back without doing an Inv, * that's fine. Then treat this as a WB-Inv. * Skipping the invalidate is merely an optimization. * * All operations: * Valid virtual addresses must be passed to each * cache operation. */ void (*cf_icache_sync_range) (vm_offset_t, vm_size_t); void (*cf_dcache_wbinv_all) (void); void (*cf_dcache_wbinv_range) (vm_offset_t, vm_size_t); void (*cf_dcache_inv_range) (vm_offset_t, vm_size_t); void (*cf_dcache_wb_range) (vm_offset_t, vm_size_t); void (*cf_idcache_inv_all) (void); void (*cf_idcache_wbinv_all) (void); void (*cf_idcache_wbinv_range) (vm_offset_t, vm_size_t); #endif void (*cf_l2cache_wbinv_all) (void); void (*cf_l2cache_wbinv_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_inv_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_wb_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_drain_writebuf) (void); /* Other functions */ #if __ARM_ARCH < 6 void (*cf_drain_writebuf) (void); #endif void (*cf_sleep) (int mode); #if __ARM_ARCH < 6 /* Soft functions */ void (*cf_context_switch) (void); #endif void (*cf_setup) (void); }; extern struct cpu_functions cpufuncs; extern u_int cputype; #if __ARM_ARCH < 6 #define cpu_cpwait() cpufuncs.cf_cpwait() #define cpu_control(c, e) cpufuncs.cf_control(c, e) #define cpu_setttb(t) cpufuncs.cf_setttb(t) #define cpu_tlb_flushID() cpufuncs.cf_tlb_flushID() #define cpu_tlb_flushID_SE(e) cpufuncs.cf_tlb_flushID_SE(e) #define cpu_tlb_flushD() cpufuncs.cf_tlb_flushD() #define cpu_tlb_flushD_SE(e) cpufuncs.cf_tlb_flushD_SE(e) #define cpu_icache_sync_range(a, s) cpufuncs.cf_icache_sync_range((a), (s)) #define cpu_dcache_wbinv_all() cpufuncs.cf_dcache_wbinv_all() #define cpu_dcache_wbinv_range(a, s) cpufuncs.cf_dcache_wbinv_range((a), (s)) #define cpu_dcache_inv_range(a, s) cpufuncs.cf_dcache_inv_range((a), (s)) #define cpu_dcache_wb_range(a, s) cpufuncs.cf_dcache_wb_range((a), (s)) #define cpu_idcache_inv_all() cpufuncs.cf_idcache_inv_all() #define cpu_idcache_wbinv_all() cpufuncs.cf_idcache_wbinv_all() #define cpu_idcache_wbinv_range(a, s) cpufuncs.cf_idcache_wbinv_range((a), (s)) #endif #define cpu_l2cache_wbinv_all() cpufuncs.cf_l2cache_wbinv_all() #define cpu_l2cache_wb_range(a, s) cpufuncs.cf_l2cache_wb_range((a), (s)) #define cpu_l2cache_inv_range(a, s) cpufuncs.cf_l2cache_inv_range((a), (s)) #define cpu_l2cache_wbinv_range(a, s) cpufuncs.cf_l2cache_wbinv_range((a), (s)) #define cpu_l2cache_drain_writebuf() cpufuncs.cf_l2cache_drain_writebuf() #if __ARM_ARCH < 6 #define cpu_drain_writebuf() cpufuncs.cf_drain_writebuf() #endif #define cpu_sleep(m) cpufuncs.cf_sleep(m) #define cpu_setup() cpufuncs.cf_setup() int set_cpufuncs (void); #define ARCHITECTURE_NOT_PRESENT 1 /* known but not configured */ #define ARCHITECTURE_NOT_SUPPORTED 2 /* not known */ void cpufunc_nullop (void); u_int cpu_ident (void); u_int cpufunc_control (u_int clear, u_int bic); void cpu_domains (u_int domains); u_int cpu_faultstatus (void); u_int cpu_faultaddress (void); u_int cpu_get_control (void); u_int cpu_pfr (int); #if defined(CPU_FA526) void fa526_setup (void); void fa526_setttb (u_int ttb); void fa526_context_switch (void); void fa526_cpu_sleep (int); void fa526_tlb_flushID_SE (u_int); void fa526_icache_sync_range(vm_offset_t start, vm_size_t end); void fa526_dcache_wbinv_all (void); void fa526_dcache_wbinv_range(vm_offset_t start, vm_size_t end); void fa526_dcache_inv_range (vm_offset_t start, vm_size_t end); void fa526_dcache_wb_range (vm_offset_t start, vm_size_t end); void fa526_idcache_wbinv_all(void); void fa526_idcache_wbinv_range(vm_offset_t start, vm_size_t end); #endif #if defined(CPU_ARM9) || defined(CPU_ARM9E) void arm9_setttb (u_int); void arm9_tlb_flushID_SE (u_int va); void arm9_context_switch (void); #endif #if defined(CPU_ARM9) void arm9_icache_sync_range (vm_offset_t, vm_size_t); void arm9_dcache_wbinv_all (void); void arm9_dcache_wbinv_range (vm_offset_t, vm_size_t); void arm9_dcache_inv_range (vm_offset_t, vm_size_t); void arm9_dcache_wb_range (vm_offset_t, vm_size_t); void arm9_idcache_wbinv_all (void); void arm9_idcache_wbinv_range (vm_offset_t, vm_size_t); void arm9_setup (void); extern unsigned arm9_dcache_sets_max; extern unsigned arm9_dcache_sets_inc; extern unsigned arm9_dcache_index_max; extern unsigned arm9_dcache_index_inc; #endif #if defined(CPU_ARM9E) void arm10_setup (void); u_int sheeva_control_ext (u_int, u_int); void sheeva_cpu_sleep (int); void sheeva_setttb (u_int); void sheeva_dcache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_dcache_inv_range (vm_offset_t, vm_size_t); void sheeva_dcache_wb_range (vm_offset_t, vm_size_t); void sheeva_idcache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_inv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wb_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wbinv_all (void); #endif #if defined(CPU_MV_PJ4B) void armv6_idcache_wbinv_all (void); #endif #if defined(CPU_CORTEXA8) || defined(CPU_CORTEXA_MP) || \ defined(CPU_MV_PJ4B) || defined(CPU_KRAIT) void armv7_setttb (u_int); void armv7_idcache_wbinv_all (void); void armv7_cpu_sleep (int); void armv7_setup (void); void armv7_drain_writebuf (void); -void armv7_sev (void); void armadaxp_idcache_wbinv_all (void); void cortexa_setup (void); #endif #if defined(CPU_MV_PJ4B) void pj4b_config (void); void pj4bv7_setup (void); #endif #if defined(CPU_ARM1176) void arm11_drain_writebuf (void); void arm11x6_setttb (u_int); void arm11x6_setup (void); void arm11x6_sleep (int); /* no ref. for errata */ #endif #if defined(CPU_ARM9E) void armv5_ec_setttb(u_int); void armv5_ec_icache_sync_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_wbinv_all(void); void armv5_ec_dcache_wbinv_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_inv_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_wb_range(vm_offset_t, vm_size_t); void armv5_ec_idcache_wbinv_all(void); void armv5_ec_idcache_wbinv_range(vm_offset_t, vm_size_t); #endif #if defined(CPU_ARM9) || defined(CPU_ARM9E) || \ defined(CPU_FA526) || \ defined(CPU_XSCALE_PXA2X0) || defined(CPU_XSCALE_IXP425) || \ defined(CPU_XSCALE_81342) void armv4_tlb_flushID (void); void armv4_tlb_flushD (void); void armv4_tlb_flushD_SE (u_int va); void armv4_drain_writebuf (void); void armv4_idcache_inv_all (void); #endif #if defined(CPU_XSCALE_PXA2X0) || defined(CPU_XSCALE_IXP425) || \ defined(CPU_XSCALE_81342) void xscale_cpwait (void); void xscale_cpu_sleep (int mode); u_int xscale_control (u_int clear, u_int bic); void xscale_setttb (u_int ttb); void xscale_tlb_flushID_SE (u_int va); void xscale_cache_flushID (void); void xscale_cache_flushI (void); void xscale_cache_flushD (void); void xscale_cache_flushD_SE (u_int entry); void xscale_cache_cleanID (void); void xscale_cache_cleanD (void); void xscale_cache_cleanD_E (u_int entry); void xscale_cache_clean_minidata (void); void xscale_cache_purgeID (void); void xscale_cache_purgeID_E (u_int entry); void xscale_cache_purgeD (void); void xscale_cache_purgeD_E (u_int entry); void xscale_cache_syncI (void); void xscale_cache_cleanID_rng (vm_offset_t start, vm_size_t end); void xscale_cache_cleanD_rng (vm_offset_t start, vm_size_t end); void xscale_cache_purgeID_rng (vm_offset_t start, vm_size_t end); void xscale_cache_purgeD_rng (vm_offset_t start, vm_size_t end); void xscale_cache_syncI_rng (vm_offset_t start, vm_size_t end); void xscale_cache_flushD_rng (vm_offset_t start, vm_size_t end); void xscale_context_switch (void); void xscale_setup (void); #endif /* CPU_XSCALE_PXA2X0 || CPU_XSCALE_IXP425 */ #ifdef CPU_XSCALE_81342 void xscalec3_l2cache_purge (void); void xscalec3_cache_purgeID (void); void xscalec3_cache_purgeD (void); void xscalec3_cache_cleanID (void); void xscalec3_cache_cleanD (void); void xscalec3_cache_syncI (void); void xscalec3_cache_purgeID_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_purgeD_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_cleanID_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_cleanD_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_syncI_rng (vm_offset_t start, vm_size_t end); void xscalec3_l2cache_flush_rng (vm_offset_t, vm_size_t); void xscalec3_l2cache_clean_rng (vm_offset_t start, vm_size_t end); void xscalec3_l2cache_purge_rng (vm_offset_t start, vm_size_t end); void xscalec3_setttb (u_int ttb); void xscalec3_context_switch (void); #endif /* CPU_XSCALE_81342 */ /* * Macros for manipulating CPU interrupts */ #if __ARM_ARCH < 6 #define __ARM_INTR_BITS (PSR_I | PSR_F) #else #define __ARM_INTR_BITS (PSR_I | PSR_F | PSR_A) #endif static __inline uint32_t __set_cpsr(uint32_t bic, uint32_t eor) { uint32_t tmp, ret; __asm __volatile( "mrs %0, cpsr\n" /* Get the CPSR */ "bic %1, %0, %2\n" /* Clear bits */ "eor %1, %1, %3\n" /* XOR bits */ "msr cpsr_xc, %1\n" /* Set the CPSR */ : "=&r" (ret), "=&r" (tmp) : "r" (bic), "r" (eor) : "memory"); return ret; } static __inline uint32_t disable_interrupts(uint32_t mask) { return (__set_cpsr(mask & __ARM_INTR_BITS, mask & __ARM_INTR_BITS)); } static __inline uint32_t enable_interrupts(uint32_t mask) { return (__set_cpsr(mask & __ARM_INTR_BITS, 0)); } static __inline uint32_t restore_interrupts(uint32_t old_cpsr) { return (__set_cpsr(__ARM_INTR_BITS, old_cpsr & __ARM_INTR_BITS)); } static __inline register_t intr_disable(void) { return (disable_interrupts(PSR_I | PSR_F)); } static __inline void intr_restore(register_t s) { restore_interrupts(s); } #undef __ARM_INTR_BITS /* * Functions to manipulate cpu r13 * (in arm/arm32/setstack.S) */ void set_stackptr (u_int mode, u_int address); u_int get_stackptr (u_int mode); /* * Miscellany */ int get_pc_str_offset (void); /* * CPU functions from locore.S */ void cpu_reset (void) __attribute__((__noreturn__)); /* * Cache info variables. */ /* PRIMARY CACHE VARIABLES */ extern int arm_picache_size; extern int arm_picache_line_size; extern int arm_picache_ways; extern int arm_pdcache_size; /* and unified */ extern int arm_pdcache_line_size; extern int arm_pdcache_ways; extern int arm_pcache_type; extern int arm_pcache_unified; extern int arm_dcache_align; extern int arm_dcache_align_mask; extern u_int arm_cache_level; extern u_int arm_cache_loc; extern u_int arm_cache_type[14]; #endif /* _KERNEL */ #endif /* _MACHINE_CPUFUNC_H_ */ /* End of cpufunc.h */ Index: head/sys/arm/mv/armada38x/pmsu.c =================================================================== --- head/sys/arm/mv/armada38x/pmsu.c (revision 306755) +++ head/sys/arm/mv/armada38x/pmsu.c (revision 306756) @@ -1,155 +1,156 @@ /*- * Copyright (c) 2015 Semihalf. * Copyright (c) 2015 Stormshield. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pmsu.h" static struct resource_spec pmsu_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; struct pmsu_softc { device_t dev; struct resource *res; }; static int pmsu_probe(device_t dev); static int pmsu_attach(device_t dev); static int pmsu_detach(device_t dev); static device_method_t pmsu_methods[] = { DEVMETHOD(device_probe, pmsu_probe), DEVMETHOD(device_attach, pmsu_attach), DEVMETHOD(device_detach, pmsu_detach), { 0, 0 } }; static driver_t pmsu_driver = { "pmsu", pmsu_methods, sizeof(struct pmsu_softc) }; static devclass_t pmsu_devclass; DRIVER_MODULE(pmsu, simplebus, pmsu_driver, pmsu_devclass, 0, 0); DRIVER_MODULE(pmsu, ofwbus, pmsu_driver, pmsu_devclass, 0, 0); static int pmsu_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "marvell,armada-380-pmsu")) return (ENXIO); device_set_desc(dev, "Power Management Service Unit"); return (BUS_PROBE_DEFAULT); } static int pmsu_attach(device_t dev) { struct pmsu_softc *sc; int err; sc = device_get_softc(dev); sc->dev = dev; err = bus_alloc_resources(dev, pmsu_spec, &sc->res); if (err != 0) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } return (0); } static int pmsu_detach(device_t dev) { struct pmsu_softc *sc; sc = device_get_softc(dev); bus_release_resources(dev, pmsu_spec, &sc->res); return (0); } #ifdef SMP int pmsu_boot_secondary_cpu(void) { bus_space_handle_t vaddr; int rv; rv = bus_space_map(fdtbus_bs_tag, (bus_addr_t)MV_PMSU_BASE, MV_PMSU_REGS_LEN, 0, &vaddr); if (rv != 0) return (rv); /* Boot cpu1 */ bus_space_write_4(fdtbus_bs_tag, vaddr, PMSU_BOOT_ADDR_REDIRECT_OFFSET(1), pmap_kextract((vm_offset_t)mpentry)); dcache_wbinv_poc_all(); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, vaddr, MV_PMSU_REGS_LEN); return (0); } #endif Index: head/sys/arm/nvidia/tegra124/tegra124_mp.c =================================================================== --- head/sys/arm/nvidia/tegra124/tegra124_mp.c (revision 306755) +++ head/sys/arm/nvidia/tegra124/tegra124_mp.c (revision 306756) @@ -1,127 +1,128 @@ /*- * Copyright (c) 2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define PMC_PHYSBASE 0x7000e400 #define PMC_SIZE 0x400 #define PMC_CONTROL_REG 0x0 #define PMC_PWRGATE_TOGGLE 0x30 #define PCM_PWRGATE_TOGGLE_START (1 << 8) #define PMC_PWRGATE_STATUS 0x38 #define TEGRA_EXCEPTION_VECTORS_BASE 0x6000F000 /* exception vectors */ #define TEGRA_EXCEPTION_VECTORS_SIZE 1024 #define TEGRA_EXCEPTION_VECTOR_ENTRY 0x100 void tegra124_mp_setmaxid(platform_t plat) { int ncpu; /* If we've already set the global vars don't bother to do it again. */ if (mp_ncpus != 0) return; /* Read current CP15 Cache Size ID Register */ ncpu = cp15_l2ctlr_get(); ncpu = CPUV7_L2CTLR_NPROC(ncpu); mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void tegra124_mp_start_ap(platform_t plat) { bus_space_handle_t pmc; bus_space_handle_t exvec; int i; uint32_t val; uint32_t mask; if (bus_space_map(fdtbus_bs_tag, PMC_PHYSBASE, PMC_SIZE, 0, &pmc) != 0) panic("Couldn't map the PMC\n"); if (bus_space_map(fdtbus_bs_tag, TEGRA_EXCEPTION_VECTORS_BASE, TEGRA_EXCEPTION_VECTORS_SIZE, 0, &exvec) != 0) panic("Couldn't map the exception vectors\n"); bus_space_write_4(fdtbus_bs_tag, exvec , TEGRA_EXCEPTION_VECTOR_ENTRY, pmap_kextract((vm_offset_t)mpentry)); bus_space_read_4(fdtbus_bs_tag, exvec , TEGRA_EXCEPTION_VECTOR_ENTRY); /* Wait until POWERGATE is ready (max 20 APB cycles). */ do { val = bus_space_read_4(fdtbus_bs_tag, pmc, PMC_PWRGATE_TOGGLE); } while ((val & PCM_PWRGATE_TOGGLE_START) != 0); for (i = 1; i < mp_ncpus; i++) { val = bus_space_read_4(fdtbus_bs_tag, pmc, PMC_PWRGATE_STATUS); mask = 1 << (i + 8); /* cpu mask */ if ((val & mask) == 0) { /* Wait until POWERGATE is ready (max 20 APB cycles). */ do { val = bus_space_read_4(fdtbus_bs_tag, pmc, PMC_PWRGATE_TOGGLE); } while ((val & PCM_PWRGATE_TOGGLE_START) != 0); bus_space_write_4(fdtbus_bs_tag, pmc, PMC_PWRGATE_TOGGLE, PCM_PWRGATE_TOGGLE_START | (8 + i)); /* Wait until CPU is powered */ do { val = bus_space_read_4(fdtbus_bs_tag, pmc, PMC_PWRGATE_STATUS); } while ((val & mask) == 0); } } - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, pmc, PMC_SIZE); bus_space_unmap(fdtbus_bs_tag, exvec, TEGRA_EXCEPTION_VECTORS_SIZE); } Index: head/sys/arm/rockchip/rk30xx_mp.c =================================================================== --- head/sys/arm/rockchip/rk30xx_mp.c (revision 306755) +++ head/sys/arm/rockchip/rk30xx_mp.c (revision 306756) @@ -1,171 +1,172 @@ /*- * Copyright (c) 2014 Ganbold Tsagaankhuu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define SCU_PHYSBASE 0x1013c000 #define SCU_SIZE 0x100 #define SCU_CONTROL_REG 0x00 #define SCU_CONTROL_ENABLE (1 << 0) #define SCU_STANDBY_EN (1 << 5) #define SCU_CONFIG_REG 0x04 #define SCU_CONFIG_REG_NCPU_MASK 0x03 #define SCU_CPUPOWER_REG 0x08 #define SCU_INV_TAGS_REG 0x0c #define SCU_FILTER_START_REG 0x10 #define SCU_FILTER_END_REG 0x14 #define SCU_SECURE_ACCESS_REG 0x18 #define SCU_NONSECURE_ACCESS_REG 0x1c #define IMEM_PHYSBASE 0x10080000 #define IMEM_SIZE 0x20 #define PMU_PHYSBASE 0x20004000 #define PMU_SIZE 0x100 #define PMU_PWRDN_CON 0x08 #define PMU_PWRDN_SCU (1 << 4) extern char *mpentry_addr; static void rk30xx_boot2(void); static void rk30xx_boot2(void) { __asm __volatile( "ldr pc, 1f\n" ".globl mpentry_addr\n" "mpentry_addr:\n" "1: .space 4\n"); } void platform_mp_setmaxid(void) { bus_space_handle_t scu; int ncpu; uint32_t val; if (mp_ncpus != 0) return; if (bus_space_map(fdtbus_bs_tag, SCU_PHYSBASE, SCU_SIZE, 0, &scu) != 0) panic("Could not map the SCU"); val = bus_space_read_4(fdtbus_bs_tag, scu, SCU_CONFIG_REG); ncpu = (val & SCU_CONFIG_REG_NCPU_MASK) + 1; bus_space_unmap(fdtbus_bs_tag, scu, SCU_SIZE); mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void platform_mp_start_ap(void) { bus_space_handle_t scu; bus_space_handle_t imem; bus_space_handle_t pmu; uint32_t val; int i; if (bus_space_map(fdtbus_bs_tag, SCU_PHYSBASE, SCU_SIZE, 0, &scu) != 0) panic("Could not map the SCU"); if (bus_space_map(fdtbus_bs_tag, IMEM_PHYSBASE, IMEM_SIZE, 0, &imem) != 0) panic("Could not map the IMEM"); if (bus_space_map(fdtbus_bs_tag, PMU_PHYSBASE, PMU_SIZE, 0, &pmu) != 0) panic("Could not map the PMU"); /* * Invalidate SCU cache tags. The 0x0000ffff constant invalidates all * ways on all cores 0-3. Per the ARM docs, it's harmless to write to * the bits for cores that are not present. */ bus_space_write_4(fdtbus_bs_tag, scu, SCU_INV_TAGS_REG, 0x0000ffff); /* Make sure all cores except the first are off */ val = bus_space_read_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON); for (i = 1; i < mp_ncpus; i++) val |= 1 << i; bus_space_write_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON, val); /* Enable SCU power domain */ val = bus_space_read_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON); val &= ~PMU_PWRDN_SCU; bus_space_write_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON, val); /* Enable SCU */ val = bus_space_read_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG); bus_space_write_4(fdtbus_bs_tag, scu, SCU_CONTROL_REG, val | SCU_CONTROL_ENABLE); /* * Cores will execute the code which resides at the start of * the on-chip bootram/sram after power-on. This sram region * should be reserved and the trampoline code that directs * the core to the real startup code in ram should be copied * into this sram region. * * First set boot function for the sram code. */ mpentry_addr = (char *)pmap_kextract((vm_offset_t)mpentry); /* Copy trampoline to sram, that runs during startup of the core */ bus_space_write_region_4(fdtbus_bs_tag, imem, 0, (uint32_t *)&rk30xx_boot2, 8); dcache_wbinv_poc_all(); /* Start all cores */ val = bus_space_read_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON); for (i = 1; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, pmu, PMU_PWRDN_CON, val); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, scu, SCU_SIZE); bus_space_unmap(fdtbus_bs_tag, imem, IMEM_SIZE); bus_space_unmap(fdtbus_bs_tag, pmu, PMU_SIZE); } Index: head/sys/arm/samsung/exynos/exynos5_mp.c =================================================================== --- head/sys/arm/samsung/exynos/exynos5_mp.c (revision 306755) +++ head/sys/arm/samsung/exynos/exynos5_mp.c (revision 306756) @@ -1,130 +1,131 @@ /*- * Copyright (c) 2013-2014 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #define EXYNOS_CHIPID 0x10000000 #define EXYNOS5250_SOC_ID 0x43520000 #define EXYNOS5420_SOC_ID 0xE5420000 #define EXYNOS5_SOC_ID_MASK 0xFFFFF000 #define EXYNOS_SYSRAM 0x02020000 #define EXYNOS5420_SYSRAM_NS (EXYNOS_SYSRAM + 0x53000 + 0x1c) #define EXYNOS_PMU_BASE 0x10040000 #define CORE_CONFIG(n) (0x2000 + (0x80 * (n))) #define CORE_STATUS(n) (CORE_CONFIG(n) + 0x4) #define CORE_PWR_EN 0x3 static int exynos_get_soc_id(void) { bus_addr_t chipid; int reg; if (bus_space_map(fdtbus_bs_tag, EXYNOS_CHIPID, 0x1000, 0, &chipid) != 0) panic("Couldn't map chipid\n"); reg = bus_space_read_4(fdtbus_bs_tag, chipid, 0x0); bus_space_unmap(fdtbus_bs_tag, chipid, 0x1000); return (reg & EXYNOS5_SOC_ID_MASK); } void platform_mp_setmaxid(void) { if (exynos_get_soc_id() == EXYNOS5420_SOC_ID) mp_ncpus = 4; else mp_ncpus = 2; mp_maxid = mp_ncpus - 1; } void platform_mp_start_ap(void) { bus_addr_t sysram, pmu; int err, i, j; int status; int reg; err = bus_space_map(fdtbus_bs_tag, EXYNOS_PMU_BASE, 0x20000, 0, &pmu); if (err != 0) panic("Couldn't map pmu\n"); if (exynos_get_soc_id() == EXYNOS5420_SOC_ID) reg = EXYNOS5420_SYSRAM_NS; else reg = EXYNOS_SYSRAM; err = bus_space_map(fdtbus_bs_tag, reg, 0x100, 0, &sysram); if (err != 0) panic("Couldn't map sysram\n"); /* Give power to CPUs */ for (i = 1; i < mp_ncpus; i++) { bus_space_write_4(fdtbus_bs_tag, pmu, CORE_CONFIG(i), CORE_PWR_EN); for (j = 10; j >= 0; j--) { status = bus_space_read_4(fdtbus_bs_tag, pmu, CORE_STATUS(i)); if ((status & CORE_PWR_EN) == CORE_PWR_EN) break; DELAY(10); if (j == 0) printf("Can't power on CPU%d\n", i); } } bus_space_write_4(fdtbus_bs_tag, sysram, 0x0, pmap_kextract((vm_offset_t)mpentry)); dcache_wbinv_poc_all(); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, sysram, 0x100); bus_space_unmap(fdtbus_bs_tag, pmu, 0x20000); } Index: head/sys/arm/ti/omap4/omap4_mp.c =================================================================== --- head/sys/arm/ti/omap4/omap4_mp.c (revision 306755) +++ head/sys/arm/ti/omap4/omap4_mp.c (revision 306756) @@ -1,69 +1,70 @@ /*- * Copyright (c) 2012 Olivier Houchard. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include void platform_mp_setmaxid(void) { mp_maxid = 1; mp_ncpus = 2; } void platform_mp_start_ap(void) { bus_addr_t scu_addr; if (bus_space_map(fdtbus_bs_tag, 0x48240000, 0x1000, 0, &scu_addr) != 0) panic("Couldn't map the SCU\n"); /* Enable the SCU */ *(volatile unsigned int *)scu_addr |= 1; //*(volatile unsigned int *)(scu_addr + 0x30) |= 1; dcache_wbinv_poc_all(); ti_smc0(0x200, 0xfffffdff, MODIFY_AUX_CORE_0); ti_smc0(pmap_kextract((vm_offset_t)mpentry), 0, WRITE_AUX_CORE_1); - armv7_sev(); + dsb(); + sev(); bus_space_unmap(fdtbus_bs_tag, scu_addr, 0x1000); } Index: head/sys/arm/xilinx/zy7_mp.c =================================================================== --- head/sys/arm/xilinx/zy7_mp.c (revision 306755) +++ head/sys/arm/xilinx/zy7_mp.c (revision 306756) @@ -1,98 +1,99 @@ /*- * Copyright (c) 2013 Thomas Skibo. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define ZYNQ7_CPU1_ENTRY 0xfffffff0 #define SCU_CONTROL_REG 0xf8f00000 #define SCU_CONTROL_ENABLE (1 << 0) void platform_mp_setmaxid(void) { mp_maxid = 1; mp_ncpus = 2; } void platform_mp_start_ap(void) { bus_space_handle_t scu_handle; bus_space_handle_t ocm_handle; uint32_t scu_ctrl; /* Map in SCU control register. */ if (bus_space_map(fdtbus_bs_tag, SCU_CONTROL_REG, 4, 0, &scu_handle) != 0) panic("platform_mp_start_ap: Couldn't map SCU config reg\n"); /* Set SCU enable bit. */ scu_ctrl = bus_space_read_4(fdtbus_bs_tag, scu_handle, 0); scu_ctrl |= SCU_CONTROL_ENABLE; bus_space_write_4(fdtbus_bs_tag, scu_handle, 0, scu_ctrl); bus_space_unmap(fdtbus_bs_tag, scu_handle, 4); /* Map in magic location to give entry address to CPU1. */ if (bus_space_map(fdtbus_bs_tag, ZYNQ7_CPU1_ENTRY, 4, 0, &ocm_handle) != 0) panic("platform_mp_start_ap: Couldn't map OCM\n"); /* Write start address for CPU1. */ bus_space_write_4(fdtbus_bs_tag, ocm_handle, 0, pmap_kextract((vm_offset_t)mpentry)); bus_space_unmap(fdtbus_bs_tag, ocm_handle, 4); /* * The SCU is enabled above but I think the second CPU doesn't * turn on filtering until after the wake-up below. I think that's why * things don't work if I don't put these cache ops here. Also, the * magic location, 0xfffffff0, isn't in the SCU's filtering range so it * needs a write-back too. */ dcache_wbinv_poc_all(); /* Wake up CPU1. */ - armv7_sev(); + dsb(); + sev(); }