diff --git a/sys/arm/allwinner/aw_mp.c b/sys/arm/allwinner/aw_mp.c index df798443a44a..901b50ffd2bf 100644 --- a/sys/arm/allwinner/aw_mp.c +++ b/sys/arm/allwinner/aw_mp.c @@ -1,288 +1,287 @@ /*- * Copyright (c) 2014 Ganbold Tsagaankhuu * Copyright (c) 2016 Emmanuel Vadot * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include /* Register for all dual-core SoC */ #define A20_CPUCFG_BASE 0x01c25c00 /* Register for all quad-core SoC */ #define CPUCFG_BASE 0x01f01c00 #define CPUCFG_SIZE 0x400 #define PRCM_BASE 0x01f01400 #define PRCM_SIZE 0x800 /* Register for multi-cluster SoC */ #define CPUXCFG_BASE 0x01700000 #define CPUXCFG_SIZE 0x400 #define CPU_OFFSET 0x40 #define CPU_OFFSET_CTL 0x04 #define CPU_OFFSET_STATUS 0x08 #define CPU_RST_CTL(cpuid) ((cpuid + 1) * CPU_OFFSET) #define CPU_CTL(cpuid) (((cpuid + 1) * CPU_OFFSET) + CPU_OFFSET_CTL) #define CPU_STATUS(cpuid) (((cpuid + 1) * CPU_OFFSET) + CPU_OFFSET_STATUS) #define CPU_RESET (1 << 0) #define CPU_CORE_RESET (1 << 1) #define CPUCFG_GENCTL 0x184 #define CPUCFG_P_REG0 0x1a4 #define A20_CPU1_PWR_CLAMP 0x1b0 #define CPU_PWR_CLAMP_REG 0x140 #define CPU_PWR_CLAMP(cpu) ((cpu * 4) + CPU_PWR_CLAMP_REG) #define CPU_PWR_CLAMP_STEPS 8 #define A20_CPU1_PWROFF_REG 0x1b4 #define CPU_PWROFF 0x100 #define CPUCFG_DBGCTL0 0x1e0 #define CPUCFG_DBGCTL1 0x1e4 #define CPUS_CL_RST(cl) (0x30 + (cl) * 0x4) #define CPUX_CL_CTRL0(cl) (0x0 + (cl) * 0x10) #define CPUX_CL_CTRL1(cl) (0x4 + (cl) * 0x10) #define CPUX_CL_CPU_STATUS(cl) (0x30 + (cl) * 0x4) #define CPUX_CL_RST(cl) (0x80 + (cl) * 0x4) #define PRCM_CL_PWROFF(cl) (0x100 + (cl) * 0x4) #define PRCM_CL_PWR_CLAMP(cl, cpu) (0x140 + (cl) * 0x4 + (cpu) * 0x4) void aw_mp_setmaxid(platform_t plat) { int ncpu; uint32_t reg; if (mp_ncpus != 0) return; reg = cp15_l2ctlr_get(); ncpu = CPUV7_L2CTLR_NPROC(reg); mp_ncpus = ncpu; mp_maxid = ncpu - 1; } void aw_mp_start_ap(platform_t plat) { bus_space_handle_t cpucfg; bus_space_handle_t prcm; int i, j, soc_family; uint32_t val; soc_family = allwinner_soc_family(); if (soc_family == ALLWINNERSOC_SUN7I) { if (bus_space_map(fdtbus_bs_tag, A20_CPUCFG_BASE, CPUCFG_SIZE, 0, &cpucfg) != 0) panic("Couldn't map the CPUCFG\n"); } else { if (bus_space_map(fdtbus_bs_tag, CPUCFG_BASE, CPUCFG_SIZE, 0, &cpucfg) != 0) panic("Couldn't map the CPUCFG\n"); if (bus_space_map(fdtbus_bs_tag, PRCM_BASE, PRCM_SIZE, 0, &prcm) != 0) panic("Couldn't map the PRCM\n"); } dcache_wbinv_poc_all(); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_P_REG0, pmap_kextract((vm_offset_t)mpentry)); /* * Assert nCOREPORESET low and set L1RSTDISABLE low. * Ensure DBGPWRDUP is set to LOW to prevent any external * debug access to the processor. */ for (i = 1; i < mp_ncpus; i++) bus_space_write_4(fdtbus_bs_tag, cpucfg, CPU_RST_CTL(i), 0); /* Set L1RSTDISABLE low */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_GENCTL); for (i = 1; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_GENCTL, val); /* Set DBGPWRDUP low */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1); for (i = 1; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1, val); /* Release power clamp */ for (i = 1; i < mp_ncpus; i++) for (j = 0; j <= CPU_PWR_CLAMP_STEPS; j++) { if (soc_family != ALLWINNERSOC_SUN7I) { bus_space_write_4(fdtbus_bs_tag, prcm, CPU_PWR_CLAMP(i), 0xff >> j); } else { bus_space_write_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWR_CLAMP, 0xff >> j); } } DELAY(10000); /* Clear power-off gating */ if (soc_family != ALLWINNERSOC_SUN7I) { val = bus_space_read_4(fdtbus_bs_tag, prcm, CPU_PWROFF); for (i = 0; i < mp_ncpus; i++) val &= ~(1 << i); bus_space_write_4(fdtbus_bs_tag, prcm, CPU_PWROFF, val); } else { val = bus_space_read_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWROFF_REG); val &= ~(1 << 0); bus_space_write_4(fdtbus_bs_tag, cpucfg, A20_CPU1_PWROFF_REG, val); } DELAY(1000); /* De-assert cpu core reset */ for (i = 1; i < mp_ncpus; i++) bus_space_write_4(fdtbus_bs_tag, cpucfg, CPU_RST_CTL(i), CPU_RESET | CPU_CORE_RESET); /* Assert DBGPWRDUP signal */ val = bus_space_read_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1); for (i = 1; i < mp_ncpus; i++) val |= (1 << i); bus_space_write_4(fdtbus_bs_tag, cpucfg, CPUCFG_DBGCTL1, val); dsb(); sev(); bus_space_unmap(fdtbus_bs_tag, cpucfg, CPUCFG_SIZE); if (soc_family != ALLWINNERSOC_SUN7I) bus_space_unmap(fdtbus_bs_tag, prcm, PRCM_SIZE); } static void aw_mc_mp_start_cpu(bus_space_handle_t cpuscfg, bus_space_handle_t cpuxcfg, bus_space_handle_t prcm, int cluster, int cpu) { uint32_t val; int i; /* Assert core reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster), val); /* Assert power-on reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster), val); /* Disable automatic L1 cache invalidate at reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_CTRL0(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_CTRL0(cluster), val); /* Release power clamp */ for (i = 0; i <= CPU_PWR_CLAMP_STEPS; i++) bus_space_write_4(fdtbus_bs_tag, prcm, PRCM_CL_PWR_CLAMP(cluster, cpu), 0xff >> i); while (bus_space_read_4(fdtbus_bs_tag, prcm, PRCM_CL_PWR_CLAMP(cluster, cpu)) != 0) ; /* Clear power-off gating */ val = bus_space_read_4(fdtbus_bs_tag, prcm, PRCM_CL_PWROFF(cluster)); val &= ~(1 << cpu); bus_space_write_4(fdtbus_bs_tag, prcm, PRCM_CL_PWROFF(cluster), val); /* De-assert power-on reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster)); val |= (1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUS_CL_RST(cluster), val); /* De-assert core reset */ val = bus_space_read_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster)); val |= (1 << cpu); bus_space_write_4(fdtbus_bs_tag, cpuxcfg, CPUX_CL_RST(cluster), val); } static void aw_mc_mp_start_ap(bus_space_handle_t cpuscfg, bus_space_handle_t cpuxcfg, bus_space_handle_t prcm) { int cluster, cpu; KASSERT(mp_ncpus <= 4, ("multiple clusters not yet supported")); dcache_wbinv_poc_all(); bus_space_write_4(fdtbus_bs_tag, cpuscfg, CPUCFG_P_REG0, pmap_kextract((vm_offset_t)mpentry)); cluster = 0; for (cpu = 1; cpu < mp_ncpus; cpu++) aw_mc_mp_start_cpu(cpuscfg, cpuxcfg, prcm, cluster, cpu); } void a83t_mp_start_ap(platform_t plat) { bus_space_handle_t cpuscfg, cpuxcfg, prcm; if (bus_space_map(fdtbus_bs_tag, CPUCFG_BASE, CPUCFG_SIZE, 0, &cpuscfg) != 0) panic("Couldn't map the CPUCFG\n"); if (bus_space_map(fdtbus_bs_tag, CPUXCFG_BASE, CPUXCFG_SIZE, 0, &cpuxcfg) != 0) panic("Couldn't map the CPUXCFG\n"); if (bus_space_map(fdtbus_bs_tag, PRCM_BASE, PRCM_SIZE, 0, &prcm) != 0) panic("Couldn't map the PRCM\n"); aw_mc_mp_start_ap(cpuscfg, cpuxcfg, prcm); dsb(); sev(); bus_space_unmap(fdtbus_bs_tag, cpuxcfg, CPUXCFG_SIZE); bus_space_unmap(fdtbus_bs_tag, cpuscfg, CPUCFG_SIZE); bus_space_unmap(fdtbus_bs_tag, prcm, PRCM_SIZE); } diff --git a/sys/arm/annapurna/alpine/alpine_machdep_mp.c b/sys/arm/annapurna/alpine/alpine_machdep_mp.c index 2a341540104d..94186e26e81e 100644 --- a/sys/arm/annapurna/alpine/alpine_machdep_mp.c +++ b/sys/arm/annapurna/alpine/alpine_machdep_mp.c @@ -1,252 +1,252 @@ /*- * Copyright (c) 2013 Ruslan Bukin * Copyright (c) 2015 Semihalf * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include -#include +#include #include #include #include #include #include #include #define AL_CPU_RESUME_WATERMARK_REG 0x00 #define AL_CPU_RESUME_FLAGS_REG 0x04 #define AL_CPU_RESUME_PCPU_RADDR_REG(cpu) (0x08 + 0x04 + 8*(cpu)) #define AL_CPU_RESUME_PCPU_FLAGS(cpu) (0x08 + 8*(cpu)) /* Per-CPU flags */ #define AL_CPU_RESUME_FLG_PERCPU_DONT_RESUME (1 << 2) /* The expected magic number for validating the resume addresses */ #define AL_CPU_RESUME_MAGIC_NUM 0xf0e1d200 #define AL_CPU_RESUME_MAGIC_NUM_MASK 0xffffff00 /* The expected minimal version number for validating the capabilities */ #define AL_CPU_RESUME_MIN_VER 0x000000c3 #define AL_CPU_RESUME_MIN_VER_MASK 0x000000ff /* Field controlling the boot-up of companion cores */ #define AL_NB_INIT_CONTROL (0x8) #define AL_NB_CONFIG_STATUS_PWR_CTRL(cpu) (0x2020 + (cpu)*0x100) extern bus_addr_t al_devmap_pa; extern bus_addr_t al_devmap_size; extern void mpentry(void); static int platform_mp_get_core_cnt(void); static int alpine_get_cpu_resume_base(u_long *pbase, u_long *psize); static int alpine_get_nb_base(u_long *pbase, u_long *psize); static bool alpine_validate_cpu(u_int, phandle_t, u_int, pcell_t *); static bool alpine_validate_cpu(u_int id, phandle_t child, u_int addr_cell, pcell_t *reg) { return ofw_bus_node_is_compatible(child, "arm,cortex-a15"); } static int platform_mp_get_core_cnt(void) { static int ncores = 0; int nchilds; uint32_t reg; /* Calculate ncores value only once */ if (ncores) return (ncores); reg = cp15_l2ctlr_get(); ncores = CPUV7_L2CTLR_NPROC(reg); nchilds = ofw_cpu_early_foreach(alpine_validate_cpu, false); /* Limit CPUs if DTS has configured less than available */ if ((nchilds > 0) && (nchilds < ncores)) { printf("SMP: limiting number of active CPUs to %d out of %d\n", nchilds, ncores); ncores = nchilds; } return (ncores); } void alpine_mp_setmaxid(platform_t plat) { mp_ncpus = platform_mp_get_core_cnt(); mp_maxid = mp_ncpus - 1; } static int alpine_get_cpu_resume_base(u_long *pbase, u_long *psize) { phandle_t node; u_long base = 0; u_long size = 0; if (pbase == NULL || psize == NULL) return (EINVAL); if ((node = OF_finddevice("/")) == -1) return (EFAULT); if ((node = ofw_bus_find_compatible(node, "annapurna-labs,al-cpu-resume")) == 0) return (EFAULT); if (fdt_regsize(node, &base, &size)) return (EFAULT); *pbase = base; *psize = size; return (0); } static int alpine_get_nb_base(u_long *pbase, u_long *psize) { phandle_t node; u_long base = 0; u_long size = 0; if (pbase == NULL || psize == NULL) return (EINVAL); if ((node = OF_finddevice("/")) == -1) return (EFAULT); if ((node = ofw_bus_find_compatible(node, "annapurna-labs,al-nb-service")) == 0) return (EFAULT); if (fdt_regsize(node, &base, &size)) return (EFAULT); *pbase = base; *psize = size; return (0); } void alpine_mp_start_ap(platform_t plat) { uint32_t physaddr; vm_offset_t vaddr; uint32_t val; uint32_t start_mask; u_long cpu_resume_base; u_long nb_base; u_long cpu_resume_size; u_long nb_size; bus_addr_t cpu_resume_baddr; bus_addr_t nb_baddr; int a; if (alpine_get_cpu_resume_base(&cpu_resume_base, &cpu_resume_size)) panic("Couldn't resolve cpu_resume_base address\n"); if (alpine_get_nb_base(&nb_base, &nb_size)) panic("Couldn't resolve_nb_base address\n"); /* Proceed with start addresses for additional CPUs */ if (bus_space_map(fdtbus_bs_tag, al_devmap_pa + cpu_resume_base, cpu_resume_size, 0, &cpu_resume_baddr)) panic("Couldn't map CPU-resume area"); if (bus_space_map(fdtbus_bs_tag, al_devmap_pa + nb_base, nb_size, 0, &nb_baddr)) panic("Couldn't map NB-service area"); /* Proceed with start addresses for additional CPUs */ val = bus_space_read_4(fdtbus_bs_tag, cpu_resume_baddr, AL_CPU_RESUME_WATERMARK_REG); if (((val & AL_CPU_RESUME_MAGIC_NUM_MASK) != AL_CPU_RESUME_MAGIC_NUM) || ((val & AL_CPU_RESUME_MIN_VER_MASK) < AL_CPU_RESUME_MIN_VER)) { panic("CPU-resume device is not compatible"); } vaddr = (vm_offset_t)mpentry; physaddr = pmap_kextract(vaddr); for (a = 1; a < platform_mp_get_core_cnt(); a++) { /* Power up the core */ bus_space_write_4(fdtbus_bs_tag, nb_baddr, AL_NB_CONFIG_STATUS_PWR_CTRL(a), 0); mb(); /* Enable resume */ val = bus_space_read_4(fdtbus_bs_tag, cpu_resume_baddr, AL_CPU_RESUME_PCPU_FLAGS(a)); val &= ~AL_CPU_RESUME_FLG_PERCPU_DONT_RESUME; bus_space_write_4(fdtbus_bs_tag, cpu_resume_baddr, AL_CPU_RESUME_PCPU_FLAGS(a), val); mb(); /* Set resume physical address */ bus_space_write_4(fdtbus_bs_tag, cpu_resume_baddr, AL_CPU_RESUME_PCPU_RADDR_REG(a), physaddr); mb(); } /* Release cores from reset */ if (bus_space_map(fdtbus_bs_tag, al_devmap_pa + nb_base, nb_size, 0, &nb_baddr)) panic("Couldn't map NB-service area"); start_mask = (1 << platform_mp_get_core_cnt()) - 1; /* Release cores from reset */ val = bus_space_read_4(fdtbus_bs_tag, nb_baddr, AL_NB_INIT_CONTROL); val |= start_mask; bus_space_write_4(fdtbus_bs_tag, nb_baddr, AL_NB_INIT_CONTROL, val); dsb(); bus_space_unmap(fdtbus_bs_tag, nb_baddr, nb_size); bus_space_unmap(fdtbus_bs_tag, cpu_resume_baddr, cpu_resume_size); } diff --git a/sys/arm/include/cpu-v6.h b/sys/arm/include/cpu-v6.h deleted file mode 100644 index 4b301cfd1c4e..000000000000 --- a/sys/arm/include/cpu-v6.h +++ /dev/null @@ -1,685 +0,0 @@ -/*- - * Copyright 2014 Svatopluk Kraus - * Copyright 2014 Michal Meloun - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ -#ifndef MACHINE_CPU_V6_H -#define MACHINE_CPU_V6_H - -/* There are no user serviceable parts here, they may change without notice */ -#ifndef _KERNEL -#error Only include this file in the kernel -#endif - -#include -#include -#include -#include - -/* - * Some kernel modules (dtrace all for example) are compiled - * unconditionally with -DSMP. Although it looks like a bug, - * handle this case here and in #elif condition in ARM_SMP_UP macro. - */ -#if __ARM_ARCH <= 6 && defined(SMP) && !defined(KLD_MODULE) -#error SMP option is not supported on ARMv6 -#endif - -#if __ARM_ARCH <= 6 && defined(SMP_ON_UP) -#error SMP_ON_UP option is only supported on ARMv7+ CPUs -#endif - -#if !defined(SMP) && defined(SMP_ON_UP) -#error SMP option must be defined for SMP_ON_UP option -#endif - -#define CPU_ASID_KERNEL 0 - -#if defined(SMP_ON_UP) -#define ARM_SMP_UP(smp_code, up_code) \ -do { \ - if (cpuinfo.mp_ext != 0) { \ - smp_code; \ - } else { \ - up_code; \ - } \ -} while (0) -#elif defined(SMP) && __ARM_ARCH > 6 -#define ARM_SMP_UP(smp_code, up_code) \ -do { \ - smp_code; \ -} while (0) -#else -#define ARM_SMP_UP(smp_code, up_code) \ -do { \ - up_code; \ -} while (0) -#endif - -void dcache_wbinv_poc_all(void); /* !!! NOT SMP coherent function !!! */ -vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t); -vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t); - -#ifdef DEV_PMU -#include -#define PMU_OVSR_C 0x80000000 /* Cycle Counter */ -extern uint32_t ccnt_hi[MAXCPU]; -extern int pmu_attched; -#endif /* DEV_PMU */ - -#define sev() __asm __volatile("sev" : : : "memory") -#define wfe() __asm __volatile("wfe" : : : "memory") - -/* - * Macros to generate CP15 (system control processor) read/write functions. - */ -#define _FX(s...) #s - -#define _RF0(fname, aname...) \ -static __inline uint32_t \ -fname(void) \ -{ \ - uint32_t reg; \ - __asm __volatile("mrc\t" _FX(aname): "=r" (reg)); \ - return(reg); \ -} - -#define _R64F0(fname, aname) \ -static __inline uint64_t \ -fname(void) \ -{ \ - uint64_t reg; \ - __asm __volatile("mrrc\t" _FX(aname): "=r" (reg)); \ - return(reg); \ -} - -#define _WF0(fname, aname...) \ -static __inline void \ -fname(void) \ -{ \ - __asm __volatile("mcr\t" _FX(aname)); \ -} - -#define _WF1(fname, aname...) \ -static __inline void \ -fname(uint32_t reg) \ -{ \ - __asm __volatile("mcr\t" _FX(aname):: "r" (reg)); \ -} - -#define _W64F1(fname, aname...) \ -static __inline void \ -fname(uint64_t reg) \ -{ \ - __asm __volatile("mcrr\t" _FX(aname):: "r" (reg)); \ -} - -/* - * Raw CP15 maintenance operations - * !!! not for external use !!! - */ - -/* TLB */ - -_WF0(_CP15_TLBIALL, CP15_TLBIALL) /* Invalidate entire unified TLB */ -#if __ARM_ARCH >= 7 && defined(SMP) -_WF0(_CP15_TLBIALLIS, CP15_TLBIALLIS) /* Invalidate entire unified TLB IS */ -#endif -_WF1(_CP15_TLBIASID, CP15_TLBIASID(%0)) /* Invalidate unified TLB by ASID */ -#if __ARM_ARCH >= 7 && defined(SMP) -_WF1(_CP15_TLBIASIDIS, CP15_TLBIASIDIS(%0)) /* Invalidate unified TLB by ASID IS */ -#endif -_WF1(_CP15_TLBIMVAA, CP15_TLBIMVAA(%0)) /* Invalidate unified TLB by MVA, all ASID */ -#if __ARM_ARCH >= 7 && defined(SMP) -_WF1(_CP15_TLBIMVAAIS, CP15_TLBIMVAAIS(%0)) /* Invalidate unified TLB by MVA, all ASID IS */ -#endif -_WF1(_CP15_TLBIMVA, CP15_TLBIMVA(%0)) /* Invalidate unified TLB by MVA */ - -_WF1(_CP15_TTB_SET, CP15_TTBR0(%0)) - -/* Cache and Branch predictor */ - -_WF0(_CP15_BPIALL, CP15_BPIALL) /* Branch predictor invalidate all */ -#if __ARM_ARCH >= 7 && defined(SMP) -_WF0(_CP15_BPIALLIS, CP15_BPIALLIS) /* Branch predictor invalidate all IS */ -#endif -_WF1(_CP15_BPIMVA, CP15_BPIMVA(%0)) /* Branch predictor invalidate by MVA */ -_WF1(_CP15_DCCIMVAC, CP15_DCCIMVAC(%0)) /* Data cache clean and invalidate by MVA PoC */ -_WF1(_CP15_DCCISW, CP15_DCCISW(%0)) /* Data cache clean and invalidate by set/way */ -_WF1(_CP15_DCCMVAC, CP15_DCCMVAC(%0)) /* Data cache clean by MVA PoC */ -#if __ARM_ARCH >= 7 -_WF1(_CP15_DCCMVAU, CP15_DCCMVAU(%0)) /* Data cache clean by MVA PoU */ -#endif -_WF1(_CP15_DCCSW, CP15_DCCSW(%0)) /* Data cache clean by set/way */ -_WF1(_CP15_DCIMVAC, CP15_DCIMVAC(%0)) /* Data cache invalidate by MVA PoC */ -_WF1(_CP15_DCISW, CP15_DCISW(%0)) /* Data cache invalidate by set/way */ -_WF0(_CP15_ICIALLU, CP15_ICIALLU) /* Instruction cache invalidate all PoU */ -#if __ARM_ARCH >= 7 && defined(SMP) -_WF0(_CP15_ICIALLUIS, CP15_ICIALLUIS) /* Instruction cache invalidate all PoU IS */ -#endif -_WF1(_CP15_ICIMVAU, CP15_ICIMVAU(%0)) /* Instruction cache invalidate */ - -/* - * Publicly accessible functions - */ - -/* CP14 Debug Registers */ -_RF0(cp14_dbgdidr_get, CP14_DBGDIDR(%0)) -_RF0(cp14_dbgprsr_get, CP14_DBGPRSR(%0)) -_RF0(cp14_dbgoslsr_get, CP14_DBGOSLSR(%0)) -_RF0(cp14_dbgosdlr_get, CP14_DBGOSDLR(%0)) -_RF0(cp14_dbgdscrint_get, CP14_DBGDSCRint(%0)) - -_WF1(cp14_dbgdscr_v6_set, CP14_DBGDSCRext_V6(%0)) -_WF1(cp14_dbgdscr_v7_set, CP14_DBGDSCRext_V7(%0)) -_WF1(cp14_dbgvcr_set, CP14_DBGVCR(%0)) -_WF1(cp14_dbgoslar_set, CP14_DBGOSLAR(%0)) - -/* Various control registers */ - -_RF0(cp15_cpacr_get, CP15_CPACR(%0)) -_WF1(cp15_cpacr_set, CP15_CPACR(%0)) -_RF0(cp15_dfsr_get, CP15_DFSR(%0)) -_RF0(cp15_ifsr_get, CP15_IFSR(%0)) -_WF1(cp15_prrr_set, CP15_PRRR(%0)) -_WF1(cp15_nmrr_set, CP15_NMRR(%0)) -_RF0(cp15_ttbr_get, CP15_TTBR0(%0)) -_RF0(cp15_dfar_get, CP15_DFAR(%0)) -#if __ARM_ARCH >= 7 -_RF0(cp15_ifar_get, CP15_IFAR(%0)) -_RF0(cp15_l2ctlr_get, CP15_L2CTLR(%0)) -#endif -_RF0(cp15_actlr_get, CP15_ACTLR(%0)) -_WF1(cp15_actlr_set, CP15_ACTLR(%0)) -_WF1(cp15_ats1cpr_set, CP15_ATS1CPR(%0)) -_WF1(cp15_ats1cpw_set, CP15_ATS1CPW(%0)) -_WF1(cp15_ats1cur_set, CP15_ATS1CUR(%0)) -_WF1(cp15_ats1cuw_set, CP15_ATS1CUW(%0)) -_RF0(cp15_par_get, CP15_PAR(%0)) -_RF0(cp15_sctlr_get, CP15_SCTLR(%0)) - -/*CPU id registers */ -_RF0(cp15_midr_get, CP15_MIDR(%0)) -_RF0(cp15_ctr_get, CP15_CTR(%0)) -_RF0(cp15_tcmtr_get, CP15_TCMTR(%0)) -_RF0(cp15_tlbtr_get, CP15_TLBTR(%0)) -_RF0(cp15_mpidr_get, CP15_MPIDR(%0)) -_RF0(cp15_revidr_get, CP15_REVIDR(%0)) -_RF0(cp15_ccsidr_get, CP15_CCSIDR(%0)) -_RF0(cp15_clidr_get, CP15_CLIDR(%0)) -_RF0(cp15_aidr_get, CP15_AIDR(%0)) -_WF1(cp15_csselr_set, CP15_CSSELR(%0)) -_RF0(cp15_id_pfr0_get, CP15_ID_PFR0(%0)) -_RF0(cp15_id_pfr1_get, CP15_ID_PFR1(%0)) -_RF0(cp15_id_dfr0_get, CP15_ID_DFR0(%0)) -_RF0(cp15_id_afr0_get, CP15_ID_AFR0(%0)) -_RF0(cp15_id_mmfr0_get, CP15_ID_MMFR0(%0)) -_RF0(cp15_id_mmfr1_get, CP15_ID_MMFR1(%0)) -_RF0(cp15_id_mmfr2_get, CP15_ID_MMFR2(%0)) -_RF0(cp15_id_mmfr3_get, CP15_ID_MMFR3(%0)) -_RF0(cp15_id_isar0_get, CP15_ID_ISAR0(%0)) -_RF0(cp15_id_isar1_get, CP15_ID_ISAR1(%0)) -_RF0(cp15_id_isar2_get, CP15_ID_ISAR2(%0)) -_RF0(cp15_id_isar3_get, CP15_ID_ISAR3(%0)) -_RF0(cp15_id_isar4_get, CP15_ID_ISAR4(%0)) -_RF0(cp15_id_isar5_get, CP15_ID_ISAR5(%0)) -_RF0(cp15_cbar_get, CP15_CBAR(%0)) - -/* Performance Monitor registers */ - -#if __ARM_ARCH == 6 && defined(CPU_ARM1176) -_RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) -_WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) -_RF0(cp15_pmcr_get, CP15_PMCR(%0)) -_WF1(cp15_pmcr_set, CP15_PMCR(%0)) -_RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) -_WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) -#elif __ARM_ARCH > 6 -_RF0(cp15_pmcr_get, CP15_PMCR(%0)) -_WF1(cp15_pmcr_set, CP15_PMCR(%0)) -_RF0(cp15_pmcnten_get, CP15_PMCNTENSET(%0)) -_WF1(cp15_pmcnten_set, CP15_PMCNTENSET(%0)) -_WF1(cp15_pmcnten_clr, CP15_PMCNTENCLR(%0)) -_RF0(cp15_pmovsr_get, CP15_PMOVSR(%0)) -_WF1(cp15_pmovsr_set, CP15_PMOVSR(%0)) -_WF1(cp15_pmswinc_set, CP15_PMSWINC(%0)) -_RF0(cp15_pmselr_get, CP15_PMSELR(%0)) -_WF1(cp15_pmselr_set, CP15_PMSELR(%0)) -_RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) -_WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) -_RF0(cp15_pmxevtyper_get, CP15_PMXEVTYPER(%0)) -_WF1(cp15_pmxevtyper_set, CP15_PMXEVTYPER(%0)) -_RF0(cp15_pmxevcntr_get, CP15_PMXEVCNTRR(%0)) -_WF1(cp15_pmxevcntr_set, CP15_PMXEVCNTRR(%0)) -_RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) -_WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) -_RF0(cp15_pminten_get, CP15_PMINTENSET(%0)) -_WF1(cp15_pminten_set, CP15_PMINTENSET(%0)) -_WF1(cp15_pminten_clr, CP15_PMINTENCLR(%0)) -#endif - -_RF0(cp15_tpidrurw_get, CP15_TPIDRURW(%0)) -_WF1(cp15_tpidrurw_set, CP15_TPIDRURW(%0)) -_RF0(cp15_tpidruro_get, CP15_TPIDRURO(%0)) -_WF1(cp15_tpidruro_set, CP15_TPIDRURO(%0)) -_RF0(cp15_tpidrpwr_get, CP15_TPIDRPRW(%0)) -_WF1(cp15_tpidrpwr_set, CP15_TPIDRPRW(%0)) - -/* Generic Timer registers - only use when you know the hardware is available */ -_RF0(cp15_cntfrq_get, CP15_CNTFRQ(%0)) -_WF1(cp15_cntfrq_set, CP15_CNTFRQ(%0)) -_RF0(cp15_cntkctl_get, CP15_CNTKCTL(%0)) -_WF1(cp15_cntkctl_set, CP15_CNTKCTL(%0)) -_RF0(cp15_cntp_tval_get, CP15_CNTP_TVAL(%0)) -_WF1(cp15_cntp_tval_set, CP15_CNTP_TVAL(%0)) -_RF0(cp15_cntp_ctl_get, CP15_CNTP_CTL(%0)) -_WF1(cp15_cntp_ctl_set, CP15_CNTP_CTL(%0)) -_RF0(cp15_cntv_tval_get, CP15_CNTV_TVAL(%0)) -_WF1(cp15_cntv_tval_set, CP15_CNTV_TVAL(%0)) -_RF0(cp15_cntv_ctl_get, CP15_CNTV_CTL(%0)) -_WF1(cp15_cntv_ctl_set, CP15_CNTV_CTL(%0)) -_RF0(cp15_cnthctl_get, CP15_CNTHCTL(%0)) -_WF1(cp15_cnthctl_set, CP15_CNTHCTL(%0)) -_RF0(cp15_cnthp_tval_get, CP15_CNTHP_TVAL(%0)) -_WF1(cp15_cnthp_tval_set, CP15_CNTHP_TVAL(%0)) -_RF0(cp15_cnthp_ctl_get, CP15_CNTHP_CTL(%0)) -_WF1(cp15_cnthp_ctl_set, CP15_CNTHP_CTL(%0)) - -_R64F0(cp15_cntpct_get, CP15_CNTPCT(%Q0, %R0)) -_R64F0(cp15_cntvct_get, CP15_CNTVCT(%Q0, %R0)) -_R64F0(cp15_cntp_cval_get, CP15_CNTP_CVAL(%Q0, %R0)) -_W64F1(cp15_cntp_cval_set, CP15_CNTP_CVAL(%Q0, %R0)) -_R64F0(cp15_cntv_cval_get, CP15_CNTV_CVAL(%Q0, %R0)) -_W64F1(cp15_cntv_cval_set, CP15_CNTV_CVAL(%Q0, %R0)) -_R64F0(cp15_cntvoff_get, CP15_CNTVOFF(%Q0, %R0)) -_W64F1(cp15_cntvoff_set, CP15_CNTVOFF(%Q0, %R0)) -_R64F0(cp15_cnthp_cval_get, CP15_CNTHP_CVAL(%Q0, %R0)) -_W64F1(cp15_cnthp_cval_set, CP15_CNTHP_CVAL(%Q0, %R0)) - -#undef _FX -#undef _RF0 -#undef _WF0 -#undef _WF1 - -/* - * TLB maintenance operations. - */ - -/* Local (i.e. not broadcasting ) operations. */ - -/* Flush all TLB entries (even global). */ -static __inline void -tlb_flush_all_local(void) -{ - - dsb(); - _CP15_TLBIALL(); - dsb(); -} - -/* Flush all not global TLB entries. */ -static __inline void -tlb_flush_all_ng_local(void) -{ - - dsb(); - _CP15_TLBIASID(CPU_ASID_KERNEL); - dsb(); -} - -/* Flush single TLB entry (even global). */ -static __inline void -tlb_flush_local(vm_offset_t va) -{ - - KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); - - dsb(); - _CP15_TLBIMVA(va | CPU_ASID_KERNEL); - dsb(); -} - -/* Flush range of TLB entries (even global). */ -static __inline void -tlb_flush_range_local(vm_offset_t va, vm_size_t size) -{ - vm_offset_t eva = va + size; - - KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); - KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, - size)); - - dsb(); - for (; va < eva; va += PAGE_SIZE) - _CP15_TLBIMVA(va | CPU_ASID_KERNEL); - dsb(); -} - -/* Broadcasting operations. */ -#if __ARM_ARCH >= 7 && defined(SMP) - -static __inline void -tlb_flush_all(void) -{ - - dsb(); - ARM_SMP_UP( - _CP15_TLBIALLIS(), - _CP15_TLBIALL() - ); - dsb(); -} - -static __inline void -tlb_flush_all_ng(void) -{ - - dsb(); - ARM_SMP_UP( - _CP15_TLBIASIDIS(CPU_ASID_KERNEL), - _CP15_TLBIASID(CPU_ASID_KERNEL) - ); - dsb(); -} - -static __inline void -tlb_flush(vm_offset_t va) -{ - - KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); - - dsb(); - ARM_SMP_UP( - _CP15_TLBIMVAAIS(va), - _CP15_TLBIMVA(va | CPU_ASID_KERNEL) - ); - dsb(); -} - -static __inline void -tlb_flush_range(vm_offset_t va, vm_size_t size) -{ - vm_offset_t eva = va + size; - - KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); - KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, - size)); - - dsb(); - ARM_SMP_UP( - { - for (; va < eva; va += PAGE_SIZE) - _CP15_TLBIMVAAIS(va); - }, - { - for (; va < eva; va += PAGE_SIZE) - _CP15_TLBIMVA(va | CPU_ASID_KERNEL); - } - ); - dsb(); -} -#else /* __ARM_ARCH < 7 */ - -#define tlb_flush_all() tlb_flush_all_local() -#define tlb_flush_all_ng() tlb_flush_all_ng_local() -#define tlb_flush(va) tlb_flush_local(va) -#define tlb_flush_range(va, size) tlb_flush_range_local(va, size) - -#endif /* __ARM_ARCH < 7 */ - -/* - * Cache maintenance operations. - */ - -/* Sync I and D caches to PoU */ -static __inline void -icache_sync(vm_offset_t va, vm_size_t size) -{ - vm_offset_t eva = va + size; - - dsb(); - va &= ~cpuinfo.dcache_line_mask; - - for ( ; va < eva; va += cpuinfo.dcache_line_size) { -#if __ARM_ARCH >= 7 - _CP15_DCCMVAU(va); -#else - _CP15_DCCMVAC(va); -#endif - } - dsb(); - ARM_SMP_UP( - _CP15_ICIALLUIS(), - _CP15_ICIALLU() - ); - dsb(); - isb(); -} - -/* Invalidate I cache */ -static __inline void -icache_inv_all(void) -{ - - ARM_SMP_UP( - _CP15_ICIALLUIS(), - _CP15_ICIALLU() - ); - dsb(); - isb(); -} - -/* Invalidate branch predictor buffer */ -static __inline void -bpb_inv_all(void) -{ - - ARM_SMP_UP( - _CP15_BPIALLIS(), - _CP15_BPIALL() - ); - dsb(); - isb(); -} - -/* Write back D-cache to PoU */ -static __inline void -dcache_wb_pou(vm_offset_t va, vm_size_t size) -{ - vm_offset_t eva = va + size; - - dsb(); - va &= ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { -#if __ARM_ARCH >= 7 - _CP15_DCCMVAU(va); -#else - _CP15_DCCMVAC(va); -#endif - } - dsb(); -} - -/* - * Invalidate D-cache to PoC - * - * Caches are invalidated from outermost to innermost as fresh cachelines - * flow in this direction. In given range, if there was no dirty cacheline - * in any cache before, no stale cacheline should remain in them after this - * operation finishes. - */ -static __inline void -dcache_inv_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) -{ - vm_offset_t eva = va + size; - - dsb(); - /* invalidate L2 first */ - cpu_l2cache_inv_range(pa, size); - - /* then L1 */ - va &= ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { - _CP15_DCIMVAC(va); - } - dsb(); -} - -/* - * Discard D-cache lines to PoC, prior to overwrite by DMA engine. - * - * Normal invalidation does L2 then L1 to ensure that stale data from L2 doesn't - * flow into L1 while invalidating. This routine is intended to be used only - * when invalidating a buffer before a DMA operation loads new data into memory. - * The concern in this case is that dirty lines are not evicted to main memory, - * overwriting the DMA data. For that reason, the L1 is done first to ensure - * that an evicted L1 line doesn't flow to L2 after the L2 has been cleaned. - */ -static __inline void -dcache_inv_poc_dma(vm_offset_t va, vm_paddr_t pa, vm_size_t size) -{ - vm_offset_t eva = va + size; - - /* invalidate L1 first */ - dsb(); - va &= ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { - _CP15_DCIMVAC(va); - } - dsb(); - - /* then L2 */ - cpu_l2cache_inv_range(pa, size); -} - -/* - * Write back D-cache to PoC - * - * Caches are written back from innermost to outermost as dirty cachelines - * flow in this direction. In given range, no dirty cacheline should remain - * in any cache after this operation finishes. - */ -static __inline void -dcache_wb_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) -{ - vm_offset_t eva = va + size; - - dsb(); - va &= ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { - _CP15_DCCMVAC(va); - } - dsb(); - - cpu_l2cache_wb_range(pa, size); -} - -/* Write back and invalidate D-cache to PoC */ -static __inline void -dcache_wbinv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size) -{ - vm_offset_t va; - vm_offset_t eva = sva + size; - - dsb(); - /* write back L1 first */ - va = sva & ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { - _CP15_DCCMVAC(va); - } - dsb(); - - /* then write back and invalidate L2 */ - cpu_l2cache_wbinv_range(pa, size); - - /* then invalidate L1 */ - va = sva & ~cpuinfo.dcache_line_mask; - for ( ; va < eva; va += cpuinfo.dcache_line_size) { - _CP15_DCIMVAC(va); - } - dsb(); -} - -/* Set TTB0 register */ -static __inline void -cp15_ttbr_set(uint32_t reg) -{ - dsb(); - _CP15_TTB_SET(reg); - dsb(); - _CP15_BPIALL(); - dsb(); - isb(); - tlb_flush_all_ng_local(); -} - -/* - * Functions for address checking: - * - * cp15_ats1cpr_check() ... check stage 1 privileged (PL1) read access - * cp15_ats1cpw_check() ... check stage 1 privileged (PL1) write access - * cp15_ats1cur_check() ... check stage 1 unprivileged (PL0) read access - * cp15_ats1cuw_check() ... check stage 1 unprivileged (PL0) write access - * - * They must be called while interrupts are disabled to get consistent result. - */ -static __inline int -cp15_ats1cpr_check(vm_offset_t addr) -{ - - cp15_ats1cpr_set(addr); - isb(); - return (cp15_par_get() & 0x01 ? EFAULT : 0); -} - -static __inline int -cp15_ats1cpw_check(vm_offset_t addr) -{ - - cp15_ats1cpw_set(addr); - isb(); - return (cp15_par_get() & 0x01 ? EFAULT : 0); -} - -static __inline int -cp15_ats1cur_check(vm_offset_t addr) -{ - - cp15_ats1cur_set(addr); - isb(); - return (cp15_par_get() & 0x01 ? EFAULT : 0); -} - -static __inline int -cp15_ats1cuw_check(vm_offset_t addr) -{ - - cp15_ats1cuw_set(addr); - isb(); - return (cp15_par_get() & 0x01 ? EFAULT : 0); -} - -#endif /* !MACHINE_CPU_V6_H */ diff --git a/sys/arm/include/cpu.h b/sys/arm/include/cpu.h index 86f16131f03f..b2c370d01cdc 100644 --- a/sys/arm/include/cpu.h +++ b/sys/arm/include/cpu.h @@ -1,89 +1,760 @@ +/*- + * Copyright 2014 Svatopluk Kraus + * Copyright 2014 Michal Meloun + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ /* $NetBSD: cpu.h,v 1.2 2001/02/23 21:23:52 reinoud Exp $ */ -/* $FreeBSD$ */ #ifndef MACHINE_CPU_H #define MACHINE_CPU_H #include #include void cpu_halt(void); #ifdef _KERNEL -#include +#include +#include +#include +#include + +/* + * Some kernel modules (dtrace all for example) are compiled + * unconditionally with -DSMP. Although it looks like a bug, + * handle this case here and in #elif condition in ARM_SMP_UP macro. + */ +#if __ARM_ARCH <= 6 && defined(SMP) && !defined(KLD_MODULE) +#error SMP option is not supported on ARMv6 +#endif + +#if __ARM_ARCH <= 6 && defined(SMP_ON_UP) +#error SMP_ON_UP option is only supported on ARMv7+ CPUs +#endif + +#if !defined(SMP) && defined(SMP_ON_UP) +#error SMP option must be defined for SMP_ON_UP option +#endif + +#define CPU_ASID_KERNEL 0 + +#if defined(SMP_ON_UP) +#define ARM_SMP_UP(smp_code, up_code) \ +do { \ + if (cpuinfo.mp_ext != 0) { \ + smp_code; \ + } else { \ + up_code; \ + } \ +} while (0) +#elif defined(SMP) && __ARM_ARCH > 6 +#define ARM_SMP_UP(smp_code, up_code) \ +do { \ + smp_code; \ +} while (0) +#else +#define ARM_SMP_UP(smp_code, up_code) \ +do { \ + up_code; \ +} while (0) +#endif + +void dcache_wbinv_poc_all(void); /* !!! NOT SMP coherent function !!! */ +vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t); +vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t); + +#ifdef DEV_PMU +#include +#define PMU_OVSR_C 0x80000000 /* Cycle Counter */ +extern uint32_t ccnt_hi[MAXCPU]; +extern int pmu_attched; +#endif /* DEV_PMU */ + +#define sev() __asm __volatile("sev" : : : "memory") +#define wfe() __asm __volatile("wfe" : : : "memory") + +/* + * Macros to generate CP15 (system control processor) read/write functions. + */ +#define _FX(s...) #s + +#define _RF0(fname, aname...) \ +static __inline uint32_t \ +fname(void) \ +{ \ + uint32_t reg; \ + __asm __volatile("mrc\t" _FX(aname): "=r" (reg)); \ + return(reg); \ +} + +#define _R64F0(fname, aname) \ +static __inline uint64_t \ +fname(void) \ +{ \ + uint64_t reg; \ + __asm __volatile("mrrc\t" _FX(aname): "=r" (reg)); \ + return(reg); \ +} + +#define _WF0(fname, aname...) \ +static __inline void \ +fname(void) \ +{ \ + __asm __volatile("mcr\t" _FX(aname)); \ +} + +#define _WF1(fname, aname...) \ +static __inline void \ +fname(uint32_t reg) \ +{ \ + __asm __volatile("mcr\t" _FX(aname):: "r" (reg)); \ +} + +#define _W64F1(fname, aname...) \ +static __inline void \ +fname(uint64_t reg) \ +{ \ + __asm __volatile("mcrr\t" _FX(aname):: "r" (reg)); \ +} + +/* + * Raw CP15 maintenance operations + * !!! not for external use !!! + */ + +/* TLB */ + +_WF0(_CP15_TLBIALL, CP15_TLBIALL) /* Invalidate entire unified TLB */ +#if __ARM_ARCH >= 7 && defined(SMP) +_WF0(_CP15_TLBIALLIS, CP15_TLBIALLIS) /* Invalidate entire unified TLB IS */ +#endif +_WF1(_CP15_TLBIASID, CP15_TLBIASID(%0)) /* Invalidate unified TLB by ASID */ +#if __ARM_ARCH >= 7 && defined(SMP) +_WF1(_CP15_TLBIASIDIS, CP15_TLBIASIDIS(%0)) /* Invalidate unified TLB by ASID IS */ +#endif +_WF1(_CP15_TLBIMVAA, CP15_TLBIMVAA(%0)) /* Invalidate unified TLB by MVA, all ASID */ +#if __ARM_ARCH >= 7 && defined(SMP) +_WF1(_CP15_TLBIMVAAIS, CP15_TLBIMVAAIS(%0)) /* Invalidate unified TLB by MVA, all ASID IS */ +#endif +_WF1(_CP15_TLBIMVA, CP15_TLBIMVA(%0)) /* Invalidate unified TLB by MVA */ + +_WF1(_CP15_TTB_SET, CP15_TTBR0(%0)) + +/* Cache and Branch predictor */ + +_WF0(_CP15_BPIALL, CP15_BPIALL) /* Branch predictor invalidate all */ +#if __ARM_ARCH >= 7 && defined(SMP) +_WF0(_CP15_BPIALLIS, CP15_BPIALLIS) /* Branch predictor invalidate all IS */ +#endif +_WF1(_CP15_BPIMVA, CP15_BPIMVA(%0)) /* Branch predictor invalidate by MVA */ +_WF1(_CP15_DCCIMVAC, CP15_DCCIMVAC(%0)) /* Data cache clean and invalidate by MVA PoC */ +_WF1(_CP15_DCCISW, CP15_DCCISW(%0)) /* Data cache clean and invalidate by set/way */ +_WF1(_CP15_DCCMVAC, CP15_DCCMVAC(%0)) /* Data cache clean by MVA PoC */ +#if __ARM_ARCH >= 7 +_WF1(_CP15_DCCMVAU, CP15_DCCMVAU(%0)) /* Data cache clean by MVA PoU */ +#endif +_WF1(_CP15_DCCSW, CP15_DCCSW(%0)) /* Data cache clean by set/way */ +_WF1(_CP15_DCIMVAC, CP15_DCIMVAC(%0)) /* Data cache invalidate by MVA PoC */ +_WF1(_CP15_DCISW, CP15_DCISW(%0)) /* Data cache invalidate by set/way */ +_WF0(_CP15_ICIALLU, CP15_ICIALLU) /* Instruction cache invalidate all PoU */ +#if __ARM_ARCH >= 7 && defined(SMP) +_WF0(_CP15_ICIALLUIS, CP15_ICIALLUIS) /* Instruction cache invalidate all PoU IS */ +#endif +_WF1(_CP15_ICIMVAU, CP15_ICIMVAU(%0)) /* Instruction cache invalidate */ + +/* + * Publicly accessible functions + */ + +/* CP14 Debug Registers */ +_RF0(cp14_dbgdidr_get, CP14_DBGDIDR(%0)) +_RF0(cp14_dbgprsr_get, CP14_DBGPRSR(%0)) +_RF0(cp14_dbgoslsr_get, CP14_DBGOSLSR(%0)) +_RF0(cp14_dbgosdlr_get, CP14_DBGOSDLR(%0)) +_RF0(cp14_dbgdscrint_get, CP14_DBGDSCRint(%0)) + +_WF1(cp14_dbgdscr_v6_set, CP14_DBGDSCRext_V6(%0)) +_WF1(cp14_dbgdscr_v7_set, CP14_DBGDSCRext_V7(%0)) +_WF1(cp14_dbgvcr_set, CP14_DBGVCR(%0)) +_WF1(cp14_dbgoslar_set, CP14_DBGOSLAR(%0)) + +/* Various control registers */ + +_RF0(cp15_cpacr_get, CP15_CPACR(%0)) +_WF1(cp15_cpacr_set, CP15_CPACR(%0)) +_RF0(cp15_dfsr_get, CP15_DFSR(%0)) +_RF0(cp15_ifsr_get, CP15_IFSR(%0)) +_WF1(cp15_prrr_set, CP15_PRRR(%0)) +_WF1(cp15_nmrr_set, CP15_NMRR(%0)) +_RF0(cp15_ttbr_get, CP15_TTBR0(%0)) +_RF0(cp15_dfar_get, CP15_DFAR(%0)) +#if __ARM_ARCH >= 7 +_RF0(cp15_ifar_get, CP15_IFAR(%0)) +_RF0(cp15_l2ctlr_get, CP15_L2CTLR(%0)) +#endif +_RF0(cp15_actlr_get, CP15_ACTLR(%0)) +_WF1(cp15_actlr_set, CP15_ACTLR(%0)) +_WF1(cp15_ats1cpr_set, CP15_ATS1CPR(%0)) +_WF1(cp15_ats1cpw_set, CP15_ATS1CPW(%0)) +_WF1(cp15_ats1cur_set, CP15_ATS1CUR(%0)) +_WF1(cp15_ats1cuw_set, CP15_ATS1CUW(%0)) +_RF0(cp15_par_get, CP15_PAR(%0)) +_RF0(cp15_sctlr_get, CP15_SCTLR(%0)) + +/*CPU id registers */ +_RF0(cp15_midr_get, CP15_MIDR(%0)) +_RF0(cp15_ctr_get, CP15_CTR(%0)) +_RF0(cp15_tcmtr_get, CP15_TCMTR(%0)) +_RF0(cp15_tlbtr_get, CP15_TLBTR(%0)) +_RF0(cp15_mpidr_get, CP15_MPIDR(%0)) +_RF0(cp15_revidr_get, CP15_REVIDR(%0)) +_RF0(cp15_ccsidr_get, CP15_CCSIDR(%0)) +_RF0(cp15_clidr_get, CP15_CLIDR(%0)) +_RF0(cp15_aidr_get, CP15_AIDR(%0)) +_WF1(cp15_csselr_set, CP15_CSSELR(%0)) +_RF0(cp15_id_pfr0_get, CP15_ID_PFR0(%0)) +_RF0(cp15_id_pfr1_get, CP15_ID_PFR1(%0)) +_RF0(cp15_id_dfr0_get, CP15_ID_DFR0(%0)) +_RF0(cp15_id_afr0_get, CP15_ID_AFR0(%0)) +_RF0(cp15_id_mmfr0_get, CP15_ID_MMFR0(%0)) +_RF0(cp15_id_mmfr1_get, CP15_ID_MMFR1(%0)) +_RF0(cp15_id_mmfr2_get, CP15_ID_MMFR2(%0)) +_RF0(cp15_id_mmfr3_get, CP15_ID_MMFR3(%0)) +_RF0(cp15_id_isar0_get, CP15_ID_ISAR0(%0)) +_RF0(cp15_id_isar1_get, CP15_ID_ISAR1(%0)) +_RF0(cp15_id_isar2_get, CP15_ID_ISAR2(%0)) +_RF0(cp15_id_isar3_get, CP15_ID_ISAR3(%0)) +_RF0(cp15_id_isar4_get, CP15_ID_ISAR4(%0)) +_RF0(cp15_id_isar5_get, CP15_ID_ISAR5(%0)) +_RF0(cp15_cbar_get, CP15_CBAR(%0)) + +/* Performance Monitor registers */ + +#if __ARM_ARCH == 6 && defined(CPU_ARM1176) +_RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) +_WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) +_RF0(cp15_pmcr_get, CP15_PMCR(%0)) +_WF1(cp15_pmcr_set, CP15_PMCR(%0)) +_RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) +_WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) +#elif __ARM_ARCH > 6 +_RF0(cp15_pmcr_get, CP15_PMCR(%0)) +_WF1(cp15_pmcr_set, CP15_PMCR(%0)) +_RF0(cp15_pmcnten_get, CP15_PMCNTENSET(%0)) +_WF1(cp15_pmcnten_set, CP15_PMCNTENSET(%0)) +_WF1(cp15_pmcnten_clr, CP15_PMCNTENCLR(%0)) +_RF0(cp15_pmovsr_get, CP15_PMOVSR(%0)) +_WF1(cp15_pmovsr_set, CP15_PMOVSR(%0)) +_WF1(cp15_pmswinc_set, CP15_PMSWINC(%0)) +_RF0(cp15_pmselr_get, CP15_PMSELR(%0)) +_WF1(cp15_pmselr_set, CP15_PMSELR(%0)) +_RF0(cp15_pmccntr_get, CP15_PMCCNTR(%0)) +_WF1(cp15_pmccntr_set, CP15_PMCCNTR(%0)) +_RF0(cp15_pmxevtyper_get, CP15_PMXEVTYPER(%0)) +_WF1(cp15_pmxevtyper_set, CP15_PMXEVTYPER(%0)) +_RF0(cp15_pmxevcntr_get, CP15_PMXEVCNTRR(%0)) +_WF1(cp15_pmxevcntr_set, CP15_PMXEVCNTRR(%0)) +_RF0(cp15_pmuserenr_get, CP15_PMUSERENR(%0)) +_WF1(cp15_pmuserenr_set, CP15_PMUSERENR(%0)) +_RF0(cp15_pminten_get, CP15_PMINTENSET(%0)) +_WF1(cp15_pminten_set, CP15_PMINTENSET(%0)) +_WF1(cp15_pminten_clr, CP15_PMINTENCLR(%0)) +#endif + +_RF0(cp15_tpidrurw_get, CP15_TPIDRURW(%0)) +_WF1(cp15_tpidrurw_set, CP15_TPIDRURW(%0)) +_RF0(cp15_tpidruro_get, CP15_TPIDRURO(%0)) +_WF1(cp15_tpidruro_set, CP15_TPIDRURO(%0)) +_RF0(cp15_tpidrpwr_get, CP15_TPIDRPRW(%0)) +_WF1(cp15_tpidrpwr_set, CP15_TPIDRPRW(%0)) + +/* Generic Timer registers - only use when you know the hardware is available */ +_RF0(cp15_cntfrq_get, CP15_CNTFRQ(%0)) +_WF1(cp15_cntfrq_set, CP15_CNTFRQ(%0)) +_RF0(cp15_cntkctl_get, CP15_CNTKCTL(%0)) +_WF1(cp15_cntkctl_set, CP15_CNTKCTL(%0)) +_RF0(cp15_cntp_tval_get, CP15_CNTP_TVAL(%0)) +_WF1(cp15_cntp_tval_set, CP15_CNTP_TVAL(%0)) +_RF0(cp15_cntp_ctl_get, CP15_CNTP_CTL(%0)) +_WF1(cp15_cntp_ctl_set, CP15_CNTP_CTL(%0)) +_RF0(cp15_cntv_tval_get, CP15_CNTV_TVAL(%0)) +_WF1(cp15_cntv_tval_set, CP15_CNTV_TVAL(%0)) +_RF0(cp15_cntv_ctl_get, CP15_CNTV_CTL(%0)) +_WF1(cp15_cntv_ctl_set, CP15_CNTV_CTL(%0)) +_RF0(cp15_cnthctl_get, CP15_CNTHCTL(%0)) +_WF1(cp15_cnthctl_set, CP15_CNTHCTL(%0)) +_RF0(cp15_cnthp_tval_get, CP15_CNTHP_TVAL(%0)) +_WF1(cp15_cnthp_tval_set, CP15_CNTHP_TVAL(%0)) +_RF0(cp15_cnthp_ctl_get, CP15_CNTHP_CTL(%0)) +_WF1(cp15_cnthp_ctl_set, CP15_CNTHP_CTL(%0)) + +_R64F0(cp15_cntpct_get, CP15_CNTPCT(%Q0, %R0)) +_R64F0(cp15_cntvct_get, CP15_CNTVCT(%Q0, %R0)) +_R64F0(cp15_cntp_cval_get, CP15_CNTP_CVAL(%Q0, %R0)) +_W64F1(cp15_cntp_cval_set, CP15_CNTP_CVAL(%Q0, %R0)) +_R64F0(cp15_cntv_cval_get, CP15_CNTV_CVAL(%Q0, %R0)) +_W64F1(cp15_cntv_cval_set, CP15_CNTV_CVAL(%Q0, %R0)) +_R64F0(cp15_cntvoff_get, CP15_CNTVOFF(%Q0, %R0)) +_W64F1(cp15_cntvoff_set, CP15_CNTVOFF(%Q0, %R0)) +_R64F0(cp15_cnthp_cval_get, CP15_CNTHP_CVAL(%Q0, %R0)) +_W64F1(cp15_cnthp_cval_set, CP15_CNTHP_CVAL(%Q0, %R0)) + +#undef _FX +#undef _RF0 +#undef _WF0 +#undef _WF1 + +/* + * TLB maintenance operations. + */ + +/* Local (i.e. not broadcasting ) operations. */ + +/* Flush all TLB entries (even global). */ +static __inline void +tlb_flush_all_local(void) +{ + + dsb(); + _CP15_TLBIALL(); + dsb(); +} + +/* Flush all not global TLB entries. */ +static __inline void +tlb_flush_all_ng_local(void) +{ + + dsb(); + _CP15_TLBIASID(CPU_ASID_KERNEL); + dsb(); +} + +/* Flush single TLB entry (even global). */ +static __inline void +tlb_flush_local(vm_offset_t va) +{ + + KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); + + dsb(); + _CP15_TLBIMVA(va | CPU_ASID_KERNEL); + dsb(); +} + +/* Flush range of TLB entries (even global). */ +static __inline void +tlb_flush_range_local(vm_offset_t va, vm_size_t size) +{ + vm_offset_t eva = va + size; + + KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); + KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, + size)); + + dsb(); + for (; va < eva; va += PAGE_SIZE) + _CP15_TLBIMVA(va | CPU_ASID_KERNEL); + dsb(); +} + +/* Broadcasting operations. */ +#if __ARM_ARCH >= 7 && defined(SMP) + +static __inline void +tlb_flush_all(void) +{ + + dsb(); + ARM_SMP_UP( + _CP15_TLBIALLIS(), + _CP15_TLBIALL() + ); + dsb(); +} + +static __inline void +tlb_flush_all_ng(void) +{ + + dsb(); + ARM_SMP_UP( + _CP15_TLBIASIDIS(CPU_ASID_KERNEL), + _CP15_TLBIASID(CPU_ASID_KERNEL) + ); + dsb(); +} + +static __inline void +tlb_flush(vm_offset_t va) +{ + + KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); + + dsb(); + ARM_SMP_UP( + _CP15_TLBIMVAAIS(va), + _CP15_TLBIMVA(va | CPU_ASID_KERNEL) + ); + dsb(); +} + +static __inline void +tlb_flush_range(vm_offset_t va, vm_size_t size) +{ + vm_offset_t eva = va + size; + + KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); + KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, + size)); + + dsb(); + ARM_SMP_UP( + { + for (; va < eva; va += PAGE_SIZE) + _CP15_TLBIMVAAIS(va); + }, + { + for (; va < eva; va += PAGE_SIZE) + _CP15_TLBIMVA(va | CPU_ASID_KERNEL); + } + ); + dsb(); +} +#else /* __ARM_ARCH < 7 */ + +#define tlb_flush_all() tlb_flush_all_local() +#define tlb_flush_all_ng() tlb_flush_all_ng_local() +#define tlb_flush(va) tlb_flush_local(va) +#define tlb_flush_range(va, size) tlb_flush_range_local(va, size) + +#endif /* __ARM_ARCH < 7 */ + +/* + * Cache maintenance operations. + */ + +/* Sync I and D caches to PoU */ +static __inline void +icache_sync(vm_offset_t va, vm_size_t size) +{ + vm_offset_t eva = va + size; + + dsb(); + va &= ~cpuinfo.dcache_line_mask; + + for ( ; va < eva; va += cpuinfo.dcache_line_size) { +#if __ARM_ARCH >= 7 + _CP15_DCCMVAU(va); +#else + _CP15_DCCMVAC(va); +#endif + } + dsb(); + ARM_SMP_UP( + _CP15_ICIALLUIS(), + _CP15_ICIALLU() + ); + dsb(); + isb(); +} + +/* Invalidate I cache */ +static __inline void +icache_inv_all(void) +{ + + ARM_SMP_UP( + _CP15_ICIALLUIS(), + _CP15_ICIALLU() + ); + dsb(); + isb(); +} + +/* Invalidate branch predictor buffer */ +static __inline void +bpb_inv_all(void) +{ + + ARM_SMP_UP( + _CP15_BPIALLIS(), + _CP15_BPIALL() + ); + dsb(); + isb(); +} + +/* Write back D-cache to PoU */ +static __inline void +dcache_wb_pou(vm_offset_t va, vm_size_t size) +{ + vm_offset_t eva = va + size; + + dsb(); + va &= ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { +#if __ARM_ARCH >= 7 + _CP15_DCCMVAU(va); +#else + _CP15_DCCMVAC(va); +#endif + } + dsb(); +} + +/* + * Invalidate D-cache to PoC + * + * Caches are invalidated from outermost to innermost as fresh cachelines + * flow in this direction. In given range, if there was no dirty cacheline + * in any cache before, no stale cacheline should remain in them after this + * operation finishes. + */ +static __inline void +dcache_inv_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) +{ + vm_offset_t eva = va + size; + + dsb(); + /* invalidate L2 first */ + cpu_l2cache_inv_range(pa, size); + + /* then L1 */ + va &= ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { + _CP15_DCIMVAC(va); + } + dsb(); +} + +/* + * Discard D-cache lines to PoC, prior to overwrite by DMA engine. + * + * Normal invalidation does L2 then L1 to ensure that stale data from L2 doesn't + * flow into L1 while invalidating. This routine is intended to be used only + * when invalidating a buffer before a DMA operation loads new data into memory. + * The concern in this case is that dirty lines are not evicted to main memory, + * overwriting the DMA data. For that reason, the L1 is done first to ensure + * that an evicted L1 line doesn't flow to L2 after the L2 has been cleaned. + */ +static __inline void +dcache_inv_poc_dma(vm_offset_t va, vm_paddr_t pa, vm_size_t size) +{ + vm_offset_t eva = va + size; + + /* invalidate L1 first */ + dsb(); + va &= ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { + _CP15_DCIMVAC(va); + } + dsb(); + + /* then L2 */ + cpu_l2cache_inv_range(pa, size); +} + +/* + * Write back D-cache to PoC + * + * Caches are written back from innermost to outermost as dirty cachelines + * flow in this direction. In given range, no dirty cacheline should remain + * in any cache after this operation finishes. + */ +static __inline void +dcache_wb_poc(vm_offset_t va, vm_paddr_t pa, vm_size_t size) +{ + vm_offset_t eva = va + size; + + dsb(); + va &= ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { + _CP15_DCCMVAC(va); + } + dsb(); + + cpu_l2cache_wb_range(pa, size); +} + +/* Write back and invalidate D-cache to PoC */ +static __inline void +dcache_wbinv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size) +{ + vm_offset_t va; + vm_offset_t eva = sva + size; + + dsb(); + /* write back L1 first */ + va = sva & ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { + _CP15_DCCMVAC(va); + } + dsb(); + + /* then write back and invalidate L2 */ + cpu_l2cache_wbinv_range(pa, size); + + /* then invalidate L1 */ + va = sva & ~cpuinfo.dcache_line_mask; + for ( ; va < eva; va += cpuinfo.dcache_line_size) { + _CP15_DCIMVAC(va); + } + dsb(); +} + +/* Set TTB0 register */ +static __inline void +cp15_ttbr_set(uint32_t reg) +{ + dsb(); + _CP15_TTB_SET(reg); + dsb(); + _CP15_BPIALL(); + dsb(); + isb(); + tlb_flush_all_ng_local(); +} + +/* + * Functions for address checking: + * + * cp15_ats1cpr_check() ... check stage 1 privileged (PL1) read access + * cp15_ats1cpw_check() ... check stage 1 privileged (PL1) write access + * cp15_ats1cur_check() ... check stage 1 unprivileged (PL0) read access + * cp15_ats1cuw_check() ... check stage 1 unprivileged (PL0) write access + * + * They must be called while interrupts are disabled to get consistent result. + */ +static __inline int +cp15_ats1cpr_check(vm_offset_t addr) +{ + + cp15_ats1cpr_set(addr); + isb(); + return (cp15_par_get() & 0x01 ? EFAULT : 0); +} + +static __inline int +cp15_ats1cpw_check(vm_offset_t addr) +{ + + cp15_ats1cpw_set(addr); + isb(); + return (cp15_par_get() & 0x01 ? EFAULT : 0); +} + +static __inline int +cp15_ats1cur_check(vm_offset_t addr) +{ + + cp15_ats1cur_set(addr); + isb(); + return (cp15_par_get() & 0x01 ? EFAULT : 0); +} + +static __inline int +cp15_ats1cuw_check(vm_offset_t addr) +{ + + cp15_ats1cuw_set(addr); + isb(); + return (cp15_par_get() & 0x01 ? EFAULT : 0); +} static __inline uint64_t get_cyclecount(void) { #if __ARM_ARCH > 6 || (__ARM_ARCH == 6 && defined(CPU_ARM1176)) #if (__ARM_ARCH > 6) && defined(DEV_PMU) if (pmu_attched) { u_int cpu; uint64_t h, h2; uint32_t l, r; cpu = PCPU_GET(cpuid); h = (uint64_t)atomic_load_acq_32(&ccnt_hi[cpu]); l = cp15_pmccntr_get(); /* In case interrupts are disabled we need to check for overflow. */ r = cp15_pmovsr_get(); if (r & PMU_OVSR_C) { atomic_add_32(&ccnt_hi[cpu], 1); /* Clear the event. */ cp15_pmovsr_set(PMU_OVSR_C); } /* Make sure there was no wrap-around while we read the lo half. */ h2 = (uint64_t)atomic_load_acq_32(&ccnt_hi[cpu]); if (h != h2) l = cp15_pmccntr_get(); return (h2 << 32 | l); } else #endif return cp15_pmccntr_get(); #else /* No performance counters, so use nanotime(9). */ struct timespec tv; nanotime(&tv); return (tv.tv_sec * (uint64_t)1000000000ull + tv.tv_nsec); #endif } #endif #define TRAPF_USERMODE(frame) ((frame->tf_spsr & PSR_MODE) == PSR_USR32_MODE) #define TRAPF_PC(tfp) ((tfp)->tf_pc) #define cpu_getstack(td) ((td)->td_frame->tf_usr_sp) #define cpu_setstack(td, sp) ((td)->td_frame->tf_usr_sp = (sp)) #define cpu_spinwait() /* nothing */ #define cpu_lock_delay() DELAY(1) #define ARM_NVEC 8 #define ARM_VEC_ALL 0xffffffff extern vm_offset_t vector_page; /* * Params passed into initarm. If you change the size of this you will * need to update locore.S to allocate more memory on the stack before * it calls initarm. */ struct arm_boot_params { register_t abp_size; /* Size of this structure */ register_t abp_r0; /* r0 from the boot loader */ register_t abp_r1; /* r1 from the boot loader */ register_t abp_r2; /* r2 from the boot loader */ register_t abp_r3; /* r3 from the boot loader */ vm_offset_t abp_physaddr; /* The kernel physical address */ vm_offset_t abp_pagetable; /* The early page table */ }; void arm_vector_init(vm_offset_t, int); void fork_trampoline(void); void identify_arm_cpu(void); void *initarm(struct arm_boot_params *); extern char btext[]; extern char etext[]; int badaddr_read(void *, size_t, void *); #endif /* !MACHINE_CPU_H */ diff --git a/sys/arm/include/pmap_var.h b/sys/arm/include/pmap_var.h index 34c101df5c4d..08d32575f82a 100644 --- a/sys/arm/include/pmap_var.h +++ b/sys/arm/include/pmap_var.h @@ -1,494 +1,494 @@ /*- * Copyright 2014 Svatopluk Kraus * Copyright 2014 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMAP_VAR_H_ #define _MACHINE_PMAP_VAR_H_ -#include #include + /* * Various PMAP defines, exports, and inline functions * definitions also usable in other MD code. */ /* A number of pages in L1 page table. */ #define NPG_IN_PT1 (NB_IN_PT1 / PAGE_SIZE) /* A number of L2 page tables in a page. */ #define NPT2_IN_PG (PAGE_SIZE / NB_IN_PT2) /* A number of L2 page table entries in a page. */ #define NPTE2_IN_PG (NPT2_IN_PG * NPTE2_IN_PT2) #ifdef _KERNEL /* * A L2 page tables page contains NPT2_IN_PG L2 page tables. Masking of * pte1_idx by PT2PG_MASK gives us an index to associated L2 page table * in a page. The PT2PG_SHIFT definition depends on NPT2_IN_PG strictly. * I.e., (1 << PT2PG_SHIFT) == NPT2_IN_PG must be fulfilled. */ #define PT2PG_SHIFT 2 #define PT2PG_MASK ((1 << PT2PG_SHIFT) - 1) /* * A PT2TAB holds all allocated L2 page table pages in a pmap. * Right shifting of virtual address by PT2TAB_SHIFT gives us an index * to L2 page table page in PT2TAB which holds the address mapping. */ #define PT2TAB_ENTRIES (NPTE1_IN_PT1 / NPT2_IN_PG) #define PT2TAB_SHIFT (PTE1_SHIFT + PT2PG_SHIFT) /* * All allocated L2 page table pages in a pmap are mapped into PT2MAP space. * An virtual address right shifting by PT2MAP_SHIFT gives us an index to PTE2 * which maps the address. */ #define PT2MAP_SIZE (NPTE1_IN_PT1 * NB_IN_PT2) #define PT2MAP_SHIFT PTE2_SHIFT extern pt1_entry_t *kern_pt1; extern pt2_entry_t *kern_pt2tab; extern pt2_entry_t *PT2MAP; /* * Virtual interface for L1 page table management. */ static __inline u_int pte1_index(vm_offset_t va) { return (va >> PTE1_SHIFT); } static __inline pt1_entry_t * pte1_ptr(pt1_entry_t *pt1, vm_offset_t va) { return (pt1 + pte1_index(va)); } static __inline vm_offset_t pte1_trunc(vm_offset_t va) { return (va & PTE1_FRAME); } static __inline vm_offset_t pte1_roundup(vm_offset_t va) { return ((va + PTE1_OFFSET) & PTE1_FRAME); } /* * Virtual interface for L1 page table entries management. * * XXX: Some of the following functions now with a synchronization barrier * are called in a loop, so it could be useful to have two versions of them. * One with the barrier and one without the barrier. In this case, pure * barrier pte1_sync() should be implemented as well. */ static __inline void pte1_sync(pt1_entry_t *pte1p) { dsb(); #ifndef PMAP_PTE_NOCACHE if (!cpuinfo.coherent_walk) dcache_wb_pou((vm_offset_t)pte1p, sizeof(*pte1p)); #endif } static __inline void pte1_sync_range(pt1_entry_t *pte1p, vm_size_t size) { dsb(); #ifndef PMAP_PTE_NOCACHE if (!cpuinfo.coherent_walk) dcache_wb_pou((vm_offset_t)pte1p, size); #endif } static __inline void pte1_store(pt1_entry_t *pte1p, pt1_entry_t pte1) { dmb(); *pte1p = pte1; pte1_sync(pte1p); } static __inline void pte1_clear(pt1_entry_t *pte1p) { pte1_store(pte1p, 0); } static __inline void pte1_clear_bit(pt1_entry_t *pte1p, uint32_t bit) { *pte1p &= ~bit; pte1_sync(pte1p); } static __inline boolean_t pte1_is_link(pt1_entry_t pte1) { return ((pte1 & L1_TYPE_MASK) == L1_TYPE_C); } static __inline int pte1_is_section(pt1_entry_t pte1) { return ((pte1 & L1_TYPE_MASK) == L1_TYPE_S); } static __inline boolean_t pte1_is_dirty(pt1_entry_t pte1) { return ((pte1 & (PTE1_NM | PTE1_RO)) == 0); } static __inline boolean_t pte1_is_global(pt1_entry_t pte1) { return ((pte1 & PTE1_NG) == 0); } static __inline boolean_t pte1_is_valid(pt1_entry_t pte1) { int l1_type; l1_type = pte1 & L1_TYPE_MASK; return ((l1_type == L1_TYPE_C) || (l1_type == L1_TYPE_S)); } static __inline boolean_t pte1_is_wired(pt1_entry_t pte1) { return (pte1 & PTE1_W); } static __inline pt1_entry_t pte1_load(pt1_entry_t *pte1p) { pt1_entry_t pte1; pte1 = *pte1p; return (pte1); } static __inline pt1_entry_t pte1_load_clear(pt1_entry_t *pte1p) { pt1_entry_t opte1; opte1 = *pte1p; *pte1p = 0; pte1_sync(pte1p); return (opte1); } static __inline void pte1_set_bit(pt1_entry_t *pte1p, uint32_t bit) { *pte1p |= bit; pte1_sync(pte1p); } static __inline vm_paddr_t pte1_pa(pt1_entry_t pte1) { return ((vm_paddr_t)(pte1 & PTE1_FRAME)); } static __inline vm_paddr_t pte1_link_pa(pt1_entry_t pte1) { return ((vm_paddr_t)(pte1 & L1_C_ADDR_MASK)); } /* * Virtual interface for L2 page table entries management. * * XXX: Some of the following functions now with a synchronization barrier * are called in a loop, so it could be useful to have two versions of them. * One with the barrier and one without the barrier. */ static __inline void pte2_sync(pt2_entry_t *pte2p) { dsb(); #ifndef PMAP_PTE_NOCACHE if (!cpuinfo.coherent_walk) dcache_wb_pou((vm_offset_t)pte2p, sizeof(*pte2p)); #endif } static __inline void pte2_sync_range(pt2_entry_t *pte2p, vm_size_t size) { dsb(); #ifndef PMAP_PTE_NOCACHE if (!cpuinfo.coherent_walk) dcache_wb_pou((vm_offset_t)pte2p, size); #endif } static __inline void pte2_store(pt2_entry_t *pte2p, pt2_entry_t pte2) { dmb(); *pte2p = pte2; pte2_sync(pte2p); } static __inline void pte2_clear(pt2_entry_t *pte2p) { pte2_store(pte2p, 0); } static __inline void pte2_clear_bit(pt2_entry_t *pte2p, uint32_t bit) { *pte2p &= ~bit; pte2_sync(pte2p); } static __inline boolean_t pte2_is_dirty(pt2_entry_t pte2) { return ((pte2 & (PTE2_NM | PTE2_RO)) == 0); } static __inline boolean_t pte2_is_global(pt2_entry_t pte2) { return ((pte2 & PTE2_NG) == 0); } static __inline boolean_t pte2_is_valid(pt2_entry_t pte2) { return (pte2 & PTE2_V); } static __inline boolean_t pte2_is_wired(pt2_entry_t pte2) { return (pte2 & PTE2_W); } static __inline pt2_entry_t pte2_load(pt2_entry_t *pte2p) { pt2_entry_t pte2; pte2 = *pte2p; return (pte2); } static __inline pt2_entry_t pte2_load_clear(pt2_entry_t *pte2p) { pt2_entry_t opte2; opte2 = *pte2p; *pte2p = 0; pte2_sync(pte2p); return (opte2); } static __inline void pte2_set_bit(pt2_entry_t *pte2p, uint32_t bit) { *pte2p |= bit; pte2_sync(pte2p); } static __inline void pte2_set_wired(pt2_entry_t *pte2p, boolean_t wired) { /* * Wired bit is transparent for page table walk, * so pte2_sync() is not needed. */ if (wired) *pte2p |= PTE2_W; else *pte2p &= ~PTE2_W; } static __inline vm_paddr_t pte2_pa(pt2_entry_t pte2) { return ((vm_paddr_t)(pte2 & PTE2_FRAME)); } static __inline u_int pte2_attr(pt2_entry_t pte2) { return ((u_int)(pte2 & PTE2_ATTR_MASK)); } /* * Virtual interface for L2 page tables mapping management. */ static __inline u_int pt2tab_index(vm_offset_t va) { return (va >> PT2TAB_SHIFT); } static __inline pt2_entry_t * pt2tab_entry(pt2_entry_t *pt2tab, vm_offset_t va) { return (pt2tab + pt2tab_index(va)); } static __inline void pt2tab_store(pt2_entry_t *pte2p, pt2_entry_t pte2) { pte2_store(pte2p,pte2); } static __inline pt2_entry_t pt2tab_load(pt2_entry_t *pte2p) { return (pte2_load(pte2p)); } static __inline pt2_entry_t pt2tab_load_clear(pt2_entry_t *pte2p) { return (pte2_load_clear(pte2p)); } static __inline u_int pt2map_index(vm_offset_t va) { return (va >> PT2MAP_SHIFT); } static __inline pt2_entry_t * pt2map_entry(vm_offset_t va) { return (PT2MAP + pt2map_index(va)); } /* * Virtual interface for pmap structure & kernel shortcuts. */ static __inline pt1_entry_t * pmap_pte1(pmap_t pmap, vm_offset_t va) { return (pte1_ptr(pmap->pm_pt1, va)); } static __inline pt1_entry_t * kern_pte1(vm_offset_t va) { return (pte1_ptr(kern_pt1, va)); } static __inline pt2_entry_t * pmap_pt2tab_entry(pmap_t pmap, vm_offset_t va) { return (pt2tab_entry(pmap->pm_pt2tab, va)); } static __inline pt2_entry_t * kern_pt2tab_entry(vm_offset_t va) { return (pt2tab_entry(kern_pt2tab, va)); } static __inline vm_page_t pmap_pt2_page(pmap_t pmap, vm_offset_t va) { pt2_entry_t pte2; pte2 = pte2_load(pmap_pt2tab_entry(pmap, va)); return (PHYS_TO_VM_PAGE(pte2 & PTE2_FRAME)); } static __inline vm_page_t kern_pt2_page(vm_offset_t va) { pt2_entry_t pte2; pte2 = pte2_load(kern_pt2tab_entry(va)); return (PHYS_TO_VM_PAGE(pte2 & PTE2_FRAME)); } #endif /* _KERNEL */ #endif /* !_MACHINE_PMAP_VAR_H_ */