Index: head/sys/arm64/arm64/debug_monitor.c =================================================================== --- head/sys/arm64/arm64/debug_monitor.c (revision 354176) +++ head/sys/arm64/arm64/debug_monitor.c (revision 354177) @@ -1,499 +1,499 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #ifdef DDB #include #include #endif enum dbg_t { DBG_TYPE_BREAKPOINT = 0, DBG_TYPE_WATCHPOINT = 1, }; static int dbg_watchpoint_num; static int dbg_breakpoint_num; static struct debug_monitor_state kernel_monitor = { .dbg_flags = DBGMON_KERNEL }; /* Watchpoints/breakpoints control register bitfields */ #define DBG_WATCH_CTRL_LEN_1 (0x1 << 5) #define DBG_WATCH_CTRL_LEN_2 (0x3 << 5) #define DBG_WATCH_CTRL_LEN_4 (0xf << 5) #define DBG_WATCH_CTRL_LEN_8 (0xff << 5) #define DBG_WATCH_CTRL_LEN_MASK(x) ((x) & (0xff << 5)) #define DBG_WATCH_CTRL_EXEC (0x0 << 3) #define DBG_WATCH_CTRL_LOAD (0x1 << 3) #define DBG_WATCH_CTRL_STORE (0x2 << 3) #define DBG_WATCH_CTRL_ACCESS_MASK(x) ((x) & (0x3 << 3)) /* Common for breakpoint and watchpoint */ #define DBG_WB_CTRL_EL1 (0x1 << 1) #define DBG_WB_CTRL_EL0 (0x2 << 1) #define DBG_WB_CTRL_ELX_MASK(x) ((x) & (0x3 << 1)) #define DBG_WB_CTRL_E (0x1 << 0) #define DBG_REG_BASE_BVR 0 #define DBG_REG_BASE_BCR (DBG_REG_BASE_BVR + 16) #define DBG_REG_BASE_WVR (DBG_REG_BASE_BCR + 16) #define DBG_REG_BASE_WCR (DBG_REG_BASE_WVR + 16) /* Watchpoint/breakpoint helpers */ #define DBG_WB_WVR "wvr" #define DBG_WB_WCR "wcr" #define DBG_WB_BVR "bvr" #define DBG_WB_BCR "bcr" #define DBG_WB_READ(reg, num, val) do { \ __asm __volatile("mrs %0, dbg" reg #num "_el1" : "=r" (val)); \ } while (0) #define DBG_WB_WRITE(reg, num, val) do { \ __asm __volatile("msr dbg" reg #num "_el1, %0" :: "r" (val)); \ } while (0) #define READ_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_READ(reg, num, val); \ break #define WRITE_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_WRITE(reg, num, val); \ break #define SWITCH_CASES_READ_WB_REG(reg, offset, val) \ READ_WB_REG_CASE(reg, 0, offset, val); \ READ_WB_REG_CASE(reg, 1, offset, val); \ READ_WB_REG_CASE(reg, 2, offset, val); \ READ_WB_REG_CASE(reg, 3, offset, val); \ READ_WB_REG_CASE(reg, 4, offset, val); \ READ_WB_REG_CASE(reg, 5, offset, val); \ READ_WB_REG_CASE(reg, 6, offset, val); \ READ_WB_REG_CASE(reg, 7, offset, val); \ READ_WB_REG_CASE(reg, 8, offset, val); \ READ_WB_REG_CASE(reg, 9, offset, val); \ READ_WB_REG_CASE(reg, 10, offset, val); \ READ_WB_REG_CASE(reg, 11, offset, val); \ READ_WB_REG_CASE(reg, 12, offset, val); \ READ_WB_REG_CASE(reg, 13, offset, val); \ READ_WB_REG_CASE(reg, 14, offset, val); \ READ_WB_REG_CASE(reg, 15, offset, val) #define SWITCH_CASES_WRITE_WB_REG(reg, offset, val) \ WRITE_WB_REG_CASE(reg, 0, offset, val); \ WRITE_WB_REG_CASE(reg, 1, offset, val); \ WRITE_WB_REG_CASE(reg, 2, offset, val); \ WRITE_WB_REG_CASE(reg, 3, offset, val); \ WRITE_WB_REG_CASE(reg, 4, offset, val); \ WRITE_WB_REG_CASE(reg, 5, offset, val); \ WRITE_WB_REG_CASE(reg, 6, offset, val); \ WRITE_WB_REG_CASE(reg, 7, offset, val); \ WRITE_WB_REG_CASE(reg, 8, offset, val); \ WRITE_WB_REG_CASE(reg, 9, offset, val); \ WRITE_WB_REG_CASE(reg, 10, offset, val); \ WRITE_WB_REG_CASE(reg, 11, offset, val); \ WRITE_WB_REG_CASE(reg, 12, offset, val); \ WRITE_WB_REG_CASE(reg, 13, offset, val); \ WRITE_WB_REG_CASE(reg, 14, offset, val); \ WRITE_WB_REG_CASE(reg, 15, offset, val) #ifdef DDB static uint64_t dbg_wb_read_reg(int reg, int n) { uint64_t val = 0; switch (reg + n) { SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to read from wrong debug register %d\n", n); } return val; } #endif /* DDB */ static void dbg_wb_write_reg(int reg, int n, uint64_t val) { switch (reg + n) { SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to write to wrong debug register %d\n", n); return; } isb(); } #ifdef DDB void kdb_cpu_set_singlestep(void) { kdb_frame->tf_spsr |= DBG_SPSR_SS; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_SS | DBG_MDSCR_KDE); /* * Disable breakpoints and watchpoints, e.g. stepping * over watched instruction will trigger break exception instead of * single-step exception and locks CPU on that instruction for ever. */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_MDE); } } void kdb_cpu_clear_singlestep(void) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_SS | DBG_MDSCR_KDE)); /* Restore breakpoints and watchpoints */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_MDE); if ((kernel_monitor.dbg_flags & DBGMON_KERNEL) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_KDE); } } } static const char * dbg_watchtype_str(uint32_t type) { switch (type) { case DBG_WATCH_CTRL_EXEC: return ("execute"); case DBG_WATCH_CTRL_STORE: return ("write"); case DBG_WATCH_CTRL_LOAD: return ("read"); case DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE: return ("read/write"); default: return ("invalid"); } } static int dbg_watchtype_len(uint32_t len) { switch (len) { case DBG_WATCH_CTRL_LEN_1: return (1); case DBG_WATCH_CTRL_LEN_2: return (2); case DBG_WATCH_CTRL_LEN_4: return (4); case DBG_WATCH_CTRL_LEN_8: return (8); default: return (0); } } void dbg_show_watchpoint(void) { uint32_t wcr, len, type; uint64_t addr; int i; db_printf("\nhardware watchpoints:\n"); db_printf(" watch status type len address symbol\n"); db_printf(" ----- -------- ---------- --- ------------------ ------------------\n"); for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) { type = DBG_WATCH_CTRL_ACCESS_MASK(wcr); len = DBG_WATCH_CTRL_LEN_MASK(wcr); addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i); db_printf(" %-5d %-8s %10s %3d 0x%16lx ", i, "enabled", dbg_watchtype_str(type), dbg_watchtype_len(len), addr); db_printsym((db_addr_t)addr, DB_STGY_ANY); db_printf("\n"); } else { db_printf(" %-5d disabled\n", i); } } } #endif /* DDB */ static int dbg_find_free_slot(struct debug_monitor_state *monitor, enum dbg_t type) { uint64_t *reg; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if ((reg[i] & DBG_WB_CTRL_E) == 0) return (i); } return (-1); } static int dbg_find_slot(struct debug_monitor_state *monitor, enum dbg_t type, vm_offset_t addr) { uint64_t *reg_addr, *reg_ctrl; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg_addr = monitor->dbg_bvr; reg_ctrl = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg_addr = monitor->dbg_wvr; reg_ctrl = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if (reg_addr[i] == addr && (reg_ctrl[i] & DBG_WB_CTRL_E) != 0) return (i); } return (-1); } int dbg_setup_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size, enum dbg_access_t access) { uint64_t wcr_size, wcr_priv, wcr_access; u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_free_slot(monitor, DBG_TYPE_WATCHPOINT); if (i == -1) { printf("Can not find slot for watchpoint, max %d" " watchpoints supported\n", dbg_watchpoint_num); return (i); } switch(size) { case 1: wcr_size = DBG_WATCH_CTRL_LEN_1; break; case 2: wcr_size = DBG_WATCH_CTRL_LEN_2; break; case 4: wcr_size = DBG_WATCH_CTRL_LEN_4; break; case 8: wcr_size = DBG_WATCH_CTRL_LEN_8; break; default: printf("Unsupported address size for watchpoint\n"); return (-1); } if ((monitor->dbg_flags & DBGMON_KERNEL) == 0) wcr_priv = DBG_WB_CTRL_EL0; else wcr_priv = DBG_WB_CTRL_EL1; switch(access) { case HW_BREAKPOINT_X: wcr_access = DBG_WATCH_CTRL_EXEC; break; case HW_BREAKPOINT_R: wcr_access = DBG_WATCH_CTRL_LOAD; break; case HW_BREAKPOINT_W: wcr_access = DBG_WATCH_CTRL_STORE; break; case HW_BREAKPOINT_RW: wcr_access = DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE; break; default: printf("Unsupported exception level for watchpoint\n"); return (-1); } monitor->dbg_wvr[i] = addr; monitor->dbg_wcr[i] = wcr_size | wcr_access | wcr_priv | DBG_WB_CTRL_E; monitor->dbg_enable_count++; monitor->dbg_flags |= DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } int dbg_remove_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size) { u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_slot(monitor, DBG_TYPE_WATCHPOINT, addr); if (i == -1) { printf("Can not find watchpoint for address 0%lx\n", addr); return (i); } monitor->dbg_wvr[i] = 0; monitor->dbg_wcr[i] = 0; monitor->dbg_enable_count--; if (monitor->dbg_enable_count == 0) monitor->dbg_flags &= ~DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } void dbg_register_sync(struct debug_monitor_state *monitor) { uint64_t mdscr; int i; if (monitor == NULL) monitor = &kernel_monitor; mdscr = READ_SPECIALREG(mdscr_el1); if ((monitor->dbg_flags & DBGMON_ENABLED) == 0) { mdscr &= ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE); } else { for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, monitor->dbg_bcr[i]); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, monitor->dbg_bvr[i]); } for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, monitor->dbg_wcr[i]); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, monitor->dbg_wvr[i]); } mdscr |= DBG_MDSCR_MDE; if ((monitor->dbg_flags & DBGMON_KERNEL) == DBGMON_KERNEL) mdscr |= DBG_MDSCR_KDE; } WRITE_SPECIALREG(mdscr_el1, mdscr); isb(); } void dbg_monitor_init(void) { u_int i; /* Find out many breakpoints and watchpoints we can use */ - dbg_watchpoint_num = ((READ_SPECIALREG(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; - dbg_breakpoint_num = ((READ_SPECIALREG(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; + dbg_watchpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 20) & 0xf) + 1; + dbg_breakpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 12) & 0xf) + 1; if (bootverbose && PCPU_GET(cpuid) == 0) { printf("%d watchpoints and %d breakpoints supported\n", dbg_watchpoint_num, dbg_breakpoint_num); } /* * We have limited number of {watch,break}points, each consists of * two registers: * - wcr/bcr regsiter configurates corresponding {watch,break}point * behaviour * - wvr/bvr register keeps address we are hunting for * * Reset all breakpoints and watchpoints. */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } dbg_enable(); } Index: head/sys/arm64/arm64/freebsd32_machdep.c =================================================================== --- head/sys/arm64/arm64/freebsd32_machdep.c (revision 354176) +++ head/sys/arm64/arm64/freebsd32_machdep.c (revision 354177) @@ -1,440 +1,440 @@ /*- * Copyright (c) 2018 Olivier Houchard * Copyright (c) 2017 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #include #include extern void freebsd32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask); /* * The first two fields of a ucontext_t are the signal mask and the machine * context. The next field is uc_link; we want to avoid destroying the link * when copying out contexts. */ #define UC32_COPY_SIZE offsetof(ucontext32_t, uc_link) #ifdef VFP static void get_fpcontext32(struct thread *td, mcontext32_vfp_t *); #endif /* * Stubs for machine dependent 32-bits system calls. */ int freebsd32_sysarch(struct thread *td, struct freebsd32_sysarch_args *uap) { int error; #define ARM_SYNC_ICACHE 0 #define ARM_DRAIN_WRITEBUF 1 #define ARM_SET_TP 2 #define ARM_GET_TP 3 #define ARM_GET_VFPSTATE 4 switch(uap->op) { case ARM_SET_TP: - WRITE_SPECIALREG(TPIDR_EL0, uap->parms); - WRITE_SPECIALREG(TPIDRRO_EL0, uap->parms); + WRITE_SPECIALREG(tpidr_el0, uap->parms); + WRITE_SPECIALREG(tpidrro_el0, uap->parms); return 0; case ARM_SYNC_ICACHE: { struct { uint32_t addr; uint32_t size; } args; if ((error = copyin(uap->parms, &args, sizeof(args))) != 0) return (error); if ((uint64_t)args.addr + (uint64_t)args.size > 0xffffffff) return (EINVAL); cpu_icache_sync_range_checked(args.addr, args.size); return 0; } case ARM_GET_VFPSTATE: { mcontext32_vfp_t mcontext_vfp; struct { uint32_t mc_vfp_size; uint32_t mc_vfp; } args; if ((error = copyin(uap->parms, &args, sizeof(args))) != 0) return (error); if (args.mc_vfp_size != sizeof(mcontext_vfp)) return (EINVAL); #ifdef VFP get_fpcontext32(td, &mcontext_vfp); #else bzero(&mcontext_vfp, sizeof(mcontext_vfp)); #endif error = copyout(&mcontext_vfp, (void *)(uintptr_t)args.mc_vfp, sizeof(mcontext_vfp)); return error; } } return (EINVAL); } #ifdef VFP static void get_fpcontext32(struct thread *td, mcontext32_vfp_t *mcp) { struct pcb *curpcb; int i; critical_enter(); curpcb = curthread->td_pcb; if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ vfp_save_state(td, curpcb); KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, ("Called get_fpcontext while the kernel is using the VFP")); KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Non-userspace FPU flags set in get_fpcontext")); for (i = 0; i < 32; i++) mcp->mcv_reg[i] = (uint64_t)curpcb->pcb_fpustate.vfp_regs[i]; mcp->mcv_fpscr = VFP_FPSCR_FROM_SRCR(curpcb->pcb_fpustate.vfp_fpcr, curpcb->pcb_fpustate.vfp_fpsr); } critical_exit(); } static void set_fpcontext32(struct thread *td, mcontext32_vfp_t *mcp) { struct pcb *pcb; int i; critical_enter(); pcb = td->td_pcb; if (td == curthread) vfp_discard(td); for (i = 0; i < 32; i++) pcb->pcb_fpustate.vfp_regs[i] = mcp->mcv_reg[i]; pcb->pcb_fpustate.vfp_fpsr = VFP_FPSR_FROM_FPSCR(mcp->mcv_fpscr); pcb->pcb_fpustate.vfp_fpcr = VFP_FPSR_FROM_FPSCR(mcp->mcv_fpscr); critical_exit(); } #endif static void get_mcontext32(struct thread *td, mcontext32_t *mcp, int flags) { struct pcb *pcb; struct trapframe *tf; int i; pcb = td->td_pcb; tf = td->td_frame; if ((flags & GET_MC_CLEAR_RET) != 0) { mcp->mc_gregset[0] = 0; mcp->mc_gregset[16] = tf->tf_spsr & ~PSR_C; } else { mcp->mc_gregset[0] = tf->tf_x[0]; mcp->mc_gregset[16] = tf->tf_spsr; } for (i = 1; i < 15; i++) mcp->mc_gregset[i] = tf->tf_x[i]; mcp->mc_gregset[15] = tf->tf_elr; mcp->mc_vfp_size = 0; mcp->mc_vfp_ptr = 0; memset(mcp->mc_spare, 0, sizeof(mcp->mc_spare)); } static int set_mcontext32(struct thread *td, mcontext32_t *mcp) { struct trapframe *tf; mcontext32_vfp_t mc_vfp; int i; tf = td->td_frame; for (i = 0; i < 15; i++) tf->tf_x[i] = mcp->mc_gregset[i]; tf->tf_elr = mcp->mc_gregset[15]; tf->tf_spsr = mcp->mc_gregset[16]; #ifdef VFP if (mcp->mc_vfp_size == sizeof(mc_vfp) && mcp->mc_vfp_ptr != 0) { if (copyin((void *)(uintptr_t)mcp->mc_vfp_ptr, &mc_vfp, sizeof(mc_vfp)) != 0) return (EFAULT); set_fpcontext32(td, &mc_vfp); } #endif return (0); } #define UC_COPY_SIZE offsetof(ucontext32_t, uc_link) int freebsd32_getcontext(struct thread *td, struct freebsd32_getcontext_args *uap) { ucontext32_t uc; int ret; if (uap->ucp == NULL) ret = EINVAL; else { memset(&uc, 0, sizeof(uc)); get_mcontext32(td, &uc.uc_mcontext, GET_MC_CLEAR_RET); PROC_LOCK(td->td_proc); uc.uc_sigmask = td->td_sigmask; PROC_UNLOCK(td->td_proc); ret = copyout(&uc, uap->ucp, UC_COPY_SIZE); } return (ret); } int freebsd32_setcontext(struct thread *td, struct freebsd32_setcontext_args *uap) { ucontext32_t uc; int ret; if (uap->ucp == NULL) ret = EINVAL; else { ret = copyin(uap->ucp, &uc, UC_COPY_SIZE); if (ret == 0) { ret = set_mcontext32(td, &uc.uc_mcontext); if (ret == 0) kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); } } return (ret); } int freebsd32_sigreturn(struct thread *td, struct freebsd32_sigreturn_args *uap) { ucontext32_t uc; int error; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); error = set_mcontext32(td, &uc.uc_mcontext); if (error != 0) return (0); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } int freebsd32_swapcontext(struct thread *td, struct freebsd32_swapcontext_args *uap) { ucontext32_t uc; int ret; if (uap->oucp == NULL || uap->ucp == NULL) ret = EINVAL; else { bzero(&uc, sizeof(uc)); get_mcontext32(td, &uc.uc_mcontext, GET_MC_CLEAR_RET); PROC_LOCK(td->td_proc); uc.uc_sigmask = td->td_sigmask; PROC_UNLOCK(td->td_proc); ret = copyout(&uc, uap->oucp, UC32_COPY_SIZE); if (ret == 0) { ret = copyin(uap->ucp, &uc, UC32_COPY_SIZE); if (ret == 0) { ret = set_mcontext32(td, &uc.uc_mcontext); kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); } } } return (ret); } void freebsd32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe32 *fp, frame; struct sigacts *psp; struct siginfo32 siginfo; struct sysentvec *sysent; int onstack; int sig; int code; siginfo_to_siginfo32(&ksi->ksi_info, &siginfo); td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; code = ksi->ksi_code; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_x[13]); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !(onstack) && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe32 *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else fp = (struct sigframe32 *)td->td_frame->tf_x[13]; /* make room on the stack */ fp--; /* make the stack aligned */ fp = (struct sigframe32 *)((unsigned long)(fp) &~ (8 - 1)); /* Populate the siginfo frame. */ get_mcontext32(td, &frame.sf_uc.uc_mcontext, 0); #ifdef VFP get_fpcontext32(td, &frame.sf_vfp); frame.sf_uc.uc_mcontext.mc_vfp_size = sizeof(fp->sf_vfp); frame.sf_uc.uc_mcontext.mc_vfp_ptr = (uint32_t)(uintptr_t)&fp->sf_vfp; #else frame.sf_uc.uc_mcontext.mc_vfp_size = 0; frame.sf_uc.uc_mcontext.mc_vfp_ptr = (uint32_t)NULL; #endif frame.sf_si = siginfo; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK ) ? ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; frame.sf_uc.uc_stack.ss_sp = (uintptr_t)td->td_sigstk.ss_sp; frame.sf_uc.uc_stack.ss_size = td->td_sigstk.ss_size; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } /* * Build context to run handler in. We invoke the handler * directly, only returning via the trampoline. Note the * trampoline version numbers are coordinated with machine- * dependent code in libc. */ tf->tf_x[0] = sig; tf->tf_x[1] = (register_t)&fp->sf_si; tf->tf_x[2] = (register_t)&fp->sf_uc; /* the trampoline uses r5 as the uc address */ tf->tf_x[5] = (register_t)&fp->sf_uc; tf->tf_elr = (register_t)catcher; tf->tf_x[13] = (register_t)fp; sysent = p->p_sysent; if (sysent->sv_sigcode_base != 0) tf->tf_x[14] = (register_t)sysent->sv_sigcode_base; else tf->tf_x[14] = (register_t)(sysent->sv_psstrings - *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ if ((register_t)catcher & 1) tf->tf_spsr |= PSR_T; else tf->tf_spsr &= ~PSR_T; CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_x[14], tf->tf_x[13]); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } #ifdef COMPAT_43 /* * COMPAT_FREEBSD32 assumes we have this system call when COMPAT_43 is defined. * FreeBSD/arm provies a similar getpagesize() syscall. */ #define ARM32_PAGE_SIZE 4096 int ofreebsd32_getpagesize(struct thread *td, struct ofreebsd32_getpagesize_args *uap) { td->td_retval[0] = ARM32_PAGE_SIZE; return (0); } /* * Mirror the osigreturn definition in kern_sig.c for !i386 platforms. This * mirrors what's connected to the FreeBSD/arm syscall. */ int ofreebsd32_sigreturn(struct thread *td, struct ofreebsd32_sigreturn_args *uap) { return (nosys(td, (struct nosys_args *)uap)); } #endif Index: head/sys/arm64/arm64/gic_v3_reg.h =================================================================== --- head/sys/arm64/arm64/gic_v3_reg.h (revision 354176) +++ head/sys/arm64/arm64/gic_v3_reg.h (revision 354177) @@ -1,420 +1,420 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_V3_REG_H_ #define _GIC_V3_REG_H_ /* * Maximum number of interrupts * supported by GIC (including SGIs, PPIs and SPIs) */ #define GIC_I_NUM_MAX (1020) /* * Priority MAX/MIN values */ #define GIC_PRIORITY_MAX (0x00UL) /* Upper value is determined by LPI max priority */ #define GIC_PRIORITY_MIN (0xFCUL) /* Numbers for shared peripheral interrupts */ #define GIC_LAST_SPI (1019) /* Numbers for local peripheral interrupts */ #define GIC_FIRST_LPI (8192) /* * Registers (v2/v3) */ /* GICD_CTLR */ #define GICD_CTLR_G1 (1 << 0) #define GICD_CTLR_G1A (1 << 1) #define GICD_CTLR_ARE_NS (1 << 4) #define GICD_CTLR_RWP (1 << 31) /* GICD_TYPER */ #define GICD_TYPER_IDBITS(n) ((((n) >> 19) & 0x1F) + 1) /* * Registers (v3) */ #define GICD_IROUTER(n) (0x6000 + ((n) * 8)) #define GICD_PIDR4 0xFFD0 #define GICD_PIDR5 0xFFD4 #define GICD_PIDR6 0xFFD8 #define GICD_PIDR7 0xFFDC #define GICD_PIDR0 0xFFE0 #define GICD_PIDR1 0xFFE4 #define GICD_PIDR2 0xFFE8 #define GICR_PIDR2_ARCH_SHIFT 4 #define GICR_PIDR2_ARCH_MASK 0xF0 #define GICR_PIDR2_ARCH(x) \ (((x) & GICR_PIDR2_ARCH_MASK) >> GICR_PIDR2_ARCH_SHIFT) #define GICR_PIDR2_ARCH_GICv3 0x3 #define GICR_PIDR2_ARCH_GICv4 0x4 #define GICD_PIDR3 0xFFEC /* Redistributor registers */ #define GICR_CTLR GICD_CTLR #define GICR_CTLR_LPI_ENABLE (1 << 0) #define GICR_PIDR2 GICD_PIDR2 #define GICR_TYPER (0x0008) #define GICR_TYPER_PLPIS (1 << 0) #define GICR_TYPER_VLPIS (1 << 1) #define GICR_TYPER_LAST (1 << 4) #define GICR_TYPER_CPUNUM_SHIFT (8) #define GICR_TYPER_CPUNUM_MASK (0xFFFUL << GICR_TYPER_CPUNUM_SHIFT) #define GICR_TYPER_CPUNUM(x) \ (((x) & GICR_TYPER_CPUNUM_MASK) >> GICR_TYPER_CPUNUM_SHIFT) #define GICR_TYPER_AFF_SHIFT (32) #define GICR_WAKER (0x0014) #define GICR_WAKER_PS (1 << 1) /* Processor sleep */ #define GICR_WAKER_CA (1 << 2) /* Children asleep */ #define GICR_PROPBASER (0x0070) #define GICR_PROPBASER_IDBITS_MASK 0x1FUL /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GICR_PROPBASER_CACHE_SHIFT 7 #define GICR_PROPBASER_CACHE_DnGnRnE 0x0UL #define GICR_PROPBASER_CACHE_NIN 0x1UL #define GICR_PROPBASER_CACHE_NIRAWT 0x2UL #define GICR_PROPBASER_CACHE_NIRAWB 0x3UL #define GICR_PROPBASER_CACHE_NIWAWT 0x4UL #define GICR_PROPBASER_CACHE_NIWAWB 0x5UL #define GICR_PROPBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PROPBASER_CACHE_NIRAWAWB 0x7UL #define GICR_PROPBASER_CACHE_MASK \ (0x7UL << GICR_PROPBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GICR_PROPBASER_SHARE_SHIFT 10 #define GICR_PROPBASER_SHARE_NS 0x0UL #define GICR_PROPBASER_SHARE_IS 0x1UL #define GICR_PROPBASER_SHARE_OS 0x2UL #define GICR_PROPBASER_SHARE_RES 0x3UL #define GICR_PROPBASER_SHARE_MASK \ (0x3UL << GICR_PROPBASER_SHARE_SHIFT) #define GICR_PENDBASER (0x0078) /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GICR_PENDBASER_CACHE_SHIFT 7 #define GICR_PENDBASER_CACHE_DnGnRnE 0x0UL #define GICR_PENDBASER_CACHE_NIN 0x1UL #define GICR_PENDBASER_CACHE_NIRAWT 0x2UL #define GICR_PENDBASER_CACHE_NIRAWB 0x3UL #define GICR_PENDBASER_CACHE_NIWAWT 0x4UL #define GICR_PENDBASER_CACHE_NIWAWB 0x5UL #define GICR_PENDBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PENDBASER_CACHE_NIRAWAWB 0x7UL #define GICR_PENDBASER_CACHE_MASK \ (0x7UL << GICR_PENDBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GICR_PENDBASER_SHARE_SHIFT 10 #define GICR_PENDBASER_SHARE_NS 0x0UL #define GICR_PENDBASER_SHARE_IS 0x1UL #define GICR_PENDBASER_SHARE_OS 0x2UL #define GICR_PENDBASER_SHARE_RES 0x3UL #define GICR_PENDBASER_SHARE_MASK \ (0x3UL << GICR_PENDBASER_SHARE_SHIFT) /* Re-distributor registers for SGIs and PPIs */ #define GICR_RD_BASE_SIZE PAGE_SIZE_64K #define GICR_SGI_BASE_SIZE PAGE_SIZE_64K #define GICR_VLPI_BASE_SIZE PAGE_SIZE_64K #define GICR_RESERVED_SIZE PAGE_SIZE_64K #define GICR_IGROUPR0 (0x0080) #define GICR_ISENABLER0 (0x0100) #define GICR_ICENABLER0 (0x0180) #define GICR_I_ENABLER_SGI_MASK (0x0000FFFF) #define GICR_I_ENABLER_PPI_MASK (0xFFFF0000) #define GICR_I_PER_IPRIORITYn (GICD_I_PER_IPRIORITYn) /* ITS registers */ #define GITS_PIDR2 GICR_PIDR2 #define GITS_PIDR2_ARCH_MASK GICR_PIDR2_ARCH_MASK #define GITS_PIDR2_ARCH_GICv3 GICR_PIDR2_ARCH_GICv3 #define GITS_PIDR2_ARCH_GICv4 GICR_PIDR2_ARCH_GICv4 #define GITS_CTLR (0x0000) #define GITS_CTLR_EN (1 << 0) #define GITS_IIDR (0x0004) #define GITS_IIDR_PRODUCT_SHIFT 24 #define GITS_IIDR_PRODUCT_MASK (0xff << GITS_IIDR_PRODUCT_SHIFT) #define GITS_IIDR_VARIANT_SHIFT 16 #define GITS_IIDR_VARIANT_MASK (0xf << GITS_IIDR_VARIANT_SHIFT) #define GITS_IIDR_REVISION_SHIFT 12 #define GITS_IIDR_REVISION_MASK (0xf << GITS_IIDR_REVISION_SHIFT) #define GITS_IIDR_IMPLEMENTOR_SHIFT 0 #define GITS_IIDR_IMPLEMENTOR_MASK (0xfff << GITS_IIDR_IMPLEMENTOR_SHIFT) #define GITS_IIDR_RAW(impl, prod, var, rev) \ ((prod) << GITS_IIDR_PRODUCT_SHIFT | \ (var) << GITS_IIDR_VARIANT_SHIFT | \ (rev) << GITS_IIDR_REVISION_SHIFT | \ (impl) << GITS_IIDR_IMPLEMENTOR_SHIFT) #define GITS_IIDR_IMPL_CAVIUM (0x34c) #define GITS_IIDR_PROD_THUNDER (0xa1) #define GITS_IIDR_VAR_THUNDER_1 (0x0) #define GITS_CBASER (0x0080) #define GITS_CBASER_VALID (1UL << 63) /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GITS_CBASER_CACHE_SHIFT 59 #define GITS_CBASER_CACHE_DnGnRnE 0x0UL #define GITS_CBASER_CACHE_NIN 0x1UL #define GITS_CBASER_CACHE_NIRAWT 0x2UL #define GITS_CBASER_CACHE_NIRAWB 0x3UL #define GITS_CBASER_CACHE_NIWAWT 0x4UL #define GITS_CBASER_CACHE_NIWAWB 0x5UL #define GITS_CBASER_CACHE_NIRAWAWT 0x6UL #define GITS_CBASER_CACHE_NIRAWAWB 0x7UL #define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GITS_CBASER_SHARE_SHIFT 10 #define GITS_CBASER_SHARE_NS 0x0UL #define GITS_CBASER_SHARE_IS 0x1UL #define GITS_CBASER_SHARE_OS 0x2UL #define GITS_CBASER_SHARE_RES 0x3UL #define GITS_CBASER_SHARE_MASK \ (0x3UL << GITS_CBASER_SHARE_SHIFT) #define GITS_CBASER_PA_SHIFT 12 #define GITS_CBASER_PA_MASK (0xFFFFFFFFFUL << GITS_CBASER_PA_SHIFT) #define GITS_CWRITER (0x0088) #define GITS_CREADR (0x0090) #define GITS_BASER_BASE (0x0100) #define GITS_BASER(x) (GITS_BASER_BASE + (x) * 8) #define GITS_BASER_VALID (1UL << 63) #define GITS_BASER_TYPE_SHIFT 56 #define GITS_BASER_TYPE(x) \ (((x) & GITS_BASER_TYPE_MASK) >> GITS_BASER_TYPE_SHIFT) #define GITS_BASER_TYPE_UNIMPL 0x0UL /* Unimplemented */ #define GITS_BASER_TYPE_DEV 0x1UL /* Devices */ #define GITS_BASER_TYPE_VP 0x2UL /* Virtual Processors */ #define GITS_BASER_TYPE_PP 0x3UL /* Physical Processors */ #define GITS_BASER_TYPE_IC 0x4UL /* Interrupt Collections */ #define GITS_BASER_TYPE_RES5 0x5UL /* Reserved */ #define GITS_BASER_TYPE_RES6 0x6UL /* Reserved */ #define GITS_BASER_TYPE_RES7 0x7UL /* Reserved */ #define GITS_BASER_TYPE_MASK (0x7UL << GITS_BASER_TYPE_SHIFT) /* * Cacheability * 0x0 - Non-cacheable, non-bufferable * 0x1 - Non-cacheable * 0x2 - Read-allocate, Write-through * 0x3 - Read-allocate, Write-back * 0x4 - Write-allocate, Write-through * 0x5 - Write-allocate, Write-back * 0x6 - Read-allocate, Write-allocate, Write-through * 0x7 - Read-allocate, Write-allocate, Write-back */ #define GITS_BASER_CACHE_SHIFT 59 #define GITS_BASER_CACHE_NCNB 0x0UL #define GITS_BASER_CACHE_NC 0x1UL #define GITS_BASER_CACHE_RAWT 0x2UL #define GITS_BASER_CACHE_RAWB 0x3UL #define GITS_BASER_CACHE_WAWT 0x4UL #define GITS_BASER_CACHE_WAWB 0x5UL #define GITS_BASER_CACHE_RAWAWT 0x6UL #define GITS_BASER_CACHE_RAWAWB 0x7UL #define GITS_BASER_CACHE_MASK (0x7UL << GITS_BASER_CACHE_SHIFT) #define GITS_BASER_ESIZE_SHIFT 48 #define GITS_BASER_ESIZE_MASK (0x1FUL << GITS_BASER_ESIZE_SHIFT) #define GITS_BASER_ESIZE(x) \ ((((x) & GITS_BASER_ESIZE_MASK) >> GITS_BASER_ESIZE_SHIFT) + 1) #define GITS_BASER_PA_SHIFT 12 #define GITS_BASER_PA_MASK (0xFFFFFFFFFUL << GITS_BASER_PA_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GITS_BASER_SHARE_SHIFT 10 #define GITS_BASER_SHARE_NS 0x0UL #define GITS_BASER_SHARE_IS 0x1UL #define GITS_BASER_SHARE_OS 0x2UL #define GITS_BASER_SHARE_RES 0x3UL #define GITS_BASER_SHARE_MASK (0x3UL << GITS_BASER_SHARE_SHIFT) #define GITS_BASER_PSZ_SHIFT 8 #define GITS_BASER_PSZ_4K 0x0UL #define GITS_BASER_PSZ_16K 0x1UL #define GITS_BASER_PSZ_64K 0x2UL #define GITS_BASER_PSZ_MASK (0x3UL << GITS_BASER_PSZ_SHIFT) #define GITS_BASER_SIZE_MASK 0xFFUL #define GITS_BASER_NUM 8 #define GITS_TYPER (0x0008) #define GITS_TYPER_PTA (1UL << 19) #define GITS_TYPER_DEVB_SHIFT 13 #define GITS_TYPER_DEVB_MASK (0x1FUL << GITS_TYPER_DEVB_SHIFT) /* Number of device identifiers implemented */ #define GITS_TYPER_DEVB(x) \ ((((x) & GITS_TYPER_DEVB_MASK) >> GITS_TYPER_DEVB_SHIFT) + 1) #define GITS_TYPER_ITTES_SHIFT 4 #define GITS_TYPER_ITTES_MASK (0xFUL << GITS_TYPER_ITTES_SHIFT) /* Number of bytes per ITT Entry */ #define GITS_TYPER_ITTES(x) \ ((((x) & GITS_TYPER_ITTES_MASK) >> GITS_TYPER_ITTES_SHIFT) + 1) #define GITS_TRANSLATER (0x10040) /* * LPI related */ #define LPI_CONF_PRIO_MASK (0xFC) #define LPI_CONF_GROUP1 (1 << 1) #define LPI_CONF_ENABLE (1 << 0) /* * CPU interface */ /* * Registers list (ICC_xyz_EL1): * * PMR - Priority Mask Register * * interrupts of priority higher than specified * in this mask will be signalled to the CPU. * (0xff - lowest possible prio., 0x00 - highest prio.) * * CTLR - Control Register * * controls behavior of the CPU interface and displays * implemented features. * * IGRPEN1 - Interrupt Group 1 Enable Register * * IAR1 - Interrupt Acknowledge Register Group 1 * * contains number of the highest priority pending * interrupt from the Group 1. * * EOIR1 - End of Interrupt Register Group 1 * * Writes inform CPU interface about completed Group 1 * interrupts processing. */ #define gic_icc_write(reg, val) \ do { \ - WRITE_SPECIALREG(ICC_ ##reg ##_EL1, val); \ + WRITE_SPECIALREG(icc_ ##reg ##_el1, val); \ isb(); \ } while (0) #define gic_icc_read(reg) \ ({ \ uint64_t val; \ \ - val = READ_SPECIALREG(ICC_ ##reg ##_EL1); \ + val = READ_SPECIALREG(icc_ ##reg ##_el1); \ (val); \ }) #define gic_icc_set(reg, mask) \ do { \ uint64_t val; \ val = gic_icc_read(reg); \ val |= (mask); \ gic_icc_write(reg, val); \ } while (0) #define gic_icc_clear(reg, mask) \ do { \ uint64_t val; \ val = gic_icc_read(reg); \ val &= ~(mask); \ gic_icc_write(reg, val); \ } while (0) #endif /* _GIC_V3_REG_H_ */ Index: head/sys/arm64/arm64/machdep.c =================================================================== --- head/sys/arm64/arm64/machdep.c (revision 354176) +++ head/sys/arm64/arm64/machdep.c (revision 354177) @@ -1,1255 +1,1255 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_acpi.h" #include "opt_platform.h" #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #endif enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; struct pcpu __pcpu[MAXCPU]; static struct trapframe proc0_tf; int early_boot = 1; int cold = 1; struct kva_md_info kmi; int64_t dcache_line_size; /* The minimum D cache line size */ int64_t icache_line_size; /* The minimum I cache line size */ int64_t idcache_line_size; /* The minimum cache line size */ int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ int has_pan; /* * Physical address of the EFI System Table. Stashed from the metadata hints * passed into the kernel and used by the EFI code to call runtime services. */ vm_paddr_t efi_systbl_phys; /* pagezero_* implementations are provided in support.S */ void pagezero_simple(void *); void pagezero_cache(void *); /* pagezero_simple is default pagezero */ void (*pagezero)(void *p) = pagezero_simple; static void pan_setup(void) { uint64_t id_aa64mfr1; id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) has_pan = 1; } void pan_enable(void) { /* * The LLVM integrated assembler doesn't understand the PAN * PSTATE field. Because of this we need to manually create * the instruction in an asm block. This is equivalent to: * msr pan, #1 * * This sets the PAN bit, stopping the kernel from accessing * memory when userspace can also access it unless the kernel * uses the userspace load/store instructions. */ if (has_pan) { WRITE_SPECIALREG(sctlr_el1, READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); } } static void cpu_startup(void *dummy) { undef_init(); identify_cpu(); install_cpu_errata(); vm_ksubmap_init(&kmi); bufinit(); vm_pager_bufferinit(); } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); int cpu_idle_wakeup(int cpu) { return (0); } int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; regs->sp = frame->tf_sp; regs->lr = frame->tf_lr; regs->elr = frame->tf_elr; regs->spsr = frame->tf_spsr; memcpy(regs->x, frame->tf_x, sizeof(regs->x)); #ifdef COMPAT_FREEBSD32 /* * We may be called here for a 32bits process, if we're using a * 64bits debugger. If so, put PC and SPSR where it expects it. */ if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { regs->x[15] = frame->tf_elr; regs->x[16] = frame->tf_spsr; } #endif return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; frame->tf_sp = regs->sp; frame->tf_lr = regs->lr; frame->tf_elr = regs->elr; frame->tf_spsr &= ~PSR_FLAGS; frame->tf_spsr |= regs->spsr & PSR_FLAGS; memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); #ifdef COMPAT_FREEBSD32 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { /* * We may be called for a 32bits process if we're using * a 64bits debugger. If so, get PC and SPSR from where * it put it. */ frame->tf_elr = regs->x[15]; frame->tf_spsr = regs->x[16] & PSR_FLAGS; } #endif return (0); } int fill_fpregs(struct thread *td, struct fpreg *regs) { #ifdef VFP struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ if (td == curthread) vfp_save_state(td, pcb); KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Called fill_fpregs while the kernel is using the VFP")); memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs, sizeof(regs->fp_q)); regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr; regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr; } else #endif memset(regs, 0, sizeof(*regs)); return (0); } int set_fpregs(struct thread *td, struct fpreg *regs) { #ifdef VFP struct pcb *pcb; pcb = td->td_pcb; KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Called set_fpregs while the kernel is using the VFP")); memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q)); pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr; pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr; #endif return (0); } int fill_dbregs(struct thread *td, struct dbreg *regs) { printf("ARM64TODO: fill_dbregs"); return (EDOOFUS); } int set_dbregs(struct thread *td, struct dbreg *regs) { printf("ARM64TODO: set_dbregs"); return (EDOOFUS); } #ifdef COMPAT_FREEBSD32 int fill_regs32(struct thread *td, struct reg32 *regs) { int i; struct trapframe *tf; tf = td->td_frame; for (i = 0; i < 13; i++) regs->r[i] = tf->tf_x[i]; /* For arm32, SP is r13 and LR is r14 */ regs->r_sp = tf->tf_x[13]; regs->r_lr = tf->tf_x[14]; regs->r_pc = tf->tf_elr; regs->r_cpsr = tf->tf_spsr; return (0); } int set_regs32(struct thread *td, struct reg32 *regs) { int i; struct trapframe *tf; tf = td->td_frame; for (i = 0; i < 13; i++) tf->tf_x[i] = regs->r[i]; /* For arm 32, SP is r13 an LR is r14 */ tf->tf_x[13] = regs->r_sp; tf->tf_x[14] = regs->r_lr; tf->tf_elr = regs->r_pc; tf->tf_spsr = regs->r_cpsr; return (0); } int fill_fpregs32(struct thread *td, struct fpreg32 *regs) { printf("ARM64TODO: fill_fpregs32"); return (EDOOFUS); } int set_fpregs32(struct thread *td, struct fpreg32 *regs) { printf("ARM64TODO: set_fpregs32"); return (EDOOFUS); } int fill_dbregs32(struct thread *td, struct dbreg32 *regs) { printf("ARM64TODO: fill_dbregs32"); return (EDOOFUS); } int set_dbregs32(struct thread *td, struct dbreg32 *regs) { printf("ARM64TODO: set_dbregs32"); return (EDOOFUS); } #endif int ptrace_set_pc(struct thread *td, u_long addr) { printf("ARM64TODO: ptrace_set_pc"); return (EDOOFUS); } int ptrace_single_step(struct thread *td) { td->td_frame->tf_spsr |= PSR_SS; td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; return (0); } int ptrace_clear_single_step(struct thread *td) { td->td_frame->tf_spsr &= ~PSR_SS; td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; return (0); } void exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) { struct trapframe *tf = td->td_frame; memset(tf, 0, sizeof(struct trapframe)); tf->tf_x[0] = stack; tf->tf_sp = STACKALIGN(stack); tf->tf_lr = imgp->entry_addr; tf->tf_elr = imgp->entry_addr; } /* Sanity check these are the same size, they will be memcpy'd to and fro */ CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == sizeof((struct gpregs *)0)->gp_x); CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == sizeof((struct reg *)0)->x); int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; if (clear_ret & GET_MC_CLEAR_RET) { mcp->mc_gpregs.gp_x[0] = 0; mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; } else { mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; mcp->mc_gpregs.gp_spsr = tf->tf_spsr; } memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); mcp->mc_gpregs.gp_sp = tf->tf_sp; mcp->mc_gpregs.gp_lr = tf->tf_lr; mcp->mc_gpregs.gp_elr = tf->tf_elr; return (0); } int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf = td->td_frame; uint32_t spsr; spsr = mcp->mc_gpregs.gp_spsr; if ((spsr & PSR_M_MASK) != PSR_M_EL0t || (spsr & (PSR_AARCH32 | PSR_F | PSR_I | PSR_A | PSR_D)) != 0) return (EINVAL); memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); tf->tf_sp = mcp->mc_gpregs.gp_sp; tf->tf_lr = mcp->mc_gpregs.gp_lr; tf->tf_elr = mcp->mc_gpregs.gp_elr; tf->tf_spsr = mcp->mc_gpregs.gp_spsr; return (0); } static void get_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef VFP struct pcb *curpcb; critical_enter(); curpcb = curthread->td_pcb; if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ vfp_save_state(td, curpcb); KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, ("Called get_fpcontext while the kernel is using the VFP")); KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Non-userspace FPU flags set in get_fpcontext")); memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs, sizeof(mcp->mc_fpregs)); mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr; mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr; mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; mcp->mc_flags |= _MC_FP_VALID; } critical_exit(); #endif } static void set_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef VFP struct pcb *curpcb; critical_enter(); if ((mcp->mc_flags & _MC_FP_VALID) != 0) { curpcb = curthread->td_pcb; /* * Discard any vfp state for the current thread, we * are about to override it. */ vfp_discard(td); KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, ("Called set_fpcontext while the kernel is using the VFP")); memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q, sizeof(mcp->mc_fpregs)); curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr; curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr; curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK; } critical_exit(); #endif } void cpu_idle(int busy) { spinlock_enter(); if (!busy) cpu_idleclock(); if (!sched_runnable()) __asm __volatile( "dsb sy \n" "wfi \n"); if (!busy) cpu_activeclock(); spinlock_exit(); } void cpu_halt(void) { /* We should have shutdown by now, if not enter a low power sleep */ intr_disable(); while (1) { __asm __volatile("wfi"); } } /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { /* ARM64TODO TBD */ } /* Get current clock frequency for the given CPU ID. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { struct pcpu *pc; pc = pcpu_find(cpu_id); if (pc == NULL || rate == NULL) return (EINVAL); if (pc->pc_clock == 0) return (EOPNOTSUPP); *rate = pc->pc_clock; return (0); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { pcpu->pc_acpi_id = 0xffffffff; } void spinlock_enter(void) { struct thread *td; register_t daif; td = curthread; if (td->td_md.md_spinlock_count == 0) { daif = intr_disable(); td->td_md.md_spinlock_count = 1; td->td_md.md_saved_daif = daif; } else td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; register_t daif; td = curthread; critical_exit(); daif = td->td_md.md_saved_daif; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) intr_restore(daif); } #ifndef _SYS_SYSPROTO_H_ struct sigreturn_args { ucontext_t *ucp; }; #endif int sys_sigreturn(struct thread *td, struct sigreturn_args *uap) { ucontext_t uc; int error; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); error = set_mcontext(td, &uc.uc_mcontext); if (error != 0) return (error); set_fpcontext(td, &uc.uc_mcontext); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { int i; for (i = 0; i < PCB_LR; i++) pcb->pcb_x[i] = tf->tf_x[i]; pcb->pcb_x[PCB_LR] = tf->tf_lr; pcb->pcb_pc = tf->tf_elr; pcb->pcb_sp = tf->tf_sp; } void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; struct sysentvec *sysent; int onstack, sig; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else { fp = (struct sigframe *)td->td_frame->tf_sp; } /* Make room, keeping the stack aligned */ fp--; fp = (struct sigframe *)STACKALIGN(fp); /* Fill in the frame to copy out */ bzero(&frame, sizeof(frame)); get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); get_fpcontext(td, &frame.sf_uc.uc_mcontext); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack = td->td_sigstk; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ? (onstack ? SS_ONSTACK : 0) : SS_DISABLE; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } tf->tf_x[0]= sig; tf->tf_x[1] = (register_t)&fp->sf_si; tf->tf_x[2] = (register_t)&fp->sf_uc; tf->tf_elr = (register_t)catcher; tf->tf_sp = (register_t)fp; sysent = p->p_sysent; if (sysent->sv_sigcode_base != 0) tf->tf_lr = (register_t)sysent->sv_sigcode_base; else tf->tf_lr = (register_t)(sysent->sv_psstrings - *(sysent->sv_szsigcode)); CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, tf->tf_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } static void init_proc0(vm_offset_t kstack) { struct pcpu *pcpup = &__pcpu[0]; proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = KSTACK_PAGES; thread0.td_pcb = (struct pcb *)(thread0.td_kstack + thread0.td_kstack_pages * KSTACK_PAGES) - 1; thread0.td_pcb->pcb_fpflags = 0; thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; thread0.td_pcb->pcb_vfpcpu = UINT_MAX; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; /* Set the base address of translation table 0. */ thread0.td_proc->p_md.md_l0addr = READ_SPECIALREG(ttbr0_el1); } typedef struct { uint32_t type; uint64_t phys_start; uint64_t virt_start; uint64_t num_pages; uint64_t attr; } EFI_MEMORY_DESCRIPTOR; typedef void (*efi_map_entry_cb)(struct efi_md *); static void foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb) { struct efi_md *map, *p; size_t efisz; int ndesc, i; /* * Memory map data provided by UEFI via the GetMemoryMap * Boot Services API. */ efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; map = (struct efi_md *)((uint8_t *)efihdr + efisz); if (efihdr->descriptor_size == 0) return; ndesc = efihdr->memory_size / efihdr->descriptor_size; for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, efihdr->descriptor_size)) { cb(p); } } static void exclude_efi_map_entry(struct efi_md *p) { switch (p->md_type) { case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ break; default: arm_physmem_exclude_region(p->md_phys, p->md_pages * PAGE_SIZE, EXFLAG_NOALLOC); } } static void exclude_efi_map_entries(struct efi_map_header *efihdr) { foreach_efi_map_entry(efihdr, exclude_efi_map_entry); } static void add_efi_map_entry(struct efi_md *p) { switch (p->md_type) { case EFI_MD_TYPE_RT_DATA: /* * Runtime data will be excluded after the DMAP * region is created to stop it from being added * to phys_avail. */ case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ arm_physmem_hardware_region(p->md_phys, p->md_pages * PAGE_SIZE); break; } } static void add_efi_map_entries(struct efi_map_header *efihdr) { foreach_efi_map_entry(efihdr, add_efi_map_entry); } static void print_efi_map_entry(struct efi_md *p) { const char *type; static const char *types[] = { "Reserved", "LoaderCode", "LoaderData", "BootServicesCode", "BootServicesData", "RuntimeServicesCode", "RuntimeServicesData", "ConventionalMemory", "UnusableMemory", "ACPIReclaimMemory", "ACPIMemoryNVS", "MemoryMappedIO", "MemoryMappedIOPortSpace", "PalCode", "PersistentMemory" }; if (p->md_type < nitems(types)) type = types[p->md_type]; else type = ""; printf("%23s %012lx %12p %08lx ", type, p->md_phys, p->md_virt, p->md_pages); if (p->md_attr & EFI_MD_ATTR_UC) printf("UC "); if (p->md_attr & EFI_MD_ATTR_WC) printf("WC "); if (p->md_attr & EFI_MD_ATTR_WT) printf("WT "); if (p->md_attr & EFI_MD_ATTR_WB) printf("WB "); if (p->md_attr & EFI_MD_ATTR_UCE) printf("UCE "); if (p->md_attr & EFI_MD_ATTR_WP) printf("WP "); if (p->md_attr & EFI_MD_ATTR_RP) printf("RP "); if (p->md_attr & EFI_MD_ATTR_XP) printf("XP "); if (p->md_attr & EFI_MD_ATTR_NV) printf("NV "); if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) printf("MORE_RELIABLE "); if (p->md_attr & EFI_MD_ATTR_RO) printf("RO "); if (p->md_attr & EFI_MD_ATTR_RT) printf("RUNTIME"); printf("\n"); } static void print_efi_map_entries(struct efi_map_header *efihdr) { printf("%23s %12s %12s %8s %4s\n", "Type", "Physical", "Virtual", "#Pages", "Attr"); foreach_efi_map_entry(efihdr, print_efi_map_entry); } #ifdef FDT static void try_load_dtb(caddr_t kmdp) { vm_offset_t dtbp; dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); if (dtbp == (vm_offset_t)NULL) { printf("ERROR loading DTB\n"); return; } if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); } #endif static bool bus_probe(void) { bool has_acpi, has_fdt; char *order, *env; has_acpi = has_fdt = false; #ifdef FDT has_fdt = (OF_peer(0) != 0); #endif #ifdef DEV_ACPI has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0); #endif env = kern_getenv("kern.cfg.order"); if (env != NULL) { order = env; while (order != NULL) { if (has_acpi && strncmp(order, "acpi", 4) == 0 && (order[4] == ',' || order[4] == '\0')) { arm64_bus_method = ARM64_BUS_ACPI; break; } if (has_fdt && strncmp(order, "fdt", 3) == 0 && (order[3] == ',' || order[3] == '\0')) { arm64_bus_method = ARM64_BUS_FDT; break; } order = strchr(order, ','); } freeenv(env); /* If we set the bus method it is valid */ if (arm64_bus_method != ARM64_BUS_NONE) return (true); } /* If no order or an invalid order was set use the default */ if (arm64_bus_method == ARM64_BUS_NONE) { if (has_fdt) arm64_bus_method = ARM64_BUS_FDT; else if (has_acpi) arm64_bus_method = ARM64_BUS_ACPI; } /* * If no option was set the default is valid, otherwise we are * setting one to get cninit() working, then calling panic to tell * the user about the invalid bus setup. */ return (env == NULL); } static void cache_setup(void) { int dcache_line_shift, icache_line_shift, dczva_line_shift; uint32_t ctr_el0; uint32_t dczid_el0; ctr_el0 = READ_SPECIALREG(ctr_el0); /* Read the log2 words in each D cache line */ dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); /* Get the D cache line size */ dcache_line_size = sizeof(int) << dcache_line_shift; /* And the same for the I cache */ icache_line_shift = CTR_ILINE_SIZE(ctr_el0); icache_line_size = sizeof(int) << icache_line_shift; idcache_line_size = MIN(dcache_line_size, icache_line_size); dczid_el0 = READ_SPECIALREG(dczid_el0); /* Check if dc zva is not prohibited */ if (dczid_el0 & DCZID_DZP) dczva_line_size = 0; else { /* Same as with above calculations */ dczva_line_shift = DCZID_BS_SIZE(dczid_el0); dczva_line_size = sizeof(int) << dczva_line_shift; /* Change pagezero function */ pagezero = pagezero_cache; } } void initarm(struct arm64_bootparams *abp) { struct efi_fb *efifb; struct efi_map_header *efihdr; struct pcpu *pcpup; char *env; #ifdef FDT struct mem_region mem_regions[FDT_MEM_REGIONS]; int mem_regions_sz; #endif vm_offset_t lastaddr; caddr_t kmdp; bool valid; /* Set the module data location */ preload_metadata = (caddr_t)(uintptr_t)(abp->modulep); /* Find the kernel address */ kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0); link_elf_ireloc(kmdp); #ifdef FDT try_load_dtb(kmdp); #endif efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); /* Find the address to start allocating from */ lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); /* Load the physical memory ranges */ efihdr = (struct efi_map_header *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP); if (efihdr != NULL) add_efi_map_entries(efihdr); #ifdef FDT else { /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, NULL) != 0) panic("Cannot get physical memory regions"); arm_physmem_hardware_regions(mem_regions, mem_regions_sz); } if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) arm_physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); #endif /* Exclude the EFI framebuffer from our view of physical memory. */ efifb = (struct efi_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_FB); if (efifb != NULL) arm_physmem_exclude_region(efifb->fb_addr, efifb->fb_size, EXFLAG_NOALLOC); /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &__pcpu[0]; pcpu_init(pcpup, 0, sizeof(struct pcpu)); /* * Set the pcpu pointer with a backup in tpidr_el1 to be * loaded when entering the kernel from userland. */ __asm __volatile( "mov x18, %0 \n" "msr tpidr_el1, %0" :: "r"(pcpup)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); cache_setup(); pan_setup(); /* Bootstrap enough of pmap to enter the kernel proper */ pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, KERNBASE - abp->kern_delta, lastaddr - KERNBASE); /* Exclude entries neexed in teh DMAP region, but not phys_avail */ if (efihdr != NULL) exclude_efi_map_entries(efihdr); arm_physmem_init_kernel_globals(); devmap_bootstrap(0, NULL); valid = bus_probe(); cninit(); if (!valid) panic("Invalid bus configuration: %s", kern_getenv("kern.cfg.order")); init_proc0(abp->kern_stack); msgbufinit(msgbufp, msgbufsize); mutex_init(); init_param2(physmem); dbg_init(); kdb_init(); pan_enable(); env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); if (boothowto & RB_VERBOSE) { print_efi_map_entries(efihdr); arm_physmem_print_tables(); } early_boot = 0; } void dbg_init(void) { /* Clear OS lock */ - WRITE_SPECIALREG(OSLAR_EL1, 0); + WRITE_SPECIALREG(oslar_el1, 0); /* This permits DDB to use debug registers for watchpoints. */ dbg_monitor_init(); /* TODO: Eventually will need to initialize debug registers here. */ } #ifdef DDB #include DB_SHOW_COMMAND(specialregs, db_show_spregs) { #define PRINT_REG(reg) \ db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) PRINT_REG(actlr_el1); PRINT_REG(afsr0_el1); PRINT_REG(afsr1_el1); PRINT_REG(aidr_el1); PRINT_REG(amair_el1); PRINT_REG(ccsidr_el1); PRINT_REG(clidr_el1); PRINT_REG(contextidr_el1); PRINT_REG(cpacr_el1); PRINT_REG(csselr_el1); PRINT_REG(ctr_el0); PRINT_REG(currentel); PRINT_REG(daif); PRINT_REG(dczid_el0); PRINT_REG(elr_el1); PRINT_REG(esr_el1); PRINT_REG(far_el1); #if 0 /* ARM64TODO: Enable VFP before reading floating-point registers */ PRINT_REG(fpcr); PRINT_REG(fpsr); #endif PRINT_REG(id_aa64afr0_el1); PRINT_REG(id_aa64afr1_el1); PRINT_REG(id_aa64dfr0_el1); PRINT_REG(id_aa64dfr1_el1); PRINT_REG(id_aa64isar0_el1); PRINT_REG(id_aa64isar1_el1); PRINT_REG(id_aa64pfr0_el1); PRINT_REG(id_aa64pfr1_el1); PRINT_REG(id_afr0_el1); PRINT_REG(id_dfr0_el1); PRINT_REG(id_isar0_el1); PRINT_REG(id_isar1_el1); PRINT_REG(id_isar2_el1); PRINT_REG(id_isar3_el1); PRINT_REG(id_isar4_el1); PRINT_REG(id_isar5_el1); PRINT_REG(id_mmfr0_el1); PRINT_REG(id_mmfr1_el1); PRINT_REG(id_mmfr2_el1); PRINT_REG(id_mmfr3_el1); #if 0 /* Missing from llvm */ PRINT_REG(id_mmfr4_el1); #endif PRINT_REG(id_pfr0_el1); PRINT_REG(id_pfr1_el1); PRINT_REG(isr_el1); PRINT_REG(mair_el1); PRINT_REG(midr_el1); PRINT_REG(mpidr_el1); PRINT_REG(mvfr0_el1); PRINT_REG(mvfr1_el1); PRINT_REG(mvfr2_el1); PRINT_REG(revidr_el1); PRINT_REG(sctlr_el1); PRINT_REG(sp_el0); PRINT_REG(spsel); PRINT_REG(spsr_el1); PRINT_REG(tcr_el1); PRINT_REG(tpidr_el0); PRINT_REG(tpidr_el1); PRINT_REG(tpidrro_el0); PRINT_REG(ttbr0_el1); PRINT_REG(ttbr1_el1); PRINT_REG(vbar_el1); #undef PRINT_REG } DB_SHOW_COMMAND(vtop, db_show_vtop) { uint64_t phys; if (have_addr) { phys = arm64_address_translate_s1e1r(addr); db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); phys = arm64_address_translate_s1e1w(addr); db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); phys = arm64_address_translate_s1e0r(addr); db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); phys = arm64_address_translate_s1e0w(addr); db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); } else db_printf("show vtop \n"); } #endif Index: head/sys/arm64/arm64/trap.c =================================================================== --- head/sys/arm64/arm64/trap.c (revision 354176) +++ head/sys/arm64/arm64/trap.c (revision 354177) @@ -1,531 +1,531 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include #endif #ifdef VFP #include #endif #ifdef KDB #include #endif #ifdef DDB #include #endif extern register_t fsu_intr_fault; /* Called from exception.S */ void do_el1h_sync(struct thread *, struct trapframe *); void do_el0_sync(struct thread *, struct trapframe *); void do_el0_error(struct trapframe *); void do_serror(struct trapframe *); void unhandled_exception(struct trapframe *); static void print_registers(struct trapframe *frame); int (*dtrace_invop_jump_addr)(struct trapframe *); typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t, uint64_t, int); static abort_handler data_abort; static abort_handler *abort_handlers[] = { [ISS_DATA_DFSC_TF_L0] = data_abort, [ISS_DATA_DFSC_TF_L1] = data_abort, [ISS_DATA_DFSC_TF_L2] = data_abort, [ISS_DATA_DFSC_TF_L3] = data_abort, [ISS_DATA_DFSC_AFF_L1] = data_abort, [ISS_DATA_DFSC_AFF_L2] = data_abort, [ISS_DATA_DFSC_AFF_L3] = data_abort, [ISS_DATA_DFSC_PF_L1] = data_abort, [ISS_DATA_DFSC_PF_L2] = data_abort, [ISS_DATA_DFSC_PF_L3] = data_abort, }; static __inline void call_trapsignal(struct thread *td, int sig, int code, void *addr) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = addr; trapsignal(td, &ksi); } int cpu_fetch_syscall_args(struct thread *td) { struct proc *p; register_t *ap; struct syscall_args *sa; int nap; nap = 8; p = td->td_proc; ap = td->td_frame->tf_x; sa = &td->td_sa; sa->code = td->td_frame->tf_x[8]; if (sa->code == SYS_syscall || sa->code == SYS___syscall) { sa->code = *ap++; nap--; } if (sa->code >= p->p_sysent->sv_size) sa->callp = &p->p_sysent->sv_table[0]; else sa->callp = &p->p_sysent->sv_table[sa->code]; sa->narg = sa->callp->sy_narg; memcpy(sa->args, ap, nap * sizeof(register_t)); if (sa->narg > nap) panic("ARM64TODO: Could we have more than 8 args?"); td->td_retval[0] = 0; td->td_retval[1] = 0; return (0); } #include "../../kern/subr_syscall.c" static void svc_handler(struct thread *td, struct trapframe *frame) { if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) { syscallenter(td); syscallret(td); } else { call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr); userret(td, frame); } } static void data_abort(struct thread *td, struct trapframe *frame, uint64_t esr, uint64_t far, int lower) { struct vm_map *map; struct proc *p; struct pcb *pcb; vm_prot_t ftype; int error, sig, ucode; #ifdef KDB bool handled; #endif /* * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive * and Store-Exclusive instruction usage restrictions", state * of the exclusive monitors after data abort exception is unknown. */ clrex(); #ifdef KDB if (kdb_active) { kdb_reenter(); return; } #endif pcb = td->td_pcb; p = td->td_proc; if (lower) map = &p->p_vmspace->vm_map; else { intr_enable(); /* The top bit tells us which range to use */ if (far >= VM_MAXUSER_ADDRESS) { map = kernel_map; } else { map = &p->p_vmspace->vm_map; if (map == NULL) map = kernel_map; } } /* * Try to handle translation, access flag, and permission faults. * Translation faults may occur as a result of the required * break-before-make sequence used when promoting or demoting * superpages. Such faults must not occur while holding the pmap lock, * or pmap_fault() will recurse on that lock. */ if ((lower || map == kernel_map || pcb->pcb_onfault != 0) && pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) return; KASSERT(td->td_md.md_spinlock_count == 0, ("data abort with spinlock held")); if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) { print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); panic("data abort in critical section or under mutex"); } switch (ESR_ELx_EXCEPTION(esr)) { case EXCP_INSN_ABORT: case EXCP_INSN_ABORT_L: ftype = VM_PROT_EXECUTE; break; default: ftype = (esr & ISS_DATA_WnR) == 0 ? VM_PROT_READ : VM_PROT_READ | VM_PROT_WRITE; break; } /* Fault in the page. */ error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode); if (error != KERN_SUCCESS) { if (lower) { call_trapsignal(td, sig, ucode, (void *)far); } else { if (td->td_intr_nesting_level == 0 && pcb->pcb_onfault != 0) { frame->tf_x[0] = error; frame->tf_elr = pcb->pcb_onfault; return; } printf("Fatal data abort:\n"); print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); #ifdef KDB if (debugger_on_trap) { kdb_why = KDB_WHY_TRAP; handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0, frame); kdb_why = KDB_WHY_UNSET; if (handled) return; } #endif panic("vm_fault failed: %lx", frame->tf_elr); } } if (lower) userret(td, frame); } static void print_registers(struct trapframe *frame) { u_int reg; for (reg = 0; reg < nitems(frame->tf_x); reg++) { printf(" %sx%d: %16lx\n", (reg < 10) ? " " : "", reg, frame->tf_x[reg]); } printf(" sp: %16lx\n", frame->tf_sp); printf(" lr: %16lx\n", frame->tf_lr); printf(" elr: %16lx\n", frame->tf_elr); printf("spsr: %8x\n", frame->tf_spsr); } void do_el1h_sync(struct thread *td, struct trapframe *frame) { struct trapframe *oframe; uint32_t exception; uint64_t esr, far; int dfsc; /* Read the esr register to get the exception details */ esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); #ifdef KDTRACE_HOOKS if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) return; #endif CTR4(KTR_TRAP, "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); oframe = td->td_frame; switch (exception) { case EXCP_BRK: case EXCP_WATCHPT_EL1: case EXCP_SOFTSTP_EL1: break; default: td->td_frame = frame; break; } switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: #ifdef VFP if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) { vfp_restore_state(); } else #endif { print_registers(frame); printf(" esr: %.8lx\n", esr); panic("VFP exception in the kernel"); } break; case EXCP_INSN_ABORT: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); dfsc = esr & ISS_DATA_DFSC_MASK; if (dfsc < nitems(abort_handlers) && abort_handlers[dfsc] != NULL) abort_handlers[dfsc](td, frame, esr, far, 0); else panic("Unhandled EL1 %s abort: %x", exception == EXCP_INSN_ABORT ? "instruction" : "data", dfsc); break; case EXCP_BRK: #ifdef KDTRACE_HOOKS if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ dtrace_invop_jump_addr != 0) { dtrace_invop_jump_addr(frame); break; } #endif #ifdef KDB kdb_trap(exception, 0, (td->td_frame != NULL) ? td->td_frame : frame); #else panic("No debugger in kernel.\n"); #endif frame->tf_elr += 4; break; case EXCP_WATCHPT_EL1: case EXCP_SOFTSTP_EL1: #ifdef KDB kdb_trap(exception, 0, (td->td_frame != NULL) ? td->td_frame : frame); #else panic("No debugger in kernel.\n"); #endif break; case EXCP_UNKNOWN: if (undef_insn(1, frame)) break; /* FALLTHROUGH */ default: print_registers(frame); panic("Unknown kernel exception %x esr_el1 %lx\n", exception, esr); } td->td_frame = oframe; } void do_el0_sync(struct thread *td, struct trapframe *frame) { pcpu_bp_harden bp_harden; uint32_t exception; uint64_t esr, far; int dfsc; /* Check we have a sane environment when entering from userland */ KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, ("Invalid pcpu address from userland: %p (tpidr %lx)", get_pcpu(), READ_SPECIALREG(tpidr_el1))); esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); switch (exception) { case EXCP_INSN_ABORT_L: far = READ_SPECIALREG(far_el1); /* * Userspace may be trying to train the branch predictor to * attack the kernel. If we are on a CPU affected by this * call the handler to clear the branch predictor state. */ if (far > VM_MAXUSER_ADDRESS) { bp_harden = PCPU_GET(bp_harden); if (bp_harden != NULL) bp_harden(); } break; case EXCP_UNKNOWN: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); break; } intr_enable(); CTR4(KTR_TRAP, "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: #ifdef VFP vfp_restore_state(); #else panic("VFP exception in userland"); #endif break; case EXCP_SVC32: case EXCP_SVC64: svc_handler(td, frame); break; case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: dfsc = esr & ISS_DATA_DFSC_MASK; if (dfsc < nitems(abort_handlers) && abort_handlers[dfsc] != NULL) abort_handlers[dfsc](td, frame, esr, far, 1); else panic("Unhandled EL0 %s abort: %x", exception == EXCP_INSN_ABORT_L ? "instruction" : "data", dfsc); break; case EXCP_UNKNOWN: if (!undef_insn(0, frame)) call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far); userret(td, frame); break; case EXCP_SP_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp); userret(td, frame); break; case EXCP_PC_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_BRK: call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_MSR: call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr); userret(td, frame); break; case EXCP_SOFTSTP_EL0: td->td_frame->tf_spsr &= ~PSR_SS; td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; - WRITE_SPECIALREG(MDSCR_EL1, - READ_SPECIALREG(MDSCR_EL1) & ~DBG_MDSCR_SS); + WRITE_SPECIALREG(mdscr_el1, + READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_SS); call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)frame->tf_elr); userret(td, frame); break; default: call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr); userret(td, frame); break; } KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Kernel VFP flags set while entering userspace")); KASSERT( td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate, ("Kernel VFP state in use when entering userspace")); } /* * TODO: We will need to handle these later when we support ARMv8.2 RAS. */ void do_serror(struct trapframe *frame) { uint64_t esr, far; far = READ_SPECIALREG(far_el1); esr = frame->tf_esr; print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); panic("Unhandled System Error"); } void unhandled_exception(struct trapframe *frame) { uint64_t esr, far; far = READ_SPECIALREG(far_el1); esr = frame->tf_esr; print_registers(frame); printf(" far: %16lx\n", far); printf(" esr: %.8lx\n", esr); panic("Unhandled exception"); } Index: head/sys/dev/hwpmc/hwpmc_arm64.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_arm64.c (revision 354176) +++ head/sys/dev/hwpmc/hwpmc_arm64.c (revision 354177) @@ -1,544 +1,544 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by the University of Cambridge Computer * Laboratory with support from ARM Ltd. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int arm64_npmcs; struct arm64_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; /* * Per-processor information. */ struct arm64_cpu { struct pmc_hw *pc_arm64pmcs; }; static struct arm64_cpu **arm64_pcpu; /* * Interrupt Enable Set Register */ static __inline void arm64_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); - WRITE_SPECIALREG(PMINTENSET_EL1, reg); + WRITE_SPECIALREG(pmintenset_el1, reg); isb(); } /* * Interrupt Clear Set Register */ static __inline void arm64_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); - WRITE_SPECIALREG(PMINTENCLR_EL1, reg); + WRITE_SPECIALREG(pmintenclr_el1, reg); isb(); } /* * Counter Set Enable Register */ static __inline void arm64_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); - WRITE_SPECIALREG(PMCNTENSET_EL0, reg); + WRITE_SPECIALREG(pmcntenset_el0, reg); isb(); } /* * Counter Clear Enable Register */ static __inline void arm64_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); - WRITE_SPECIALREG(PMCNTENCLR_EL0, reg); + WRITE_SPECIALREG(pmcntenclr_el0, reg); isb(); } /* * Performance Monitors Control Register */ static uint32_t arm64_pmcr_read(void) { uint32_t reg; - reg = READ_SPECIALREG(PMCR_EL0); + reg = READ_SPECIALREG(pmcr_el0); return (reg); } static void arm64_pmcr_write(uint32_t reg) { - WRITE_SPECIALREG(PMCR_EL0, reg); + WRITE_SPECIALREG(pmcr_el0, reg); isb(); } /* * Performance Count Register N */ static uint32_t arm64_pmcn_read(unsigned int pmc) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); - WRITE_SPECIALREG(PMSELR_EL0, pmc); + WRITE_SPECIALREG(pmselr_el0, pmc); isb(); - return (READ_SPECIALREG(PMXEVCNTR_EL0)); + return (READ_SPECIALREG(pmxevcntr_el0)); } static void arm64_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); - WRITE_SPECIALREG(PMSELR_EL0, pmc); - WRITE_SPECIALREG(PMXEVCNTR_EL0, reg); + WRITE_SPECIALREG(pmselr_el0, pmc); + WRITE_SPECIALREG(pmxevcntr_el0, reg); isb(); } static int arm64_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t caps, config; struct arm64_cpu *pac; enum pmc_event pe; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pac = arm64_pcpu[cpu]; caps = a->pm_caps; if (a->pm_class != PMC_CLASS_ARMV8) { return (EINVAL); } pe = a->pm_ev; config = (pe & EVENT_ID_MASK); pm->pm_md.pm_arm64.pm_arm64_evsel = config; PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int arm64_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; tmp = arm64_pmcn_read(ri); PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); else *v = tmp; return 0; } static int arm64_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v); arm64_pmcn_write(ri, v); return 0; } static int arm64_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int arm64_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_arm64.pm_arm64_evsel; /* * Configure the event selection. */ - WRITE_SPECIALREG(PMSELR_EL0, ri); - WRITE_SPECIALREG(PMXEVTYPER_EL0, config); + WRITE_SPECIALREG(pmselr_el0, ri); + WRITE_SPECIALREG(pmxevtyper_el0, config); isb(); /* * Enable the PMC. */ arm64_interrupt_enable(ri); arm64_counter_enable(ri); return 0; } static int arm64_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. */ arm64_counter_disable(ri); arm64_interrupt_disable(ri); return 0; } static int arm64_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int arm64_intr(struct trapframe *tf) { struct arm64_cpu *pc; int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; pc = arm64_pcpu[cpu]; for (ri = 0; ri < arm64_npmcs; ri++) { pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (pm == NULL) continue; if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; /* Check if counter is overflowed */ reg = (1 << ri); - if ((READ_SPECIALREG(PMOVSCLR_EL0) & reg) == 0) + if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0) continue; /* Clear Overflow Flag */ - WRITE_SPECIALREG(PMOVSCLR_EL0, reg); + WRITE_SPECIALREG(pmovsclr_el0, reg); isb(); retval = 1; /* Found an interrupting PMC. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) arm64_stop_pmc(cpu, ri); /* Reload sampling count */ arm64_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { char arm64_name[PMC_NAME_MAX]; struct pmc_hw *phw; int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] row-index %d out of range", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; snprintf(arm64_name, sizeof(arm64_name), "ARMV8-%d", ri); if ((error = copystr(arm64_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_ARMV8; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int arm64_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; return (0); } /* * XXX don't know what we should do here. */ static int arm64_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_pcpu_init(struct pmc_mdep *md, int cpu) { struct arm64_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint64_t pmcr; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "arm64-init cpu=%d", cpu); arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC, M_WAITOK | M_ZERO); pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs, M_PMC, M_WAITOK | M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri; KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* Enable unit */ pmcr = arm64_pmcr_read(); pmcr |= PMCR_E; arm64_pmcr_write(pmcr); return (0); } static int arm64_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmcr; pmcr = arm64_pmcr_read(); pmcr &= ~PMCR_E; arm64_pmcr_write(pmcr); return (0); } struct pmc_mdep * pmc_arm64_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode; int reg; reg = arm64_pmcr_read(); arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT; idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch (idcode) { case PMCR_IDCODE_CORTEX_A57: case PMCR_IDCODE_CORTEX_A72: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57; break; default: case PMCR_IDCODE_CORTEX_A53: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8]; pcd->pcd_caps = ARMV8_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV8; pcd->pcd_num = arm64_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = arm64_allocate_pmc; pcd->pcd_config_pmc = arm64_config_pmc; pcd->pcd_pcpu_fini = arm64_pcpu_fini; pcd->pcd_pcpu_init = arm64_pcpu_init; pcd->pcd_describe = arm64_describe; pcd->pcd_get_config = arm64_get_config; pcd->pcd_read_pmc = arm64_read_pmc; pcd->pcd_release_pmc = arm64_release_pmc; pcd->pcd_start_pmc = arm64_start_pmc; pcd->pcd_stop_pmc = arm64_stop_pmc; pcd->pcd_write_pmc = arm64_write_pmc; pmc_mdep->pmd_intr = arm64_intr; pmc_mdep->pmd_switch_in = arm64_switch_in; pmc_mdep->pmd_switch_out = arm64_switch_out; pmc_mdep->pmd_npmc += arm64_npmcs; return (pmc_mdep); } void pmc_arm64_finalize(struct pmc_mdep *md) { }