diff --git a/sys/arm64/arm64/db_trace.c b/sys/arm64/arm64/db_trace.c index 2b47ae2a89c7..6abdd6c5ca7a 100644 --- a/sys/arm64/arm64/db_trace.c +++ b/sys/arm64/arm64/db_trace.c @@ -1,163 +1,163 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #define FRAME_NORMAL 0 #define FRAME_SYNC 1 #define FRAME_IRQ 2 #define FRAME_SERROR 3 #define FRAME_UNHANDLED 4 void db_md_list_watchpoints(void) { dbg_show_watchpoint(); } static void db_stack_trace_cmd(struct thread *td, struct unwind_state *frame) { c_db_sym_t sym; const char *name; db_expr_t value; db_expr_t offset; int frame_type; while (1) { sym = db_search_symbol(frame->pc, DB_STGY_ANY, &offset); if (sym == C_DB_SYM_NULL) { value = 0; name = "(null)"; } else db_symbol_values(sym, &name, &value); db_printf("%s() at ", name); db_printsym(frame->pc, DB_STGY_PROC); db_printf("\n"); if (strcmp(name, "handle_el0_sync") == 0 || strcmp(name, "handle_el1h_sync") == 0) frame_type = FRAME_SYNC; else if (strcmp(name, "handle_el0_irq") == 0 || strcmp(name, "handle_el1h_irq") == 0) frame_type = FRAME_IRQ; else if (strcmp(name, "handle_serror") == 0) frame_type = FRAME_SERROR; else if (strcmp(name, "handle_empty_exception") == 0) frame_type = FRAME_UNHANDLED; else frame_type = FRAME_NORMAL; if (frame_type != FRAME_NORMAL) { struct trapframe *tf; tf = (struct trapframe *)(uintptr_t)frame->fp - 1; if (!kstack_contains(td, (vm_offset_t)tf, sizeof(*tf))) { db_printf("--- invalid trapframe %p\n", tf); break; } switch (frame_type) { case FRAME_SYNC: - db_printf("--- exception, esr %#x\n", + db_printf("--- exception, esr %#lx\n", tf->tf_esr); break; case FRAME_IRQ: db_printf("--- interrupt\n"); break; case FRAME_SERROR: - db_printf("--- system error, esr %#x\n", + db_printf("--- system error, esr %#lx\n", tf->tf_esr); break; case FRAME_UNHANDLED: - db_printf("--- unhandled exception, esr %#x\n", + db_printf("--- unhandled exception, esr %#lx\n", tf->tf_esr); break; default: __assert_unreachable(); break; } frame->fp = tf->tf_x[29]; frame->pc = ADDR_MAKE_CANONICAL(tf->tf_elr); if (!INKERNEL(frame->fp)) break; } else { if (strcmp(name, "fork_trampoline") == 0) break; if (!unwind_frame(td, frame)) break; } } } int db_trace_thread(struct thread *thr, int count) { struct unwind_state frame; struct pcb *ctx; if (thr != curthread) { ctx = kdb_thr_ctx(thr); frame.fp = (uintptr_t)ctx->pcb_x[PCB_FP]; frame.pc = (uintptr_t)ctx->pcb_lr; db_stack_trace_cmd(thr, &frame); } else db_trace_self(); return (0); } void db_trace_self(void) { struct unwind_state frame; frame.fp = (uintptr_t)__builtin_frame_address(0); frame.pc = (uintptr_t)db_trace_self; db_stack_trace_cmd(curthread, &frame); } diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S index fd55e1f39b58..e23a7868b56f 100644 --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -1,298 +1,298 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include __FBSDID("$FreeBSD$"); #include "assym.inc" .text /* * This is limited to 28 instructions as it's placed in the exception vector * slot that is 32 instructions long. We need one for the branch, and three * for the prologue. */ .macro save_registers_head el .if \el == 1 mov x18, sp stp x0, x1, [sp, #(TF_X - TF_SIZE - 128)]! .else stp x0, x1, [sp, #(TF_X - TF_SIZE)]! .endif stp x2, x3, [sp, #(2 * 8)] stp x4, x5, [sp, #(4 * 8)] stp x6, x7, [sp, #(6 * 8)] stp x8, x9, [sp, #(8 * 8)] stp x10, x11, [sp, #(10 * 8)] stp x12, x13, [sp, #(12 * 8)] stp x14, x15, [sp, #(14 * 8)] stp x16, x17, [sp, #(16 * 8)] stp x18, x19, [sp, #(18 * 8)] stp x20, x21, [sp, #(20 * 8)] stp x22, x23, [sp, #(22 * 8)] stp x24, x25, [sp, #(24 * 8)] stp x26, x27, [sp, #(26 * 8)] stp x28, x29, [sp, #(28 * 8)] .if \el == 0 mrs x18, sp_el0 .endif mrs x10, elr_el1 mrs x11, spsr_el1 mrs x12, esr_el1 stp x18, lr, [sp, #(TF_SP - TF_X)]! str x10, [sp, #(TF_ELR)] - stp w11, w12, [sp, #(TF_SPSR)] + stp x11, x12, [sp, #(TF_SPSR)] mrs x18, tpidr_el1 .endm .macro save_registers el .if \el == 0 #if defined(PERTHREAD_SSP) /* Load the SSP canary to sp_el0 */ ldr x1, [x18, #(PC_CURTHREAD)] add x1, x1, #(TD_MD_CANARY) msr sp_el0, x1 #endif /* Apply the SSBD (CVE-2018-3639) workaround if needed */ ldr x1, [x18, #PC_SSBD] cbz x1, 1f mov w0, #1 blr x1 1: ldr x0, [x18, #PC_CURTHREAD] bl ptrauth_exit_el0 ldr x0, [x18, #(PC_CURTHREAD)] bl dbg_monitor_enter /* Unmask debug and SError exceptions */ msr daifclr, #(DAIF_D | DAIF_A) .else /* * Unmask debug and SError exceptions. * For EL1, debug exceptions are conditionally unmasked in * do_el1h_sync(). */ msr daifclr, #(DAIF_A) .endif .endm .macro restore_registers el /* * Mask all exceptions, x18 may change in the interrupt exception * handler. */ msr daifset, #(DAIF_ALL) .if \el == 0 ldr x0, [x18, #PC_CURTHREAD] mov x1, sp bl dbg_monitor_exit ldr x0, [x18, #PC_CURTHREAD] bl ptrauth_enter_el0 /* Remove the SSBD (CVE-2018-3639) workaround if needed */ ldr x1, [x18, #PC_SSBD] cbz x1, 1f mov w0, #0 blr x1 1: .endif ldp x18, lr, [sp, #(TF_SP)] ldp x10, x11, [sp, #(TF_ELR)] .if \el == 0 msr sp_el0, x18 .endif msr spsr_el1, x11 msr elr_el1, x10 ldp x0, x1, [sp, #(TF_X + 0 * 8)] ldp x2, x3, [sp, #(TF_X + 2 * 8)] ldp x4, x5, [sp, #(TF_X + 4 * 8)] ldp x6, x7, [sp, #(TF_X + 6 * 8)] ldp x8, x9, [sp, #(TF_X + 8 * 8)] ldp x10, x11, [sp, #(TF_X + 10 * 8)] ldp x12, x13, [sp, #(TF_X + 12 * 8)] ldp x14, x15, [sp, #(TF_X + 14 * 8)] ldp x16, x17, [sp, #(TF_X + 16 * 8)] .if \el == 0 /* * We only restore the callee saved registers when returning to * userland as they may have been updated by a system call or signal. */ ldp x18, x19, [sp, #(TF_X + 18 * 8)] ldp x20, x21, [sp, #(TF_X + 20 * 8)] ldp x22, x23, [sp, #(TF_X + 22 * 8)] ldp x24, x25, [sp, #(TF_X + 24 * 8)] ldp x26, x27, [sp, #(TF_X + 26 * 8)] ldp x28, x29, [sp, #(TF_X + 28 * 8)] .else ldr x29, [sp, #(TF_X + 29 * 8)] .endif .if \el == 0 add sp, sp, #(TF_SIZE) .else mov sp, x18 mrs x18, tpidr_el1 .endif .endm .macro do_ast mrs x19, daif /* Make sure the IRQs are enabled before calling ast() */ bic x19, x19, #PSR_I 1: /* * Mask interrupts while checking the ast pending flag */ msr daifset, #(DAIF_INTR) /* Read the current thread AST mask */ ldr x1, [x18, #PC_CURTHREAD] /* Load curthread */ ldr w1, [x1, #(TD_AST)] /* Check if we have a non-zero AST mask */ cbz w1, 2f /* Restore interrupts */ msr daif, x19 /* handle the ast */ mov x0, sp bl _C_LABEL(ast) /* Re-check for new ast scheduled */ b 1b 2: .endm ENTRY(handle_el1h_sync) save_registers 1 ldr x0, [x18, #PC_CURTHREAD] mov x1, sp bl do_el1h_sync restore_registers 1 ERET END(handle_el1h_sync) ENTRY(handle_el1h_irq) save_registers 1 mov x0, sp bl intr_irq_handler restore_registers 1 ERET END(handle_el1h_irq) ENTRY(handle_el0_sync) /* * Read the fault address early. The current thread structure may * be transiently unmapped if it is part of a memory range being * promoted or demoted to/from a superpage. As this involves a * break-before-make sequence there is a short period of time where * an access will raise an exception. If this happens the fault * address will be changed to the kernel address so a later read of * far_el1 will give the wrong value. * * The earliest memory access that could trigger a fault is in a * function called by the save_registers macro so this is the latest * we can read the userspace value. */ mrs x19, far_el1 save_registers 0 ldr x0, [x18, #PC_CURTHREAD] mov x1, sp str x1, [x0, #TD_FRAME] mov x2, x19 bl do_el0_sync do_ast restore_registers 0 ERET END(handle_el0_sync) ENTRY(handle_el0_irq) save_registers 0 mov x0, sp bl intr_irq_handler do_ast restore_registers 0 ERET END(handle_el0_irq) ENTRY(handle_serror) save_registers 0 mov x0, sp 1: bl do_serror b 1b END(handle_serror) ENTRY(handle_empty_exception) save_registers 0 mov x0, sp 1: bl unhandled_exception b 1b END(handle_empty_exception) .macro vector name, el .align 7 save_registers_head \el b handle_\name dsb sy isb /* Break instruction to ensure we aren't executing code here. */ brk 0x42 .endm .macro vempty el vector empty_exception \el .endm .align 11 .globl exception_vectors exception_vectors: vempty 1 /* Synchronous EL1t */ vempty 1 /* IRQ EL1t */ vempty 1 /* FIQ EL1t */ vempty 1 /* Error EL1t */ vector el1h_sync 1 /* Synchronous EL1h */ vector el1h_irq 1 /* IRQ EL1h */ vempty 1 /* FIQ EL1h */ vector serror 1 /* Error EL1h */ vector el0_sync 0 /* Synchronous 64-bit EL0 */ vector el0_irq 0 /* IRQ 64-bit EL0 */ vempty 0 /* FIQ 64-bit EL0 */ vector serror 0 /* Error 64-bit EL0 */ vector el0_sync 0 /* Synchronous 32-bit EL0 */ vector el0_irq 0 /* IRQ 32-bit EL0 */ vempty 0 /* FIQ 32-bit EL0 */ vector serror 0 /* Error 32-bit EL0 */ diff --git a/sys/arm64/arm64/exec_machdep.c b/sys/arm64/arm64/exec_machdep.c index 27ee2f80858d..7ead30a05663 100644 --- a/sys/arm64/arm64/exec_machdep.c +++ b/sys/arm64/arm64/exec_machdep.c @@ -1,643 +1,643 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif _Static_assert(sizeof(mcontext_t) == 880, "mcontext_t size incorrect"); _Static_assert(sizeof(ucontext_t) == 960, "ucontext_t size incorrect"); _Static_assert(sizeof(siginfo_t) == 80, "siginfo_t size incorrect"); static void get_fpcontext(struct thread *td, mcontext_t *mcp); static void set_fpcontext(struct thread *td, mcontext_t *mcp); int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; regs->sp = frame->tf_sp; regs->lr = frame->tf_lr; regs->elr = frame->tf_elr; regs->spsr = frame->tf_spsr; memcpy(regs->x, frame->tf_x, sizeof(regs->x)); #ifdef COMPAT_FREEBSD32 /* * We may be called here for a 32bits process, if we're using a * 64bits debugger. If so, put PC and SPSR where it expects it. */ if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { regs->x[15] = frame->tf_elr; regs->x[16] = frame->tf_spsr; } #endif return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; frame->tf_sp = regs->sp; frame->tf_lr = regs->lr; memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); #ifdef COMPAT_FREEBSD32 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { /* * We may be called for a 32bits process if we're using * a 64bits debugger. If so, get PC and SPSR from where * it put it. */ frame->tf_elr = regs->x[15]; frame->tf_spsr &= ~PSR_SETTABLE_32; frame->tf_spsr |= regs->x[16] & PSR_SETTABLE_32; /* Don't allow userspace to ask to continue single stepping. * The SPSR.SS field doesn't exist when the EL1 is AArch32. * As the SPSR.DIT field has moved in its place don't * allow userspace to set the SPSR.SS field. */ } else #endif { frame->tf_elr = regs->elr; frame->tf_spsr &= ~PSR_SETTABLE_64; frame->tf_spsr |= regs->spsr & PSR_SETTABLE_64; /* Enable single stepping if userspace asked fot it */ if ((frame->tf_spsr & PSR_SS) != 0) { td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | MDSCR_SS); isb(); } } return (0); } int fill_fpregs(struct thread *td, struct fpreg *regs) { #ifdef VFP struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ if (td == curthread) vfp_save_state(td, pcb); } KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Called fill_fpregs while the kernel is using the VFP")); memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs, sizeof(regs->fp_q)); regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr; regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr; #else memset(regs, 0, sizeof(*regs)); #endif return (0); } int set_fpregs(struct thread *td, struct fpreg *regs) { #ifdef VFP struct pcb *pcb; pcb = td->td_pcb; KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Called set_fpregs while the kernel is using the VFP")); memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q)); pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr; pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr; #endif return (0); } int fill_dbregs(struct thread *td, struct dbreg *regs) { struct debug_monitor_state *monitor; int i; uint8_t debug_ver, nbkpts, nwtpts; memset(regs, 0, sizeof(*regs)); extract_user_id_field(ID_AA64DFR0_EL1, ID_AA64DFR0_DebugVer_SHIFT, &debug_ver); extract_user_id_field(ID_AA64DFR0_EL1, ID_AA64DFR0_BRPs_SHIFT, &nbkpts); extract_user_id_field(ID_AA64DFR0_EL1, ID_AA64DFR0_WRPs_SHIFT, &nwtpts); /* * The BRPs field contains the number of breakpoints - 1. Armv8-A * allows the hardware to provide 2-16 breakpoints so this won't * overflow an 8 bit value. The same applies to the WRPs field. */ nbkpts++; nwtpts++; regs->db_debug_ver = debug_ver; regs->db_nbkpts = nbkpts; regs->db_nwtpts = nwtpts; monitor = &td->td_pcb->pcb_dbg_regs; if ((monitor->dbg_flags & DBGMON_ENABLED) != 0) { for (i = 0; i < nbkpts; i++) { regs->db_breakregs[i].dbr_addr = monitor->dbg_bvr[i]; regs->db_breakregs[i].dbr_ctrl = monitor->dbg_bcr[i]; } for (i = 0; i < nwtpts; i++) { regs->db_watchregs[i].dbw_addr = monitor->dbg_wvr[i]; regs->db_watchregs[i].dbw_ctrl = monitor->dbg_wcr[i]; } } return (0); } int set_dbregs(struct thread *td, struct dbreg *regs) { struct debug_monitor_state *monitor; uint64_t addr; uint32_t ctrl; int i; monitor = &td->td_pcb->pcb_dbg_regs; monitor->dbg_enable_count = 0; for (i = 0; i < DBG_BRP_MAX; i++) { addr = regs->db_breakregs[i].dbr_addr; ctrl = regs->db_breakregs[i].dbr_ctrl; /* * Don't let the user set a breakpoint on a kernel or * non-canonical user address. */ if (addr >= VM_MAXUSER_ADDRESS) return (EINVAL); /* * The lowest 2 bits are ignored, so record the effective * address. */ addr = rounddown2(addr, 4); /* * Some control fields are ignored, and other bits reserved. * Only unlinked, address-matching breakpoints are supported. * * XXX: fields that appear unvalidated, such as BAS, have * constrained undefined behaviour. If the user mis-programs * these, there is no risk to the system. */ ctrl &= DBGBCR_EN | DBGBCR_PMC | DBGBCR_BAS; if ((ctrl & DBGBCR_EN) != 0) { /* Only target EL0. */ if ((ctrl & DBGBCR_PMC) != DBGBCR_PMC_EL0) return (EINVAL); monitor->dbg_enable_count++; } monitor->dbg_bvr[i] = addr; monitor->dbg_bcr[i] = ctrl; } for (i = 0; i < DBG_WRP_MAX; i++) { addr = regs->db_watchregs[i].dbw_addr; ctrl = regs->db_watchregs[i].dbw_ctrl; /* * Don't let the user set a watchpoint on a kernel or * non-canonical user address. */ if (addr >= VM_MAXUSER_ADDRESS) return (EINVAL); /* * Some control fields are ignored, and other bits reserved. * Only unlinked watchpoints are supported. */ ctrl &= DBGWCR_EN | DBGWCR_PAC | DBGWCR_LSC | DBGWCR_BAS | DBGWCR_MASK; if ((ctrl & DBGWCR_EN) != 0) { /* Only target EL0. */ if ((ctrl & DBGWCR_PAC) != DBGWCR_PAC_EL0) return (EINVAL); /* Must set at least one of the load/store bits. */ if ((ctrl & DBGWCR_LSC) == 0) return (EINVAL); /* * When specifying the address range with BAS, the MASK * field must be zero. */ if ((ctrl & DBGWCR_BAS) != DBGWCR_BAS && (ctrl & DBGWCR_MASK) != 0) return (EINVAL); monitor->dbg_enable_count++; } monitor->dbg_wvr[i] = addr; monitor->dbg_wcr[i] = ctrl; } if (monitor->dbg_enable_count > 0) monitor->dbg_flags |= DBGMON_ENABLED; return (0); } #ifdef COMPAT_FREEBSD32 int fill_regs32(struct thread *td, struct reg32 *regs) { int i; struct trapframe *tf; tf = td->td_frame; for (i = 0; i < 13; i++) regs->r[i] = tf->tf_x[i]; /* For arm32, SP is r13 and LR is r14 */ regs->r_sp = tf->tf_x[13]; regs->r_lr = tf->tf_x[14]; regs->r_pc = tf->tf_elr; regs->r_cpsr = tf->tf_spsr; return (0); } int set_regs32(struct thread *td, struct reg32 *regs) { int i; struct trapframe *tf; tf = td->td_frame; for (i = 0; i < 13; i++) tf->tf_x[i] = regs->r[i]; /* For arm 32, SP is r13 an LR is r14 */ tf->tf_x[13] = regs->r_sp; tf->tf_x[14] = regs->r_lr; tf->tf_elr = regs->r_pc; tf->tf_spsr &= ~PSR_SETTABLE_32; tf->tf_spsr |= regs->r_cpsr & PSR_SETTABLE_32; return (0); } /* XXX fill/set dbregs/fpregs are stubbed on 32-bit arm. */ int fill_fpregs32(struct thread *td, struct fpreg32 *regs) { memset(regs, 0, sizeof(*regs)); return (0); } int set_fpregs32(struct thread *td, struct fpreg32 *regs) { return (0); } int fill_dbregs32(struct thread *td, struct dbreg32 *regs) { memset(regs, 0, sizeof(*regs)); return (0); } int set_dbregs32(struct thread *td, struct dbreg32 *regs) { return (0); } #endif void exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack) { struct trapframe *tf = td->td_frame; struct pcb *pcb = td->td_pcb; memset(tf, 0, sizeof(struct trapframe)); tf->tf_x[0] = stack; tf->tf_sp = STACKALIGN(stack); tf->tf_lr = imgp->entry_addr; tf->tf_elr = imgp->entry_addr; td->td_pcb->pcb_tpidr_el0 = 0; td->td_pcb->pcb_tpidrro_el0 = 0; WRITE_SPECIALREG(tpidrro_el0, 0); WRITE_SPECIALREG(tpidr_el0, 0); #ifdef VFP vfp_reset_state(td, pcb); #endif /* * Clear debug register state. It is not applicable to the new process. */ bzero(&pcb->pcb_dbg_regs, sizeof(pcb->pcb_dbg_regs)); /* Generate new pointer authentication keys */ ptrauth_exec(td); } /* Sanity check these are the same size, they will be memcpy'd to and from */ CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == sizeof((struct gpregs *)0)->gp_x); CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == sizeof((struct reg *)0)->x); int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; if (clear_ret & GET_MC_CLEAR_RET) { mcp->mc_gpregs.gp_x[0] = 0; mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; } else { mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; mcp->mc_gpregs.gp_spsr = tf->tf_spsr; } memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); mcp->mc_gpregs.gp_sp = tf->tf_sp; mcp->mc_gpregs.gp_lr = tf->tf_lr; mcp->mc_gpregs.gp_elr = tf->tf_elr; get_fpcontext(td, mcp); return (0); } int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf = td->td_frame; - uint32_t spsr; + uint64_t spsr; spsr = mcp->mc_gpregs.gp_spsr; if ((spsr & PSR_M_MASK) != PSR_M_EL0t || (spsr & PSR_AARCH32) != 0 || (spsr & PSR_DAIF) != (td->td_frame->tf_spsr & PSR_DAIF)) return (EINVAL); memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); tf->tf_sp = mcp->mc_gpregs.gp_sp; tf->tf_lr = mcp->mc_gpregs.gp_lr; tf->tf_elr = mcp->mc_gpregs.gp_elr; tf->tf_spsr = mcp->mc_gpregs.gp_spsr; if ((tf->tf_spsr & PSR_SS) != 0) { td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | MDSCR_SS); isb(); } set_fpcontext(td, mcp); return (0); } static void get_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef VFP struct pcb *curpcb; MPASS(td == curthread); curpcb = curthread->td_pcb; if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running VFP instructions we will * need to save the state to memcpy it below. */ vfp_save_state(td, curpcb); } KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, ("Called get_fpcontext while the kernel is using the VFP")); KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Non-userspace FPU flags set in get_fpcontext")); memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs, sizeof(mcp->mc_fpregs.fp_q)); mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr; mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr; mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; mcp->mc_flags |= _MC_FP_VALID; #endif } static void set_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef VFP struct pcb *curpcb; MPASS(td == curthread); if ((mcp->mc_flags & _MC_FP_VALID) != 0) { curpcb = curthread->td_pcb; /* * Discard any vfp state for the current thread, we * are about to override it. */ critical_enter(); vfp_discard(td); critical_exit(); KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, ("Called set_fpcontext while the kernel is using the VFP")); memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q, sizeof(mcp->mc_fpregs.fp_q)); curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr; curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr; curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK; } #endif } int sys_sigreturn(struct thread *td, struct sigreturn_args *uap) { ucontext_t uc; int error; if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); error = set_mcontext(td, &uc.uc_mcontext); if (error != 0) return (error); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; int onstack, sig; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else { fp = (struct sigframe *)td->td_frame->tf_sp; } /* Make room, keeping the stack aligned */ fp--; fp = (struct sigframe *)STACKALIGN(fp); /* Fill in the frame to copy out */ bzero(&frame, sizeof(frame)); get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack = td->td_sigstk; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ? (onstack ? SS_ONSTACK : 0) : SS_DISABLE; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } tf->tf_x[0] = sig; tf->tf_x[1] = (register_t)&fp->sf_si; tf->tf_x[2] = (register_t)&fp->sf_uc; tf->tf_x[8] = (register_t)catcher; tf->tf_sp = (register_t)fp; tf->tf_elr = (register_t)PROC_SIGCODE(p); /* Clear the single step flag while in the signal handler */ if ((td->td_pcb->pcb_flags & PCB_SINGLE_STEP) != 0) { td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS); isb(); } CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, tf->tf_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c index 1b33d7aa60c4..50f6e9de874b 100644 --- a/sys/arm64/arm64/trap.c +++ b/sys/arm64/arm64/trap.c @@ -1,735 +1,735 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include #endif #ifdef VFP #include #endif #ifdef KDB #include #endif #ifdef DDB #include #include #endif /* Called from exception.S */ void do_el1h_sync(struct thread *, struct trapframe *); void do_el0_sync(struct thread *, struct trapframe *, uint64_t far); void do_el0_error(struct trapframe *); void do_serror(struct trapframe *); void unhandled_exception(struct trapframe *); static void print_gp_register(const char *name, uint64_t value); static void print_registers(struct trapframe *frame); int (*dtrace_invop_jump_addr)(struct trapframe *); typedef void (abort_handler)(struct thread *, struct trapframe *, uint64_t, uint64_t, int); static abort_handler align_abort; static abort_handler data_abort; static abort_handler external_abort; static abort_handler *abort_handlers[] = { [ISS_DATA_DFSC_TF_L0] = data_abort, [ISS_DATA_DFSC_TF_L1] = data_abort, [ISS_DATA_DFSC_TF_L2] = data_abort, [ISS_DATA_DFSC_TF_L3] = data_abort, [ISS_DATA_DFSC_AFF_L1] = data_abort, [ISS_DATA_DFSC_AFF_L2] = data_abort, [ISS_DATA_DFSC_AFF_L3] = data_abort, [ISS_DATA_DFSC_PF_L1] = data_abort, [ISS_DATA_DFSC_PF_L2] = data_abort, [ISS_DATA_DFSC_PF_L3] = data_abort, [ISS_DATA_DFSC_ALIGN] = align_abort, [ISS_DATA_DFSC_EXT] = external_abort, [ISS_DATA_DFSC_EXT_L0] = external_abort, [ISS_DATA_DFSC_EXT_L1] = external_abort, [ISS_DATA_DFSC_EXT_L2] = external_abort, [ISS_DATA_DFSC_EXT_L3] = external_abort, [ISS_DATA_DFSC_ECC] = external_abort, [ISS_DATA_DFSC_ECC_L0] = external_abort, [ISS_DATA_DFSC_ECC_L1] = external_abort, [ISS_DATA_DFSC_ECC_L2] = external_abort, [ISS_DATA_DFSC_ECC_L3] = external_abort, }; static __inline void call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = addr; ksi.ksi_trapno = trapno; trapsignal(td, &ksi); } int cpu_fetch_syscall_args(struct thread *td) { struct proc *p; syscallarg_t *ap, *dst_ap; struct syscall_args *sa; p = td->td_proc; sa = &td->td_sa; ap = td->td_frame->tf_x; dst_ap = &sa->args[0]; sa->code = td->td_frame->tf_x[8]; sa->original_code = sa->code; if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) { sa->code = *ap++; } else { *dst_ap++ = *ap++; } if (__predict_false(sa->code >= p->p_sysent->sv_size)) sa->callp = &p->p_sysent->sv_table[0]; else sa->callp = &p->p_sysent->sv_table[sa->code]; KASSERT(sa->callp->sy_narg <= nitems(sa->args), ("Syscall %d takes too many arguments", sa->code)); memcpy(dst_ap, ap, (nitems(sa->args) - 1) * sizeof(*dst_ap)); td->td_retval[0] = 0; td->td_retval[1] = 0; return (0); } #include "../../kern/subr_syscall.c" /* * Test for fault generated by given access instruction in * bus_peek_ or bus_poke_ bus function. */ extern uint32_t generic_bs_peek_1f, generic_bs_peek_2f; extern uint32_t generic_bs_peek_4f, generic_bs_peek_8f; extern uint32_t generic_bs_poke_1f, generic_bs_poke_2f; extern uint32_t generic_bs_poke_4f, generic_bs_poke_8f; static bool test_bs_fault(void *addr) { return (addr == &generic_bs_peek_1f || addr == &generic_bs_peek_2f || addr == &generic_bs_peek_4f || addr == &generic_bs_peek_8f || addr == &generic_bs_poke_1f || addr == &generic_bs_poke_2f || addr == &generic_bs_poke_4f || addr == &generic_bs_poke_8f); } static void svc_handler(struct thread *td, struct trapframe *frame) { if ((frame->tf_esr & ESR_ELx_ISS_MASK) == 0) { syscallenter(td); syscallret(td); } else { call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr, ESR_ELx_EXCEPTION(frame->tf_esr)); userret(td, frame); } } static void align_abort(struct thread *td, struct trapframe *frame, uint64_t esr, uint64_t far, int lower) { if (!lower) { print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("Misaligned access from kernel space!"); } call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr, ESR_ELx_EXCEPTION(frame->tf_esr)); userret(td, frame); } static void external_abort(struct thread *td, struct trapframe *frame, uint64_t esr, uint64_t far, int lower) { /* * Try to handle synchronous external aborts caused by * bus_space_peek() and/or bus_space_poke() functions. */ if (!lower && test_bs_fault((void *)frame->tf_elr)) { frame->tf_elr = (uint64_t)generic_bs_fault; return; } print_registers(frame); print_gp_register("far", far); panic("Unhandled EL%d external data abort", lower ? 0: 1); } /* * It is unsafe to access the stack canary value stored in "td" until * kernel map translation faults are handled, see the pmap_klookup() call below. * Thus, stack-smashing detection with per-thread canaries must be disabled in * this function. */ static void NO_PERTHREAD_SSP data_abort(struct thread *td, struct trapframe *frame, uint64_t esr, uint64_t far, int lower) { struct vm_map *map; struct pcb *pcb; vm_prot_t ftype; int error, sig, ucode; #ifdef KDB bool handled; #endif /* * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive * and Store-Exclusive instruction usage restrictions", state * of the exclusive monitors after data abort exception is unknown. */ clrex(); #ifdef KDB if (kdb_active) { kdb_reenter(); return; } #endif if (lower) { map = &td->td_proc->p_vmspace->vm_map; } else if (!ADDR_IS_CANONICAL(far)) { /* We received a TBI/PAC/etc. fault from the kernel */ error = KERN_INVALID_ADDRESS; goto bad_far; } else if (ADDR_IS_KERNEL(far)) { /* * Handle a special case: the data abort was caused by accessing * a thread structure while its mapping was being promoted or * demoted, as a consequence of the break-before-make rule. It * is not safe to enable interrupts or dereference "td" before * this case is handled. * * In principle, if pmap_klookup() fails, there is no need to * call pmap_fault() below, but avoiding that call is not worth * the effort. */ if (ESR_ELx_EXCEPTION(esr) == EXCP_DATA_ABORT) { switch (esr & ISS_DATA_DFSC_MASK) { case ISS_DATA_DFSC_TF_L0: case ISS_DATA_DFSC_TF_L1: case ISS_DATA_DFSC_TF_L2: case ISS_DATA_DFSC_TF_L3: if (pmap_klookup(far, NULL)) return; break; } } intr_enable(); map = kernel_map; } else { intr_enable(); map = &td->td_proc->p_vmspace->vm_map; if (map == NULL) map = kernel_map; } pcb = td->td_pcb; /* * Try to handle translation, access flag, and permission faults. * Translation faults may occur as a result of the required * break-before-make sequence used when promoting or demoting * superpages. Such faults must not occur while holding the pmap lock, * or pmap_fault() will recurse on that lock. */ if ((lower || map == kernel_map || pcb->pcb_onfault != 0) && pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) return; KASSERT(td->td_md.md_spinlock_count == 0, ("data abort with spinlock held")); if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) { print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("data abort in critical section or under mutex"); } switch (ESR_ELx_EXCEPTION(esr)) { case EXCP_INSN_ABORT: case EXCP_INSN_ABORT_L: ftype = VM_PROT_EXECUTE; break; default: /* * If the exception was because of a read or cache operation * pass a read fault type into the vm code. Cache operations * need read permission but will set the WnR flag when the * memory is unmapped. */ if ((esr & ISS_DATA_WnR) == 0 || (esr & ISS_DATA_CM) != 0) ftype = VM_PROT_READ; else ftype = VM_PROT_WRITE; break; } /* Fault in the page. */ error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode); if (error != KERN_SUCCESS) { if (lower) { call_trapsignal(td, sig, ucode, (void *)far, ESR_ELx_EXCEPTION(esr)); } else { bad_far: if (td->td_intr_nesting_level == 0 && pcb->pcb_onfault != 0) { frame->tf_x[0] = error; frame->tf_elr = pcb->pcb_onfault; return; } printf("Fatal data abort:\n"); print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); #ifdef KDB if (debugger_on_trap) { kdb_why = KDB_WHY_TRAP; handled = kdb_trap(ESR_ELx_EXCEPTION(esr), 0, frame); kdb_why = KDB_WHY_UNSET; if (handled) return; } #endif panic("vm_fault failed: %lx error %d", frame->tf_elr, error); } } if (lower) userret(td, frame); } static void print_gp_register(const char *name, uint64_t value) { #if defined(DDB) c_db_sym_t sym; const char *sym_name; db_expr_t sym_value; db_expr_t offset; #endif printf(" %s: %16lx", name, value); #if defined(DDB) /* If this looks like a kernel address try to find the symbol */ if (value >= VM_MIN_KERNEL_ADDRESS) { sym = db_search_symbol(value, DB_STGY_ANY, &offset); if (sym != C_DB_SYM_NULL) { db_symbol_values(sym, &sym_name, &sym_value); printf(" (%s + %lx)", sym_name, offset); } } #endif printf("\n"); } static void print_registers(struct trapframe *frame) { char name[4]; u_int reg; for (reg = 0; reg < nitems(frame->tf_x); reg++) { snprintf(name, sizeof(name), "%sx%d", (reg < 10) ? " " : "", reg); print_gp_register(name, frame->tf_x[reg]); } printf(" sp: %16lx\n", frame->tf_sp); print_gp_register(" lr", frame->tf_lr); print_gp_register("elr", frame->tf_elr); - printf("spsr: %8x\n", frame->tf_spsr); + printf("spsr: %16lx\n", frame->tf_spsr); } #ifdef VFP static void fpe_trap(struct thread *td, void *addr, uint32_t exception) { int code; code = FPE_FLTIDO; if ((exception & ISS_FP_TFV) != 0) { if ((exception & ISS_FP_IOF) != 0) code = FPE_FLTINV; else if ((exception & ISS_FP_DZF) != 0) code = FPE_FLTDIV; else if ((exception & ISS_FP_OFF) != 0) code = FPE_FLTOVF; else if ((exception & ISS_FP_UFF) != 0) code = FPE_FLTUND; else if ((exception & ISS_FP_IXF) != 0) code = FPE_FLTRES; } call_trapsignal(td, SIGFPE, code, addr, exception); } #endif /* * See the comment above data_abort(). */ void NO_PERTHREAD_SSP do_el1h_sync(struct thread *td, struct trapframe *frame) { uint32_t exception; uint64_t esr, far; int dfsc; /* Read the esr register to get the exception details */ esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); #ifdef KDTRACE_HOOKS if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) return; #endif CTR4(KTR_TRAP, "do_el1_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); /* * Enable debug exceptions if we aren't already handling one. They will * be masked again in the exception handler's epilogue. */ if (exception != EXCP_BRK && exception != EXCP_WATCHPT_EL1 && exception != EXCP_SOFTSTP_EL1) dbg_enable(); switch (exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: #ifdef VFP if ((td->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) { vfp_restore_state(); } else #endif { print_registers(frame); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("VFP exception in the kernel"); } break; case EXCP_INSN_ABORT: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); dfsc = esr & ISS_DATA_DFSC_MASK; if (dfsc < nitems(abort_handlers) && abort_handlers[dfsc] != NULL) { abort_handlers[dfsc](td, frame, esr, far, 0); } else { print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("Unhandled EL1 %s abort: %x", exception == EXCP_INSN_ABORT ? "instruction" : "data", dfsc); } break; case EXCP_BRK: #ifdef KDTRACE_HOOKS if ((esr & ESR_ELx_ISS_MASK) == 0x40d && \ dtrace_invop_jump_addr != 0) { dtrace_invop_jump_addr(frame); break; } #endif #ifdef KDB kdb_trap(exception, 0, frame); #else panic("No debugger in kernel."); #endif break; case EXCP_WATCHPT_EL1: case EXCP_SOFTSTP_EL1: #ifdef KDB kdb_trap(exception, 0, frame); #else panic("No debugger in kernel."); #endif break; case EXCP_FPAC: /* We can see this if the authentication on PAC fails */ print_registers(frame); printf(" far: %16lx\n", READ_SPECIALREG(far_el1)); panic("FPAC kernel exception"); break; case EXCP_UNKNOWN: if (undef_insn(1, frame)) break; printf("Undefined instruction: %08x\n", *(uint32_t *)frame->tf_elr); /* FALLTHROUGH */ default: print_registers(frame); print_gp_register("far", READ_SPECIALREG(far_el1)); panic("Unknown kernel exception %x esr_el1 %lx", exception, esr); } } void do_el0_sync(struct thread *td, struct trapframe *frame, uint64_t far) { pcpu_bp_harden bp_harden; uint32_t exception; uint64_t esr; int dfsc; /* Check we have a sane environment when entering from userland */ KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS, ("Invalid pcpu address from userland: %p (tpidr %lx)", get_pcpu(), READ_SPECIALREG(tpidr_el1))); esr = frame->tf_esr; exception = ESR_ELx_EXCEPTION(esr); if (exception == EXCP_INSN_ABORT_L && far > VM_MAXUSER_ADDRESS) { /* * Userspace may be trying to train the branch predictor to * attack the kernel. If we are on a CPU affected by this * call the handler to clear the branch predictor state. */ bp_harden = PCPU_GET(bp_harden); if (bp_harden != NULL) bp_harden(); } intr_enable(); CTR4(KTR_TRAP, "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p", td, esr, frame->tf_elr, frame); switch (exception) { case EXCP_FP_SIMD: #ifdef VFP vfp_restore_state(); #else panic("VFP exception in userland"); #endif break; case EXCP_TRAP_FP: #ifdef VFP fpe_trap(td, (void *)frame->tf_elr, esr); userret(td, frame); #else panic("VFP exception in userland"); #endif break; case EXCP_SVE: call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_elr, exception); userret(td, frame); break; case EXCP_SVC32: case EXCP_SVC64: svc_handler(td, frame); break; case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: dfsc = esr & ISS_DATA_DFSC_MASK; if (dfsc < nitems(abort_handlers) && abort_handlers[dfsc] != NULL) abort_handlers[dfsc](td, frame, esr, far, 1); else { print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("Unhandled EL0 %s abort: %x", exception == EXCP_INSN_ABORT_L ? "instruction" : "data", dfsc); } break; case EXCP_UNKNOWN: if (!undef_insn(0, frame)) call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)far, exception); userret(td, frame); break; case EXCP_FPAC: call_trapsignal(td, SIGILL, ILL_ILLOPN, (void *)frame->tf_elr, exception); userret(td, frame); break; case EXCP_SP_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sp, exception); userret(td, frame); break; case EXCP_PC_ALIGN: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_elr, exception); userret(td, frame); break; case EXCP_BRKPT_EL0: case EXCP_BRK: #ifdef COMPAT_FREEBSD32 case EXCP_BRKPT_32: #endif /* COMPAT_FREEBSD32 */ call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_elr, exception); userret(td, frame); break; case EXCP_WATCHPT_EL0: call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)far, exception); userret(td, frame); break; case EXCP_MSR: /* * The CPU can raise EXCP_MSR when userspace executes an mrs * instruction to access a special register userspace doesn't * have access to. */ if (!undef_insn(0, frame)) call_trapsignal(td, SIGILL, ILL_PRVOPC, (void *)frame->tf_elr, exception); userret(td, frame); break; case EXCP_SOFTSTP_EL0: PROC_LOCK(td->td_proc); if ((td->td_dbgflags & TDB_STEP) != 0) { td->td_frame->tf_spsr &= ~PSR_SS; td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~MDSCR_SS); } PROC_UNLOCK(td->td_proc); call_trapsignal(td, SIGTRAP, TRAP_TRACE, (void *)frame->tf_elr, exception); userret(td, frame); break; default: call_trapsignal(td, SIGBUS, BUS_OBJERR, (void *)frame->tf_elr, exception); userret(td, frame); break; } KASSERT((td->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Kernel VFP flags set while entering userspace")); KASSERT( td->td_pcb->pcb_fpusaved == &td->td_pcb->pcb_fpustate, ("Kernel VFP state in use when entering userspace")); } /* * TODO: We will need to handle these later when we support ARMv8.2 RAS. */ void do_serror(struct trapframe *frame) { uint64_t esr, far; far = READ_SPECIALREG(far_el1); esr = frame->tf_esr; print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("Unhandled System Error"); } void unhandled_exception(struct trapframe *frame) { uint64_t esr, far; far = READ_SPECIALREG(far_el1); esr = frame->tf_esr; print_registers(frame); print_gp_register("far", far); - printf(" esr: %.8lx\n", esr); + printf(" esr: %.16lx\n", esr); panic("Unhandled exception"); } diff --git a/sys/arm64/include/frame.h b/sys/arm64/include/frame.h index 0a8b53ebb01e..91ed6dbce920 100644 --- a/sys/arm64/include/frame.h +++ b/sys/arm64/include/frame.h @@ -1,83 +1,84 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_FRAME_H_ #define _MACHINE_FRAME_H_ #ifndef LOCORE #include #include /* * NOTE: keep this structure in sync with struct reg and struct mcontext. */ struct trapframe { uint64_t tf_sp; uint64_t tf_lr; uint64_t tf_elr; - uint32_t tf_spsr; - uint32_t tf_esr; + uint64_t tf_spsr; + uint64_t tf_esr; + uint64_t pad; /* struct must be 16B aligned */ uint64_t tf_x[30]; }; struct arm64_frame { struct arm64_frame *f_frame; u_long f_retaddr; }; /* * Signal frame, pushed onto the user stack. */ struct sigframe { siginfo_t sf_si; /* actual saved siginfo */ ucontext_t sf_uc; /* actual saved ucontext */ }; /* * There is no fixed frame layout, other than to be 16-byte aligned. */ struct frame { int dummy; }; #ifdef COMPAT_FREEBSD32 struct sigframe32 { struct siginfo32 sf_si; ucontext32_t sf_uc; mcontext32_vfp_t sf_vfp; }; #endif /* COMPAT_FREEBSD32 */ #endif /* !LOCORE */ #endif /* !_MACHINE_FRAME_H_ */ diff --git a/sys/arm64/include/reg.h b/sys/arm64/include/reg.h index 44b2e2b21b72..4e8ca4f4e834 100644 --- a/sys/arm64/include/reg.h +++ b/sys/arm64/include/reg.h @@ -1,93 +1,93 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014-2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_REG_H_ #define _MACHINE_REG_H_ #include struct reg { __uint64_t x[30]; __uint64_t lr; __uint64_t sp; __uint64_t elr; - __uint32_t spsr; + __uint64_t spsr; }; struct reg32 { unsigned int r[13]; unsigned int r_sp; unsigned int r_lr; unsigned int r_pc; unsigned int r_cpsr; }; struct fpreg { __uint128_t fp_q[32]; __uint32_t fp_sr; __uint32_t fp_cr; }; struct fpreg32 { int dummy; }; struct dbreg { __uint8_t db_debug_ver; __uint8_t db_nbkpts; __uint8_t db_nwtpts; __uint8_t db_pad[5]; struct { __uint64_t dbr_addr; __uint32_t dbr_ctrl; __uint32_t dbr_pad; } db_breakregs[16]; struct { __uint64_t dbw_addr; __uint32_t dbw_ctrl; __uint32_t dbw_pad; } db_watchregs[16]; }; struct dbreg32 { int dummy; }; struct arm64_addr_mask { __uint64_t code; __uint64_t data; }; #define __HAVE_REG32 #endif /* !_MACHINE_REG_H_ */ diff --git a/sys/arm64/include/ucontext.h b/sys/arm64/include/ucontext.h index a81fdf9ad724..edb4cf8e63e3 100644 --- a/sys/arm64/include/ucontext.h +++ b/sys/arm64/include/ucontext.h @@ -1,89 +1,88 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014-2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_UCONTEXT_H_ #define _MACHINE_UCONTEXT_H_ struct gpregs { __register_t gp_x[30]; __register_t gp_lr; __register_t gp_sp; __register_t gp_elr; - __uint32_t gp_spsr; - int gp_pad; + __uint64_t gp_spsr; }; struct fpregs { __uint128_t fp_q[32]; __uint32_t fp_sr; __uint32_t fp_cr; int fp_flags; int fp_pad; }; struct __mcontext { struct gpregs mc_gpregs; struct fpregs mc_fpregs; int mc_flags; #define _MC_FP_VALID 0x1 /* Set when mc_fpregs has valid data */ int mc_pad; /* Padding */ __uint64_t mc_spare[8]; /* Space for expansion, set to zero */ }; typedef struct __mcontext mcontext_t; #ifdef COMPAT_FREEBSD32 #include typedef struct __mcontext32 { uint32_t mc_gregset[17]; uint32_t mc_vfp_size; uint32_t mc_vfp_ptr; uint32_t mc_spare[33]; } mcontext32_t; typedef struct __ucontext32 { sigset_t uc_sigmask; mcontext32_t uc_mcontext; u_int32_t uc_link; struct sigaltstack32 uc_stack; u_int32_t uc_flags; u_int32_t __spare__[4]; } ucontext32_t; typedef struct __mcontext32_vfp { __uint64_t mcv_reg[32]; __uint32_t mcv_fpscr; } mcontext32_vfp_t; #endif /* COMPAT_FREEBSD32 */ #endif /* !_MACHINE_UCONTEXT_H_ */