Index: sys/arm/arm/exception.S =================================================================== --- sys/arm/arm/exception.S +++ sys/arm/arm/exception.S @@ -53,6 +53,7 @@ #include #include #include +#include __FBSDID("$FreeBSD$"); @@ -68,6 +69,8 @@ .text .align 2 +#define DATA_ABORT 1 + /* * ASM macros for pushing and pulling trapframes from the stack * @@ -147,70 +150,119 @@ * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ + #if __ARM_ARCH < 6 -#define PUSHFRAMEINSVC \ - stmdb sp, {r0-r3}; /* Save 4 registers */ \ - mov r0, lr; /* Save xxx32 r14 */ \ - mov r1, sp; /* Save xxx32 sp */ \ - mrs r3, spsr; /* Save xxx32 spsr */ \ - mrs r2, cpsr; /* Get the CPSR */ \ - bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ - orr r2, r2, #(PSR_SVC32_MODE); \ - msr cpsr_c, r2; /* Punch into SVC mode */ \ - mov r2, sp; /* Save SVC sp */ \ - bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ - sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ - /* and for dtrace to emulate push/pop */ \ - str r0, [sp, #-4]!; /* Push return address */ \ - str lr, [sp, #-4]!; /* Push SVC lr */ \ - str r2, [sp, #-4]!; /* Push SVC sp */ \ - msr spsr_fsxc, r3; /* Restore correct spsr */ \ - ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ - sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ - stmia sp, {r0-r12}; /* Push the user mode registers */ \ - add r0, sp, #(4*13); /* Adjust the stack pointer */ \ - stmia r0, {r13-r14}^; /* Push the user mode registers */ \ - mov r0, r0; /* NOP for previous instruction */ \ - ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ \ - ldr r4, [r5, #4]; /* reset it to point at the */ \ - cmp r4, #0xffffffff; /* end of memory if necessary; */ \ - movne r1, #0xffffffff; /* leave value in r4 for later */ \ - strne r1, [r5, #4]; /* comparision against PC. */ \ - ldr r3, [r5]; /* Retrieve global RAS_START */ \ - cmp r3, #0; /* and reset it if non-zero. */ \ - movne r1, #0; /* If non-zero RAS_START and */ \ - strne r1, [r5]; /* PC was lower than RAS_END, */ \ - ldrne r1, [r0, #16]; /* adjust the saved PC so that */ \ - cmpne r4, r1; /* execution later resumes at */ \ - strhi r3, [r0, #16]; /* the RAS_START location. */ \ - mrs r0, spsr; \ +.macro PUSHFRAMEINSVC gp + stmdb sp, {r0-r5}; /* Save 6 registers */ +.if \gp == DATA_ABORT + mrc p15, 0, r4, c6, c0, 0; /* Read DFAR */ + lsr r4, r4, #(PAGE_SHIFT); +.endif + mov r0, lr; /* Save xxx32 r14 */ + mov r1, sp; /* Save xxx32 sp */ + mrs r3, spsr; /* Save xxx32 spsr */ + mrs r2, cpsr; /* Get the CPSR */ + bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ + orr r2, r2, #(PSR_SVC32_MODE); + msr cpsr_c, r2; /* Punch into SVC mode */ + mov r2, sp; /* Save SVC sp */ +.if \gp == DATA_ABORT + mov r5, r3; /* If we were in usermode, skip guard page checking */ + and r5, r5, #(PSR_MODE); + teq r5, #(PSR_USR32_MODE); + beq 1f; + lsr r5, sp, #(PAGE_SHIFT); /* Check if SVC SP is in page that faulted */ + cmp r4, r5; + bne 1f; + sub r5, sp, #(20*4); /* Check if trapframe falls into faulted page */ + lsr r5, r5, #(PAGE_SHIFT); + cmp r4, r5; + bne 1f; + ldr sp, .Laux_rescue_stack; /* + Kernel SP-trapframe is inside faulted page. + Use special stack instead + */ + add sp, sp, #(AUX_RESCUE_STACK_SIZE); + 1: +.endif + bic sp, sp, #7; /* Align sp to an 8-byte addrress */ + sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ + /* and for dtrace to emulate push/pop */ + str r0, [sp, #-4]!; /* Push return address */ + str lr, [sp, #-4]!; /* Push SVC lr */ + str r2, [sp, #-4]!; /* Push SVC sp */ + msr spsr_fsxc, r3; /* Restore correct spsr */ + ldmdb r1, {r0-r5}; /* Restore 6 regs from xxx mode */ + sub sp, sp, #(4*15); /* Adjust the stack pointer */ + stmia sp, {r0-r12}; /* Push the user mode registers */ + add r0, sp, #(4*13); /* Adjust the stack pointer */ + stmia r0, {r13-r14}^; /* Push the user mode registers */ + mov r0, r0; /* NOP for previous instruction */ + ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ + ldr r4, [r5, #4]; /* reset it to point at the */ + cmp r4, #0xffffffff; /* end of memory if necessary; */ + movne r1, #0xffffffff; /* leave value in r4 for later */ + strne r1, [r5, #4]; /* comparision against PC. */ + ldr r3, [r5]; /* Retrieve global RAS_START */ + cmp r3, #0; /* and reset it if non-zero. */ + movne r1, #0; /* If non-zero RAS_START and */ + strne r1, [r5]; /* PC was lower than RAS_END, */ + ldrne r1, [r0, #16]; /* adjust the saved PC so that */ + cmpne r4, r1; /* execution later resumes at */ + strhi r3, [r0, #16]; /* the RAS_START location. */ + mrs r0, spsr; str r0, [sp, #-4]! +.endm #else -#define PUSHFRAMEINSVC \ - stmdb sp, {r0-r3}; /* Save 4 registers */ \ - mov r0, lr; /* Save xxx32 r14 */ \ - mov r1, sp; /* Save xxx32 sp */ \ - mrs r3, spsr; /* Save xxx32 spsr */ \ - mrs r2, cpsr; /* Get the CPSR */ \ - bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ - orr r2, r2, #(PSR_SVC32_MODE); \ - msr cpsr_c, r2; /* Punch into SVC mode */ \ - mov r2, sp; /* Save SVC sp */ \ - bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ - sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ - /* and for dtrace to emulate push/pop */ \ - str r0, [sp, #-4]!; /* Push return address */ \ - str lr, [sp, #-4]!; /* Push SVC lr */ \ - str r2, [sp, #-4]!; /* Push SVC sp */ \ - msr spsr_fsxc, r3; /* Restore correct spsr */ \ - ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ - sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ - stmia sp, {r0-r12}; /* Push the user mode registers */ \ - add r0, sp, #(4*13); /* Adjust the stack pointer */ \ - stmia r0, {r13-r14}^; /* Push the user mode registers */ \ - mov r0, r0; /* NOP for previous instruction */ \ - mrs r0, spsr; /* Put the SPSR on the stack */ \ +.macro PUSHFRAMEINSVC gp + stmdb sp, {r0-r5}; /* Save 6 registers */ +.if \gp == DATA_ABORT + mrc p15, 0, r4, c6, c0, 0; /* Read DFAR */ + lsr r4, r4, #(PAGE_SHIFT); +.endif + mov r0, lr; /* Save xxx32 r14 */ + mov r1, sp; /* Save xxx32 sp */ + mrs r3, spsr; /* Save xxx32 spsr */ + mrs r2, cpsr; /* Get the CPSR */ + bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ + orr r2, r2, #(PSR_SVC32_MODE); + msr cpsr_c, r2; /* Punch into SVC mode */ + mov r2, sp; /* Save SVC sp */ +.if \gp == DATA_ABORT + mov r5, r3; /* If we were in usermode, skip guard page checking */ + and r5, r5, #(PSR_MODE); + teq r5, #(PSR_USR32_MODE); + beq 1f; + lsr r5, sp, #(PAGE_SHIFT); /* Check if SVC SP is in page that faulted */ + cmp r4, r5; + bne 1f; + sub r5, sp, #(20*4); /* Check if trapframe falls into faulted page */ + lsr r5, r5, #(PAGE_SHIFT); + cmp r4, r5; + bne 1f; + ldr sp, .Laux_rescue_stack; /* + Kernel SP-trapframe is inside faulted page. + Use special stack instead + */ + add sp, sp, #(AUX_RESCUE_STACK_SIZE); + 1: +.endif + bic sp, sp, #7; /* Align sp to an 8-byte addrress */ + sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ + /* and for dtrace to emulate push/pop */ + str r0, [sp, #-4]!; /* Push return address */ + str lr, [sp, #-4]!; /* Push SVC lr */ + str r2, [sp, #-4]!; /* Push SVC sp */ + msr spsr_fsxc, r3; /* Restore correct spsr */ + ldmdb r1, {r0-r5}; /* Restore 6 regs from xxx mode */ + sub sp, sp, #(4*15); /* Adjust the stack pointer */ + stmia sp, {r0-r12}; /* Push the user mode registers */ + add r0, sp, #(4*13); /* Adjust the stack pointer */ + stmia r0, {r13-r14}^; /* Push the user mode registers */ + mov r0, r0; /* NOP for previous instruction */ + mrs r0, spsr; /* Put the SPSR on the stack */ str r0, [sp, #-4]! +.endm #endif /* @@ -327,7 +379,7 @@ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ - PUSHFRAMEINSVC /* mode stack, build trapframe there. */ + PUSHFRAMEINSVC 0 /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ mov r1, #1 /* Type flag */ @@ -347,7 +399,7 @@ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #8 /* Adjust the lr. Transition to scv32 */ - PUSHFRAMEINSVC /* mode stack, build trapframe there. */ + PUSHFRAMEINSVC DATA_ABORT /* mode stack, build trapframe there. */ adr lr, exception_exit /* Exception exit routine */ mov r0, sp /* Trapframe to the handler */ mov r1, #0 /* Type flag */ @@ -362,7 +414,7 @@ * on exit (without transitioning back through the undefined mode stack). */ ASENTRY_NP(undefined_entry) - PUSHFRAMEINSVC /* mode stack, build trapframe there. */ + PUSHFRAMEINSVC 0 /* mode stack, build trapframe there. */ mov r4, r0 /* R0 contains SPSR */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. pass frame */ @@ -407,7 +459,7 @@ */ ASENTRY_NP(irq_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ - PUSHFRAMEINSVC /* mode stack, build trapframe there. */ + PUSHFRAMEINSVC 0 /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ @@ -501,4 +553,5 @@ _C_LABEL(fiq_nullhandler_size): .word 4 - +.Laux_rescue_stack: + .word _C_LABEL(__aux_rescue_stack) Index: sys/arm/arm/machdep.c =================================================================== --- sys/arm/arm/machdep.c +++ sys/arm/arm/machdep.c @@ -104,6 +104,7 @@ #include #include #include +#include #include #include #include @@ -195,6 +196,9 @@ struct pcpu __pcpu[MAXCPU]; struct pcpu *pcpup = &__pcpu[0]; +uint8_t __aux_rescue_stack[AUX_RESCUE_STACK_SIZE] + __attribute__ ((aligned (16))); + static struct trapframe proc0_tf; uint32_t cpu_reset_address = 0; int cold = 1; Index: sys/arm/arm/trap-v4.c =================================================================== --- sys/arm/arm/trap-v4.c +++ sys/arm/arm/trap-v4.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include @@ -174,6 +175,24 @@ trapsignal(td, &ksi); } +static int +is_inside_rescue_stack(void) +{ + int variable_on_stack; + + /* + * Check if we're inside a special stack. If yes, we've hit + * a guard page with the SP, i.e. the kernel thread had overflowed + * the allocated stack space. + */ + if (((uint32_t)&variable_on_stack >= (uint32_t)&__aux_rescue_stack) && + ((uint32_t)&variable_on_stack < ((uint32_t)&__aux_rescue_stack + + AUX_RESCUE_STACK_SIZE))) + return (1); + + return (0); +} + void abort_handler(struct trapframe *tf, int type) { @@ -207,6 +226,9 @@ td = curthread; p = td->td_proc; + if (is_inside_rescue_stack()) + panic("Kernel thread hit the guard page with the Stack Pointer"); + PCPU_INC(cnt.v_trap); /* Data abort came from user mode? */ user = TRAP_USERMODE(tf); Index: sys/arm/arm/trap-v6.c =================================================================== --- sys/arm/arm/trap-v6.c +++ sys/arm/arm/trap-v6.c @@ -57,6 +57,7 @@ #include #include #include +#include #include #ifdef KDB @@ -263,6 +264,24 @@ } } +static int +is_inside_rescue_stack(void) +{ + int variable_on_stack; + + /* + * Check if we're inside a special stack. If yes, we've hit + * a guard page with the SP, i.e. the kernel thread had overflowed + * the allocated stack space. + */ + if (((uint32_t)&variable_on_stack >= (uint32_t)&__aux_rescue_stack) && + ((uint32_t)&variable_on_stack < ((uint32_t)&__aux_rescue_stack + + AUX_RESCUE_STACK_SIZE))) + return (1); + + return (0); +} + /* * Abort handler. * @@ -294,6 +313,9 @@ PCPU_INC(cnt.v_trap); td = curthread; + if (is_inside_rescue_stack()) + panic("Kernel thread hit the guard page with the Stack Pointer"); + fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get(); #if __ARM_ARCH >= 7 far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get(); Index: sys/arm/include/machdep.h =================================================================== --- sys/arm/include/machdep.h +++ sys/arm/include/machdep.h @@ -48,4 +48,6 @@ int arm_predict_branch(void *, u_int, register_t, register_t *, u_int (*)(void*, int), u_int (*)(void*, vm_offset_t, u_int*)); +extern uint8_t __aux_rescue_stack[]; + #endif /* !_MACHINE_MACHDEP_H_ */ Index: sys/arm/include/param.h =================================================================== --- sys/arm/include/param.h +++ sys/arm/include/param.h @@ -108,7 +108,9 @@ #define CACHE_LINE_SIZE (1 << CACHE_LINE_SHIFT) #define PAGE_SHIFT 12 +#ifndef PAGE_SIZE #define PAGE_SIZE (1 << PAGE_SHIFT) /* Page size */ +#endif #define PAGE_MASK (PAGE_SIZE - 1) #define PDR_SHIFT 20 /* log2(NBPDR) */ @@ -129,6 +131,7 @@ #ifndef KSTACK_GUARD_PAGES #define KSTACK_GUARD_PAGES 1 #endif /* !KSTACK_GUARD_PAGES */ +#define AUX_RESCUE_STACK_SIZE PAGE_SIZE #define USPACE_SVC_STACK_TOP (kstack_pages * PAGE_SIZE)