Index: stable/11/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c =================================================================== --- stable/11/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c (revision 318571) +++ stable/11/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c (revision 318572) @@ -1,723 +1,720 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2012,2013 Justin Hibbits * * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include "regset.h" /* Offset to the LR Save word (ppc32) */ #define RETURN_OFFSET 4 /* Offset to LR Save word (ppc64). CR Save area sits between back chain and LR */ #define RETURN_OFFSET64 16 #ifdef __powerpc64__ #define OFFSET 4 /* Account for the TOC reload slot */ +#define FRAME_OFFSET 48 #else #define OFFSET 0 +#define FRAME_OFFSET 8 #endif #define INKERNEL(x) ((x) <= VM_MAX_KERNEL_ADDRESS && \ (x) >= VM_MIN_KERNEL_ADDRESS) static __inline int -dtrace_sp_inkernel(uintptr_t sp, int aframes) +dtrace_sp_inkernel(uintptr_t sp) { + struct trapframe *frame; vm_offset_t callpc; #ifdef __powerpc64__ callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); #else callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); #endif if ((callpc & 3) || (callpc < 0x100)) return (0); /* * trapexit() and asttrapexit() are sentinels * for kernel stack tracing. - * - * Special-case this for 'aframes == 0', because fbt sets aframes to the - * trap callchain depth, so we want to break out of it. */ - if ((callpc + OFFSET == (vm_offset_t) &trapexit || - callpc + OFFSET == (vm_offset_t) &asttrapexit) && - aframes != 0) - return (0); + if (callpc + OFFSET == (vm_offset_t) &trapexit || + callpc + OFFSET == (vm_offset_t) &asttrapexit) { + if (sp == 0) + return (0); + frame = (struct trapframe *)(sp + FRAME_OFFSET); + return ((frame->srr1 & PSL_PR) == 0); + } + return (1); } static __inline uintptr_t dtrace_next_sp(uintptr_t sp) { vm_offset_t callpc; + struct trapframe *frame; #ifdef __powerpc64__ callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); #else callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); #endif /* * trapexit() and asttrapexit() are sentinels * for kernel stack tracing. - * - * Special-case this for 'aframes == 0', because fbt sets aframes to the - * trap callchain depth, so we want to break out of it. */ if ((callpc + OFFSET == (vm_offset_t) &trapexit || - callpc + OFFSET == (vm_offset_t) &asttrapexit)) - /* Access the trap frame */ -#ifdef __powerpc64__ - return (*(uintptr_t *)sp + 48 + sizeof(register_t)); -#else - return (*(uintptr_t *)sp + 8 + sizeof(register_t)); -#endif + callpc + OFFSET == (vm_offset_t) &asttrapexit)) { + /* Access the trap frame */ + frame = (struct trapframe *)(sp + FRAME_OFFSET); + return (*(uintptr_t *)(frame->fixreg[1])); + } return (*(uintptr_t*)sp); } static __inline uintptr_t dtrace_get_pc(uintptr_t sp) { + struct trapframe *frame; vm_offset_t callpc; #ifdef __powerpc64__ callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); #else callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); #endif /* * trapexit() and asttrapexit() are sentinels * for kernel stack tracing. - * - * Special-case this for 'aframes == 0', because fbt sets aframes to the - * trap callchain depth, so we want to break out of it. */ if ((callpc + OFFSET == (vm_offset_t) &trapexit || - callpc + OFFSET == (vm_offset_t) &asttrapexit)) - /* Access the trap frame */ -#ifdef __powerpc64__ - return (*(uintptr_t *)sp + 48 + offsetof(struct trapframe, lr)); -#else - return (*(uintptr_t *)sp + 8 + offsetof(struct trapframe, lr)); -#endif + callpc + OFFSET == (vm_offset_t) &asttrapexit)) { + /* Access the trap frame */ + frame = (struct trapframe *)(sp + FRAME_OFFSET); + return (frame->srr0); + } return (callpc); } greg_t dtrace_getfp(void) { return (greg_t)__builtin_frame_address(0); } void dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, uint32_t *intrpc) { int depth = 0; uintptr_t osp, sp; vm_offset_t callpc; pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; osp = PAGE_SIZE; if (intrpc != 0) pcstack[depth++] = (pc_t) intrpc; aframes++; sp = dtrace_getfp(); while (depth < pcstack_limit) { if (sp <= osp) break; - if (!dtrace_sp_inkernel(sp, aframes)) + if (!dtrace_sp_inkernel(sp)) break; callpc = dtrace_get_pc(sp); if (aframes > 0) { aframes--; if ((aframes == 0) && (caller != 0)) { pcstack[depth++] = caller; } } else { pcstack[depth++] = callpc; } osp = sp; sp = dtrace_next_sp(sp); } for (; depth < pcstack_limit; depth++) { pcstack[depth] = 0; } } static int dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, uintptr_t sp) { proc_t *p = curproc; int ret = 0; ASSERT(pcstack == NULL || pcstack_limit > 0); while (pc != 0) { ret++; if (pcstack != NULL) { *pcstack++ = (uint64_t)pc; pcstack_limit--; if (pcstack_limit <= 0) break; } if (sp == 0) break; if (SV_PROC_FLAG(p, SV_ILP32)) { pc = dtrace_fuword32((void *)(sp + RETURN_OFFSET)); sp = dtrace_fuword32((void *)sp); } else { pc = dtrace_fuword64((void *)(sp + RETURN_OFFSET64)); sp = dtrace_fuword64((void *)sp); } } return (ret); } void dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) { proc_t *p = curproc; struct trapframe *tf; uintptr_t pc, sp; volatile uint16_t *flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; int n; if (*flags & CPU_DTRACE_FAULT) return; if (pcstack_limit <= 0) return; /* * If there's no user context we still need to zero the stack. */ if (p == NULL || (tf = curthread->td_frame) == NULL) goto zero; *pcstack++ = (uint64_t)p->p_pid; pcstack_limit--; if (pcstack_limit <= 0) return; pc = tf->srr0; sp = tf->fixreg[1]; if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { /* * In an entry probe. The frame pointer has not yet been * pushed (that happens in the function prologue). The * best approach is to add the current pc as a missing top * of stack and back the pc up to the caller, which is stored * at the current stack pointer address since the call * instruction puts it there right before the branch. */ *pcstack++ = (uint64_t)pc; pcstack_limit--; if (pcstack_limit <= 0) return; pc = tf->lr; } n = dtrace_getustack_common(pcstack, pcstack_limit, pc, sp); ASSERT(n >= 0); ASSERT(n <= pcstack_limit); pcstack += n; pcstack_limit -= n; zero: while (pcstack_limit-- > 0) *pcstack++ = 0; } int dtrace_getustackdepth(void) { proc_t *p = curproc; struct trapframe *tf; uintptr_t pc, sp; int n = 0; if (p == NULL || (tf = curthread->td_frame) == NULL) return (0); if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) return (-1); pc = tf->srr0; sp = tf->fixreg[1]; if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { /* * In an entry probe. The frame pointer has not yet been * pushed (that happens in the function prologue). The * best approach is to add the current pc as a missing top * of stack and back the pc up to the caller, which is stored * at the current stack pointer address since the call * instruction puts it there right before the branch. */ if (SV_PROC_FLAG(p, SV_ILP32)) { pc = dtrace_fuword32((void *) sp); } else pc = dtrace_fuword64((void *) sp); n++; } n += dtrace_getustack_common(NULL, 0, pc, sp); return (n); } void dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) { proc_t *p = curproc; struct trapframe *tf; uintptr_t pc, sp; volatile uint16_t *flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; #ifdef notyet /* XXX signal stack */ uintptr_t oldcontext; size_t s1, s2; #endif if (*flags & CPU_DTRACE_FAULT) return; if (pcstack_limit <= 0) return; /* * If there's no user context we still need to zero the stack. */ if (p == NULL || (tf = curthread->td_frame) == NULL) goto zero; *pcstack++ = (uint64_t)p->p_pid; pcstack_limit--; if (pcstack_limit <= 0) return; pc = tf->srr0; sp = tf->fixreg[1]; #ifdef notyet /* XXX signal stack */ oldcontext = lwp->lwp_oldcontext; s1 = sizeof (struct xframe) + 2 * sizeof (long); s2 = s1 + sizeof (siginfo_t); #endif if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { *pcstack++ = (uint64_t)pc; *fpstack++ = 0; pcstack_limit--; if (pcstack_limit <= 0) return; if (SV_PROC_FLAG(p, SV_ILP32)) { pc = dtrace_fuword32((void *)sp); } else { pc = dtrace_fuword64((void *)sp); } } while (pc != 0) { *pcstack++ = (uint64_t)pc; *fpstack++ = sp; pcstack_limit--; if (pcstack_limit <= 0) break; if (sp == 0) break; #ifdef notyet /* XXX signal stack */ if (oldcontext == sp + s1 || oldcontext == sp + s2) { ucontext_t *ucp = (ucontext_t *)oldcontext; greg_t *gregs = ucp->uc_mcontext.gregs; sp = dtrace_fulword(&gregs[REG_FP]); pc = dtrace_fulword(&gregs[REG_PC]); oldcontext = dtrace_fulword(&ucp->uc_link); } else #endif /* XXX */ { if (SV_PROC_FLAG(p, SV_ILP32)) { pc = dtrace_fuword32((void *)(sp + RETURN_OFFSET)); sp = dtrace_fuword32((void *)sp); } else { pc = dtrace_fuword64((void *)(sp + RETURN_OFFSET64)); sp = dtrace_fuword64((void *)sp); } } /* * This is totally bogus: if we faulted, we're going to clear * the fault and break. This is to deal with the apparently * broken Java stacks on x86. */ if (*flags & CPU_DTRACE_FAULT) { *flags &= ~CPU_DTRACE_FAULT; break; } } zero: while (pcstack_limit-- > 0) *pcstack++ = 0; } /*ARGSUSED*/ uint64_t dtrace_getarg(int arg, int aframes) { uintptr_t val; uintptr_t *fp = (uintptr_t *)dtrace_getfp(); uintptr_t *stack; int i; /* * A total of 8 arguments are passed via registers; any argument with * index of 7 or lower is therefore in a register. */ int inreg = 7; for (i = 1; i <= aframes; i++) { fp = (uintptr_t *)*fp; /* * On ppc32 AIM, and booke, trapexit() is the immediately following * label. On ppc64 AIM trapexit() follows a nop. */ #ifdef __powerpc64__ if ((long)(fp[2]) + 4 == (long)trapexit) { #else if ((long)(fp[1]) == (long)trapexit) { #endif /* * In the case of powerpc, we will use the pointer to the regs * structure that was pushed when we took the trap. To get this * structure, we must increment beyond the frame structure. If the * argument that we're seeking is passed on the stack, we'll pull * the true stack pointer out of the saved registers and decrement * our argument by the number of arguments passed in registers; if * the argument we're seeking is passed in regsiters, we can just * load it directly. */ #ifdef __powerpc64__ struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 48); #else struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 8); #endif if (arg <= inreg) { stack = &rp->fixreg[3]; } else { stack = (uintptr_t *)(rp->fixreg[1]); arg -= inreg; } goto load; } } /* * We know that we did not come through a trap to get into * dtrace_probe() -- the provider simply called dtrace_probe() * directly. As this is the case, we need to shift the argument * that we're looking for: the probe ID is the first argument to * dtrace_probe(), so the argument n will actually be found where * one would expect to find argument (n + 1). */ arg++; if (arg <= inreg) { /* * This shouldn't happen. If the argument is passed in a * register then it should have been, well, passed in a * register... */ DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); } arg -= (inreg + 1); stack = fp + 2; load: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); val = stack[arg]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); return (val); } int dtrace_getstackdepth(int aframes) { int depth = 0; uintptr_t osp, sp; vm_offset_t callpc; osp = PAGE_SIZE; aframes++; sp = dtrace_getfp(); depth++; for(;;) { if (sp <= osp) break; - if (!dtrace_sp_inkernel(sp, aframes)) + if (!dtrace_sp_inkernel(sp)) break; if (aframes == 0) depth++; else aframes--; osp = sp; sp = dtrace_next_sp(sp); } if (depth < aframes) return (0); return (depth); } ulong_t dtrace_getreg(struct trapframe *rp, uint_t reg) { if (reg < 32) return (rp->fixreg[reg]); switch (reg) { case 33: return (rp->lr); case 34: return (rp->cr); case 35: return (rp->xer); case 36: return (rp->ctr); case 37: return (rp->srr0); case 38: return (rp->srr1); case 39: return (rp->exc); default: DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); } } static int dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) { ASSERT(INKERNEL(kaddr) && kaddr + size >= kaddr); if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = uaddr; return (0); } return (1); } void dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) { if (dtrace_copycheck(uaddr, kaddr, size)) if (copyin((const void *)uaddr, (void *)kaddr, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } void dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, volatile uint16_t *flags) { if (dtrace_copycheck(uaddr, kaddr, size)) { if (copyout((const void *)kaddr, (void *)uaddr, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } } void dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) { size_t actual; int error; if (dtrace_copycheck(uaddr, kaddr, size)) { error = copyinstr((const void *)uaddr, (void *)kaddr, size, &actual); /* ENAMETOOLONG is not a fault condition. */ if (error && error != ENAMETOOLONG) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } } /* * The bulk of this function could be replaced to match dtrace_copyinstr() * if we ever implement a copyoutstr(). */ void dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, volatile uint16_t *flags) { size_t len; if (dtrace_copycheck(uaddr, kaddr, size)) { len = strlen((const char *)kaddr); if (len > size) len = size; if (copyout((const void *)kaddr, (void *)uaddr, len)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } } uint8_t dtrace_fuword8(void *uaddr) { if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; return (0); } return (fubyte(uaddr)); } uint16_t dtrace_fuword16(void *uaddr) { uint16_t ret = 0; if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } return ret; } uint32_t dtrace_fuword32(void *uaddr) { if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; return (0); } return (fuword32(uaddr)); } uint64_t dtrace_fuword64(void *uaddr) { uint64_t ret = 0; if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } return ret; } uintptr_t dtrace_fulword(void *uaddr) { uintptr_t ret = 0; if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) { if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; } } return ret; } Index: stable/11/sys/cddl/dev/fbt/powerpc/fbt_isa.c =================================================================== --- stable/11/sys/cddl/dev/fbt/powerpc/fbt_isa.c (revision 318571) +++ stable/11/sys/cddl/dev/fbt/powerpc/fbt_isa.c (revision 318572) @@ -1,248 +1,248 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include "fbt.h" -#define FBT_PATCHVAL 0x7c810808 +#define FBT_PATCHVAL 0x7ffff808 #define FBT_MFLR_R0 0x7c0802a6 #define FBT_MTLR_R0 0x7c0803a6 #define FBT_BLR 0x4e800020 #define FBT_BCTR 0x4e800030 #define FBT_BRANCH 0x48000000 #define FBT_BR_MASK 0x03fffffc #define FBT_IS_JUMP(instr) ((instr & ~FBT_BR_MASK) == FBT_BRANCH) #define FBT_ENTRY "entry" #define FBT_RETURN "return" #define FBT_AFRAMES 7 int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu = &solaris_cpu[curcpu]; fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; uintptr_t tmp; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { if (fbt->fbtp_roffset == 0) { cpu->cpu_dtrace_caller = addr; dtrace_probe(fbt->fbtp_id, frame->fixreg[3], frame->fixreg[4], frame->fixreg[5], frame->fixreg[6], frame->fixreg[7]); cpu->cpu_dtrace_caller = 0; } else { dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); /* * The caller doesn't have the fbt item, so * fixup tail calls here. */ if (fbt->fbtp_rval == DTRACE_INVOP_JUMP) { frame->srr0 = (uintptr_t)fbt->fbtp_patchpoint; tmp = fbt->fbtp_savedval & FBT_BR_MASK; /* Sign extend. */ if (tmp & 0x02000000) #ifdef __powerpc64__ tmp |= 0xfffffffffc000000ULL; #else tmp |= 0xfc000000UL; #endif frame->srr0 += tmp; } cpu->cpu_dtrace_caller = 0; } return (fbt->fbtp_rval); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; __syncicache(fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { char *modname = opaque; const char *name = symval->name; fbt_probe_t *fbt, *retfbt; int j; uint32_t *instr, *limit; #ifdef __powerpc64__ /* * PowerPC64 uses '.' prefixes on symbol names, ignore it, but only * allow symbols with the '.' prefix, so that we don't get the function * descriptor instead. */ if (name[0] == '.') name++; else return (0); #endif if (strncmp(name, "dtrace_", 7) == 0 && strncmp(name, "dtrace_safe_", 12) != 0) { /* * Anything beginning with "dtrace_" may be called * from probe context unless it explicitly indicates * that it won't be called from probe context by * using the prefix "dtrace_safe_". */ return (0); } instr = (uint32_t *) symval->value; limit = (uint32_t *) (symval->value + symval->size); for (; instr < limit; instr++) if (*instr == FBT_MFLR_R0) break; if (*instr != FBT_MFLR_R0) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, FBT_AFRAMES, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_rval = DTRACE_INVOP_MFLR_R0; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: if (instr >= limit) return (0); /* * We (desperately) want to avoid erroneously instrumenting a * jump table. To determine if we're looking at a true instruction * sequence or an inline jump table that happens to contain the same * byte sequences, we resort to some heuristic sleeze: we treat this * instruction as being contained within a pointer, and see if that * pointer points to within the body of the function. If it does, we * refuse to instrument it. */ { uint32_t *ptr; ptr = *(uint32_t **)instr; if (ptr >= (uint32_t *) symval->value && ptr < limit) { instr++; goto again; } } if (*instr != FBT_MTLR_R0) { instr++; goto again; } instr++; for (j = 0; j < 12 && instr < limit; j++, instr++) { if ((*instr == FBT_BCTR) || (*instr == FBT_BLR) || FBT_IS_JUMP(*instr)) break; } if (!(*instr == FBT_BCTR || *instr == FBT_BLR || FBT_IS_JUMP(*instr))) goto again; /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, FBT_AFRAMES, fbt); } else { retfbt->fbtp_next = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; if (*instr == FBT_BCTR) fbt->fbtp_rval = DTRACE_INVOP_BCTR; else if (*instr == FBT_BLR) fbt->fbtp_rval = DTRACE_INVOP_RET; else fbt->fbtp_rval = DTRACE_INVOP_JUMP; fbt->fbtp_roffset = (uintptr_t)((uint8_t *)instr - (uint8_t *)symval->value); fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr += 4; goto again; } Index: stable/11/sys/powerpc/booke/trap_subr.S =================================================================== --- stable/11/sys/powerpc/booke/trap_subr.S (revision 318571) +++ stable/11/sys/powerpc/booke/trap_subr.S (revision 318572) @@ -1,963 +1,955 @@ /*- * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski * Copyright (C) 2006 Semihalf, Marian Balakowicz * Copyright (C) 2006 Juniper Networks, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ */ /* * NOTICE: This is not a standalone file. to use it, #include it in * your port's locore.S, like so: * * #include */ /* * SPRG usage notes * * SPRG0 - pcpu pointer * SPRG1 - all interrupts except TLB miss, critical, machine check * SPRG2 - critical * SPRG3 - machine check * SPRG4-6 - scratch * */ /* Get the per-CPU data structure */ #define GET_CPUINFO(r) mfsprg0 r #define RES_GRANULE 32 #define RES_LOCK 0 /* offset to the 'lock' word */ #define RES_RECURSE 4 /* offset to the 'recurse' word */ /* * Standard interrupt prolog * * sprg_sp - SPRG{1-3} reg used to temporarily store the SP * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) * isrr0-1 - save restore registers with CPU state at interrupt time (may be * SRR0-1, CSRR0-1, MCSRR0-1 * * 1. saves in the given savearea: * - R30-31 * - DEAR, ESR * - xSRR0-1 * * 2. saves CR -> R30 * * 3. switches to kstack if needed * * 4. notes: * - R31 can be used as scratch register until a new frame is layed on * the stack with FRAME_SETUP * * - potential TLB miss: NO. Saveareas are always acessible via TLB1 * permanent entries, and within this prolog we do not dereference any * locations potentially not in the TLB */ #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ mtspr sprg_sp, %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ stw %r30, (savearea+CPUSAVE_R30)(%r1); \ stw %r31, (savearea+CPUSAVE_R31)(%r1); \ mfdear %r30; \ mfesr %r31; \ stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ mfspr %r30, isrr0; \ mfspr %r31, isrr1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ isync; \ mfspr %r1, sprg_sp; /* Restore SP */ \ mfcr %r30; /* Save CR */ \ /* switch to per-thread kstack if intr taken in user mode */ \ mtcr %r31; /* MSR at interrupt time */ \ bf 17, 1f; \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1: #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ mtspr sprg_sp, %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ stw %r30, (savearea+CPUSAVE_R30)(%r1); \ stw %r31, (savearea+CPUSAVE_R31)(%r1); \ mfdear %r30; \ mfesr %r31; \ stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ mfspr %r30, isrr0; \ mfspr %r31, isrr1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ mfspr %r30, SPR_SRR0; \ mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0+8)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1+8)(%r1); \ isync; \ mfspr %r1, sprg_sp; /* Restore SP */ \ mfcr %r30; /* Save CR */ \ /* switch to per-thread kstack if intr taken in user mode */ \ mtcr %r31; /* MSR at interrupt time */ \ bf 17, 1f; \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1: /* * FRAME_SETUP assumes: * SPRG{1-3} SP at the time interrupt occured * savearea r30-r31, DEAR, ESR, xSRR0-1 * r30 CR * r31 scratch * r1 kernel stack * * sprg_sp - SPRG reg containing SP at the time interrupt occured * savearea - temp save * exc - exception number (EXC_xxx) * * 1. sets a new frame * 2. saves in the frame: * - R0, R1 (SP at the time of interrupt), R2, LR, CR * - R3-31 (R30-31 first restored from savearea) * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 * * Notes: * - potential TLB miss: YES, since we make dereferences to kstack, which * can happen not covered (we can have up to two DTLB misses if fortunate * enough i.e. when kstack crosses page boundary and both pages are * untranslated) */ #define FRAME_SETUP(sprg_sp, savearea, exc) \ mfspr %r31, sprg_sp; /* get saved SP */ \ /* establish a new stack frame and put everything on it */ \ stwu %r31, -FRAMELEN(%r1); \ stw %r0, FRAME_0+8(%r1); /* save r0 in the trapframe */ \ stw %r31, FRAME_1+8(%r1); /* save SP " " */ \ stw %r2, FRAME_2+8(%r1); /* save r2 " " */ \ mflr %r31; \ stw %r31, FRAME_LR+8(%r1); /* save LR " " */ \ stw %r30, FRAME_CR+8(%r1); /* save CR " " */ \ GET_CPUINFO(%r2); \ lwz %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ lwz %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ /* save R3-31 */ \ stmw %r3, FRAME_3+8(%r1) ; \ /* save DEAR, ESR */ \ lwz %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ lwz %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ stw %r28, FRAME_BOOKE_DEAR+8(%r1); \ stw %r29, FRAME_BOOKE_ESR+8(%r1); \ /* save XER, CTR, exc number */ \ mfxer %r3; \ mfctr %r4; \ stw %r3, FRAME_XER+8(%r1); \ stw %r4, FRAME_CTR+8(%r1); \ li %r5, exc; \ stw %r5, FRAME_EXC+8(%r1); \ /* save DBCR0 */ \ mfspr %r3, SPR_DBCR0; \ stw %r3, FRAME_BOOKE_DBCR0+8(%r1); \ /* save xSSR0-1 */ \ lwz %r30, (savearea+CPUSAVE_SRR0)(%r2); \ lwz %r31, (savearea+CPUSAVE_SRR1)(%r2); \ stw %r30, FRAME_SRR0+8(%r1); \ stw %r31, FRAME_SRR1+8(%r1); \ lwz %r2,PC_CURTHREAD(%r2) /* set curthread pointer */ /* * * isrr0-1 - save restore registers to restore CPU state to (may be * SRR0-1, CSRR0-1, MCSRR0-1 * * Notes: * - potential TLB miss: YES. The deref'd kstack may be not covered */ #define FRAME_LEAVE(isrr0, isrr1) \ /* restore CTR, XER, LR, CR */ \ lwz %r4, FRAME_CTR+8(%r1); \ lwz %r5, FRAME_XER+8(%r1); \ lwz %r6, FRAME_LR+8(%r1); \ lwz %r7, FRAME_CR+8(%r1); \ mtctr %r4; \ mtxer %r5; \ mtlr %r6; \ mtcr %r7; \ /* restore DBCR0 */ \ lwz %r4, FRAME_BOOKE_DBCR0+8(%r1); \ mtspr SPR_DBCR0, %r4; \ /* restore xSRR0-1 */ \ lwz %r30, FRAME_SRR0+8(%r1); \ lwz %r31, FRAME_SRR1+8(%r1); \ mtspr isrr0, %r30; \ mtspr isrr1, %r31; \ /* restore R2-31, SP */ \ lmw %r2, FRAME_2+8(%r1) ; \ lwz %r0, FRAME_0+8(%r1); \ lwz %r1, FRAME_1+8(%r1); \ isync /* * TLB miss prolog * * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area * * Notes: * - potential TLB miss: NO. It is crucial that we do not generate a TLB * miss within the TLB prolog itself! * - TLBSAVE is always translated */ #define TLB_PROLOG \ mtsprg4 %r1; /* Save SP */ \ mtsprg5 %r28; \ mtsprg6 %r29; \ /* calculate TLB nesting level and TLBSAVE instance address */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ addi %r28, %r28, 1; \ stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ \ /* save R20-31 */ \ mfsprg5 %r28; \ mfsprg6 %r29; \ stmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ /* save LR, CR */ \ mflr %r30; \ mfcr %r31; \ stw %r30, (TLBSAVE_BOOKE_LR)(%r1); \ stw %r31, (TLBSAVE_BOOKE_CR)(%r1); \ /* save SRR0-1 */ \ mfsrr0 %r30; /* execution addr at interrupt time */ \ mfsrr1 %r31; /* MSR at interrupt time*/ \ stw %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ stw %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ isync; \ mfsprg4 %r1 /* * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area * * same notes as for the TLB_PROLOG */ #define TLB_RESTORE \ mtsprg4 %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ /* calculate TLB nesting level and TLBSAVE instance addr */ \ lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ subi %r28, %r28, 1; \ stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ add %r1, %r1, %r29; \ \ /* restore LR, CR */ \ lwz %r30, (TLBSAVE_BOOKE_LR)(%r1); \ lwz %r31, (TLBSAVE_BOOKE_CR)(%r1); \ mtlr %r30; \ mtcr %r31; \ /* restore SRR0-1 */ \ lwz %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ lwz %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ mtsrr0 %r30; \ mtsrr1 %r31; \ /* restore R20-31 */ \ lmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ mfsprg4 %r1 #ifdef SMP #define TLB_LOCK \ GET_CPUINFO(%r20); \ lwz %r21, PC_CURTHREAD(%r20); \ lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ \ 1: lwarx %r23, 0, %r22; \ cmpwi %r23, TLB_UNLOCKED; \ beq 2f; \ \ /* check if this is recursion */ \ cmplw cr0, %r21, %r23; \ bne- 1b; \ \ 2: /* try to acquire lock */ \ stwcx. %r21, 0, %r22; \ bne- 1b; \ \ /* got it, update recursion counter */ \ lwz %r21, RES_RECURSE(%r22); \ addi %r21, %r21, 1; \ stw %r21, RES_RECURSE(%r22); \ isync; \ msync #define TLB_UNLOCK \ GET_CPUINFO(%r20); \ lwz %r21, PC_CURTHREAD(%r20); \ lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ \ /* update recursion counter */ \ lwz %r23, RES_RECURSE(%r22); \ subi %r23, %r23, 1; \ stw %r23, RES_RECURSE(%r22); \ \ cmpwi %r23, 0; \ bne 1f; \ isync; \ msync; \ \ /* release the lock */ \ li %r23, TLB_UNLOCKED; \ stw %r23, 0(%r22); \ 1: isync; \ msync #else #define TLB_LOCK #define TLB_UNLOCK #endif /* SMP */ #define INTERRUPT(label) \ .globl label; \ .align 5; \ CNAME(label): /* * Interrupt handling routines in BookE can be flexibly placed and do not have * to live in pre-defined vectors location. Note they need to be TLB-mapped at * all times in order to be able to handle exceptions. We thus arrange for * them to be part of kernel text which is always TLB-accessible. * * The interrupt handling routines have to be 16 bytes aligned: we align them * to 32 bytes (cache line length) which supposedly performs better. * */ .text .globl CNAME(interrupt_vector_base) .align 5 interrupt_vector_base: /***************************************************************************** * Catch-all handler to handle uninstalled IVORs ****************************************************************************/ INTERRUPT(int_unknown) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) b trap_common /***************************************************************************** * Critical input interrupt ****************************************************************************/ INTERRUPT(int_critical_input) STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) rfci /***************************************************************************** * Machine check interrupt ****************************************************************************/ INTERRUPT(int_machine_check) STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) rfmci /***************************************************************************** * Data storage interrupt ****************************************************************************/ INTERRUPT(int_data_storage) STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) b trap_common /***************************************************************************** * Instruction storage interrupt ****************************************************************************/ INTERRUPT(int_instr_storage) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) b trap_common /***************************************************************************** * External input interrupt ****************************************************************************/ INTERRUPT(int_external_input) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b clear_we INTERRUPT(int_alignment) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) b trap_common INTERRUPT(int_program) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) b trap_common INTERRUPT(int_fpu) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) b trap_common /***************************************************************************** * System call ****************************************************************************/ INTERRUPT(int_syscall) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) b trap_common /***************************************************************************** * Decrementer interrupt ****************************************************************************/ INTERRUPT(int_decrementer) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b clear_we /***************************************************************************** * Fixed interval timer ****************************************************************************/ INTERRUPT(int_fixed_interval_timer) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) b trap_common /***************************************************************************** * Watchdog interrupt ****************************************************************************/ INTERRUPT(int_watchdog) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) b trap_common /***************************************************************************** * Altivec Unavailable interrupt ****************************************************************************/ INTERRUPT(int_vec) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) b trap_common /***************************************************************************** * Watchdog interrupt ****************************************************************************/ INTERRUPT(int_vecast) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) b trap_common #ifdef HWPMC_HOOKS /***************************************************************************** * PMC Interrupt ****************************************************************************/ INTERRUPT(int_performance_counter) STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b trapexit #endif /***************************************************************************** * Data TLB miss interrupt * * There can be nested TLB misses - while handling a TLB miss we reference * data structures that may be not covered by translations. We support up to * TLB_NESTED_MAX-1 nested misses. * * Registers use: * r31 - dear * r30 - unused * r29 - saved mas0 * r28 - saved mas1 * r27 - saved mas2 * r26 - pmap address * r25 - pte address * * r20:r23 - scratch registers ****************************************************************************/ INTERRUPT(int_data_tlb_error) TLB_PROLOG TLB_LOCK mfdear %r31 /* * Save MAS0-MAS2 registers. There might be another tlb miss during * pte lookup overwriting current contents (which was hw filled). */ mfspr %r29, SPR_MAS0 mfspr %r28, SPR_MAS1 mfspr %r27, SPR_MAS2 /* Check faulting address. */ lis %r21, VM_MAXUSER_ADDRESS@h ori %r21, %r21, VM_MAXUSER_ADDRESS@l cmplw cr0, %r31, %r21 blt search_user_pmap /* If it's kernel address, allow only supervisor mode misses. */ mfsrr1 %r21 mtcr %r21 bt 17, search_failed /* check MSR[PR] */ search_kernel_pmap: /* Load r26 with kernel_pmap address */ bl 1f .long kernel_pmap_store-. 1: mflr %r21 lwz %r26, 0(%r21) add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ /* Force kernel tid, set TID to 0 in MAS1. */ li %r21, 0 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ tlb_miss_handle: /* This may result in nested tlb miss. */ bl pte_lookup /* returns PTE address in R25 */ cmpwi %r25, 0 /* pte found? */ beq search_failed /* Finish up, write TLB entry. */ bl tlb_fill_entry tlb_miss_return: TLB_UNLOCK TLB_RESTORE rfi search_user_pmap: /* Load r26 with current user space process pmap */ GET_CPUINFO(%r26) lwz %r26, PC_CURPMAP(%r26) b tlb_miss_handle search_failed: /* * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with * the faulting virtual address anyway, but put a fake RPN and no * access rights. This should cause a following {D,I}SI exception. */ lis %r23, 0xffff0000@h /* revoke all permissions */ /* Load MAS registers. */ mtspr SPR_MAS0, %r29 isync mtspr SPR_MAS1, %r28 isync mtspr SPR_MAS2, %r27 isync mtspr SPR_MAS3, %r23 isync bl zero_mas7 bl zero_mas8 tlbwe msync isync b tlb_miss_return /***************************************************************************** * * Return pte address that corresponds to given pmap/va. If there is no valid * entry return 0. * * input: r26 - pmap * input: r31 - dear * output: r25 - pte address * * scratch regs used: r21 * ****************************************************************************/ pte_lookup: cmpwi %r26, 0 beq 1f /* fail quickly if pmap is invalid */ srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */ add %r25, %r25, %r21 /* offset within pm_pdir[] table */ /* * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] * This load may cause a Data TLB miss for non-kernel pmap! */ lwz %r25, 0(%r25) cmpwi %r25, 0 beq 2f lis %r21, PTBL_MASK@h ori %r21, %r21, PTBL_MASK@l and %r21, %r21, %r31 /* ptbl offset, multiply by ptbl entry size */ srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) add %r25, %r25, %r21 /* address of pte entry */ /* * Get pte->flags * This load may cause a Data TLB miss for non-kernel pmap! */ lwz %r21, PTE_FLAGS(%r25) andi. %r21, %r21, PTE_VALID@l bne 2f 1: li %r25, 0 2: blr /***************************************************************************** * * Load MAS1-MAS3 registers with data, write TLB entry * * input: * r29 - mas0 * r28 - mas1 * r27 - mas2 * r25 - pte * * output: none * * scratch regs: r21-r23 * ****************************************************************************/ tlb_fill_entry: /* * Update PTE flags: we have to do it atomically, as pmap_protect() * running on other CPUs could attempt to update the flags at the same * time. */ li %r23, PTE_FLAGS 1: lwarx %r21, %r23, %r25 /* get pte->flags */ oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ beq 2f ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 2: stwcx. %r21, %r23, %r25 /* write it back */ bne- 1b /* Update MAS2. */ rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ /* Setup MAS3 value in r23. */ lwz %r23, PTE_RPN(%r25) /* get pte->rpn */ rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ /* Load MAS registers. */ mtspr SPR_MAS0, %r29 isync mtspr SPR_MAS1, %r28 isync mtspr SPR_MAS2, %r27 isync mtspr SPR_MAS3, %r22 isync mtspr SPR_MAS7, %r23 isync mflr %r21 bl zero_mas8 mtlr %r21 tlbwe isync msync blr /***************************************************************************** * Instruction TLB miss interrupt * * Same notes as for the Data TLB miss ****************************************************************************/ INTERRUPT(int_inst_tlb_error) TLB_PROLOG TLB_LOCK mfsrr0 %r31 /* faulting address */ /* * Save MAS0-MAS2 registers. There might be another tlb miss during pte * lookup overwriting current contents (which was hw filled). */ mfspr %r29, SPR_MAS0 mfspr %r28, SPR_MAS1 mfspr %r27, SPR_MAS2 mfsrr1 %r21 mtcr %r21 /* check MSR[PR] */ bt 17, search_user_pmap b search_kernel_pmap .globl interrupt_vector_top interrupt_vector_top: /***************************************************************************** * Debug interrupt ****************************************************************************/ INTERRUPT(int_debug) STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) GET_CPUINFO(%r3) lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) bl 0f .long interrupt_vector_base-. .long interrupt_vector_top-. 0: mflr %r5 lwz %r4,0(%r5) /* interrupt_vector_base in r4 */ add %r4,%r4,%r5 cmplw cr0, %r3, %r4 - blt 1f + blt trap_common lwz %r4,4(%r5) /* interrupt_vector_top in r4 */ add %r4,%r4,%r5 addi %r4,%r4,4 cmplw cr0, %r3, %r4 - bge 1f + bge trap_common /* Disable single-stepping for the interrupt handlers. */ lwz %r3, FRAME_SRR1+8(%r1); rlwinm %r3, %r3, 0, 23, 21 stw %r3, FRAME_SRR1+8(%r1); /* Restore srr0 and srr1 as they could have been clobbered. */ GET_CPUINFO(%r4) lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4); mtspr SPR_SRR0, %r3 lwz %r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4); mtspr SPR_SRR1, %r4 b 9f -1: - addi %r3, %r1, 8 - bl CNAME(trap) - /* - * Handle ASTs, needed for proper support of single-stepping. - * We actually need to return to the process with an rfi. - */ - b trapexit 9: FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) rfci /***************************************************************************** * Common trap code ****************************************************************************/ trap_common: /* Call C trap dispatcher */ addi %r3, %r1, 8 bl CNAME(trap) .globl CNAME(trapexit) /* exported for db_backtrace use */ CNAME(trapexit): /* disable interrupts */ wrteei 0 /* Test AST pending - makes sense for user process only */ lwz %r5, FRAME_SRR1+8(%r1) mtcr %r5 bf 17, 1f GET_CPUINFO(%r3) lwz %r4, PC_CURTHREAD(%r3) lwz %r4, TD_FLAGS(%r4) lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l and. %r4, %r4, %r5 beq 1f /* re-enable interrupts before calling ast() */ wrteei 1 addi %r3, %r1, 8 bl CNAME(ast) .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ CNAME(asttrapexit): b trapexit /* test ast ret value ? */ 1: FRAME_LEAVE(SPR_SRR0, SPR_SRR1) rfi #if defined(KDB) /* * Deliberate entry to dbtrap */ .globl CNAME(breakpoint) CNAME(breakpoint): mtsprg1 %r1 mfmsr %r3 mtsrr1 %r3 andi. %r3, %r3, ~(PSL_EE | PSL_ME)@l mtmsr %r3 /* disable interrupts */ isync GET_CPUINFO(%r3) stw %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) mflr %r31 mtsrr0 %r31 mfdear %r30 mfesr %r31 stw %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) mfsrr0 %r30 mfsrr1 %r31 stw %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) isync mfcr %r30 /* * Now the kdb trap catching code. */ dbtrap: FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) /* Call C trap code: */ addi %r3, %r1, 8 bl CNAME(db_trap_glue) or. %r3, %r3, %r3 bne dbleave /* This wasn't for KDB, so switch to real trap: */ b trap_common dbleave: FRAME_LEAVE(SPR_SRR0, SPR_SRR1) rfi #endif /* KDB */ clear_we: lwz %r3, (FRAME_SRR1+8)(%r1) rlwinm %r3, %r3, 0, 14, 12 stw %r3, (FRAME_SRR1+8)(%r1) b trapexit #ifdef SMP ENTRY(tlb_lock) GET_CPUINFO(%r5) lwz %r5, PC_CURTHREAD(%r5) 1: lwarx %r4, 0, %r3 cmpwi %r4, TLB_UNLOCKED bne 1b stwcx. %r5, 0, %r3 bne- 1b isync msync blr ENTRY(tlb_unlock) isync msync li %r4, TLB_UNLOCKED stw %r4, 0(%r3) isync msync blr /* * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); * only a single word from this granule will actually be used as a spin lock * for mutual exclusion between TLB miss handler and pmap layer that * manipulates page table contents. */ .data .align 5 GLOBAL(tlb0_miss_locks) .space RES_GRANULE * MAXCPU #endif Index: stable/11/sys/powerpc/include/trap.h =================================================================== --- stable/11/sys/powerpc/include/trap.h (revision 318571) +++ stable/11/sys/powerpc/include/trap.h (revision 318572) @@ -1,141 +1,141 @@ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: trap.h,v 1.7 2002/02/22 13:51:40 kleink Exp $ * $FreeBSD$ */ #ifndef _POWERPC_TRAP_H_ #define _POWERPC_TRAP_H_ #define EXC_RSVD 0x0000 /* Reserved */ #define EXC_RST 0x0100 /* Reset; all but IBM4xx */ #define EXC_MCHK 0x0200 /* Machine Check */ #define EXC_DSI 0x0300 /* Data Storage Interrupt */ #define EXC_DSE 0x0380 /* Data Segment Interrupt */ #define EXC_ISI 0x0400 /* Instruction Storage Interrupt */ #define EXC_ISE 0x0480 /* Instruction Segment Interrupt */ #define EXC_EXI 0x0500 /* External Interrupt */ #define EXC_ALI 0x0600 /* Alignment Interrupt */ #define EXC_PGM 0x0700 /* Program Interrupt */ #define EXC_FPU 0x0800 /* Floating-point Unavailable */ #define EXC_DECR 0x0900 /* Decrementer Interrupt */ #define EXC_SC 0x0c00 /* System Call */ #define EXC_TRC 0x0d00 /* Trace */ #define EXC_FPA 0x0e00 /* Floating-point Assist */ /* The following is only available on the 601: */ #define EXC_RUNMODETRC 0x2000 /* Run Mode/Trace Exception */ /* The following are only available on 970(G5): */ #define EXC_VECAST_G5 0x1700 /* AltiVec Assist */ /* The following are only available on 7400(G4): */ #define EXC_VEC 0x0f20 /* AltiVec Unavailable */ #define EXC_VECAST_G4 0x1600 /* AltiVec Assist */ /* The following are only available on 604/750/7400: */ #define EXC_PERF 0x0f00 /* Performance Monitoring */ #define EXC_BPT 0x1300 /* Instruction Breakpoint */ #define EXC_SMI 0x1400 /* System Managment Interrupt */ /* The following are only available on 750/7400: */ #define EXC_THRM 0x1700 /* Thermal Management Interrupt */ /* And these are only on the 603: */ #define EXC_IMISS 0x1000 /* Instruction translation miss */ #define EXC_DLMISS 0x1100 /* Data load translation miss */ #define EXC_DSMISS 0x1200 /* Data store translation miss */ /* Power ISA 2.06+: */ #define EXC_VSX 0x0f40 /* VSX Unavailable */ /* The following are available on 4xx and 85xx */ #define EXC_CRIT 0x0100 /* Critical Input Interrupt */ #define EXC_PIT 0x1000 /* Programmable Interval Timer */ #define EXC_FIT 0x1010 /* Fixed Interval Timer */ #define EXC_WDOG 0x1020 /* Watchdog Timer */ #define EXC_DTMISS 0x1100 /* Data TLB Miss */ #define EXC_ITMISS 0x1200 /* Instruction TLB Miss */ #define EXC_APU 0x1300 /* Auxiliary Processing Unit */ #define EXC_DEBUG 0x2f10 /* Debug trap */ #define EXC_VECAST_E 0x2f20 /* Altivec Assist (Book-E) */ #define EXC_LAST 0x2f00 /* Last possible exception vector */ #define EXC_AST 0x3000 /* Fake AST vector */ /* Trap was in user mode */ #define EXC_USER 0x10000 /* * EXC_ALI sets bits in the DSISR and DAR to provide enough * information to recover from the unaligned access without needing to * parse the offending instruction. This includes certain bits of the * opcode, and information about what registers are used. The opcode * indicator values below come from Appendix F of Book III of "The * PowerPC Architecture". */ #define EXC_ALI_OPCODE_INDICATOR(dsisr) ((dsisr >> 10) & 0x7f) #define EXC_ALI_LFD 0x09 #define EXC_ALI_STFD 0x0b /* Macros to extract register information */ #define EXC_ALI_RST(dsisr) ((dsisr >> 5) & 0x1f) /* source or target */ #define EXC_ALI_RA(dsisr) (dsisr & 0x1f) /* * SRR1 bits for program exception traps. These identify what caused * the program exception. See section 6.5.9 of the Power ISA Version * 2.05. */ #define EXC_PGM_FPENABLED (1UL << 20) #define EXC_PGM_ILLEGAL (1UL << 19) #define EXC_PGM_PRIV (1UL << 18) #define EXC_PGM_TRAP (1UL << 17) /* DTrace trap opcode. */ -#define EXC_DTRACE 0x7c810808 +#define EXC_DTRACE 0x7ffff808 /* Magic pointer to store TOC base and other info for trap handlers on ppc64 */ #define TRAP_GENTRAP 0x1f0 #define TRAP_TOCBASE 0x1f8 #ifndef LOCORE struct trapframe; struct pcb; void trap(struct trapframe *); int ppc_instr_emulate(struct trapframe *, struct pcb *); #endif #endif /* _POWERPC_TRAP_H_ */ Index: stable/11 =================================================================== --- stable/11 (revision 318571) +++ stable/11 (revision 318572) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r314370,318130,318167