diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h index 026a9ae2e669..5563f5b1bdbb 100644 --- a/sys/amd64/include/pmc_mdep.h +++ b/sys/amd64/include/pmc_mdep.h @@ -1,139 +1,137 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* Machine dependent interfaces */ #ifndef _MACHINE_PMC_MDEP_H #define _MACHINE_PMC_MDEP_H 1 #ifdef _KERNEL struct pmc_mdep; #endif #include #include #include #include /* * Intel processors implementing V2 and later of the Intel performance * measurement architecture have PMCs of the following classes: TSC, * IAF, IAP, UCF and UCP. */ #define PMC_MDEP_CLASS_INDEX_TSC 1 #define PMC_MDEP_CLASS_INDEX_K8 2 #define PMC_MDEP_CLASS_INDEX_P4 2 #define PMC_MDEP_CLASS_INDEX_IAP 2 #define PMC_MDEP_CLASS_INDEX_IAF 3 #define PMC_MDEP_CLASS_INDEX_UCP 4 #define PMC_MDEP_CLASS_INDEX_UCF 5 /* * On the amd64 platform we support the following PMCs. * * TSC The timestamp counter * K8 AMD Athlon64 and Opteron PMCs in 64 bit mode. * PIV Intel P4/HTT and P4/EMT64 * IAP Intel Core/Core2/Atom CPUs in 64 bits mode. * IAF Intel fixed-function PMCs in Core2 and later CPUs. * UCP Intel Uncore programmable PMCs. * UCF Intel Uncore fixed-function PMCs. */ union pmc_md_op_pmcallocate { struct pmc_md_amd_op_pmcallocate pm_amd; struct pmc_md_iap_op_pmcallocate pm_iap; struct pmc_md_ucf_op_pmcallocate pm_ucf; struct pmc_md_ucp_op_pmcallocate pm_ucp; uint64_t __pad[4]; }; /* Logging */ #define PMCLOG_READADDR PMCLOG_READ64 #define PMCLOG_EMITADDR PMCLOG_EMIT64 #ifdef _KERNEL union pmc_md_pmc { struct pmc_md_amd_pmc pm_amd; struct pmc_md_iaf_pmc pm_iaf; struct pmc_md_iap_pmc pm_iap; struct pmc_md_ucf_pmc pm_ucf; struct pmc_md_ucp_pmc pm_ucp; }; #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_rip) #define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_rbp) #define PMC_TRAPFRAME_TO_USER_SP(TF) ((TF)->tf_rsp) #define PMC_TRAPFRAME_TO_KERNEL_SP(TF) ((TF)->tf_rsp) #define PMC_AT_FUNCTION_PROLOGUE_PUSH_BP(I) \ (((I) & 0xffffffff) == 0xe5894855) /* pushq %rbp; movq %rsp,%rbp */ #define PMC_AT_FUNCTION_PROLOGUE_MOV_SP_BP(I) \ (((I) & 0x00ffffff) == 0x00e58948) /* movq %rsp,%rbp */ #define PMC_AT_FUNCTION_EPILOGUE_RET(I) \ (((I) & 0xFF) == 0xC3) /* ret */ #define PMC_IN_TRAP_HANDLER(PC) \ ((PC) >= (uintptr_t) start_exceptions && \ (PC) < (uintptr_t) end_exceptions) -#define PMC_IN_KERNEL_STACK(S,START,END) \ - ((S) >= (START) && (S) < (END)) -#define PMC_IN_KERNEL(va) INKERNEL(va) - -#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_IN_KERNEL_STACK(va) kstack_contains(curthread, (va), sizeof(va)) +#define PMC_IN_KERNEL(va) INKERNEL(va) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) /* Build a fake kernel trapframe from current instruction pointer. */ #define PMC_FAKE_TRAPFRAME(TF) \ do { \ (TF)->tf_cs = 0; (TF)->tf_rflags = 0; \ __asm __volatile("movq %%rbp,%0" : "=r" ((TF)->tf_rbp)); \ __asm __volatile("movq %%rsp,%0" : "=r" ((TF)->tf_rsp)); \ __asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_rip)); \ } while (0) /* * Prototypes */ void start_exceptions(void), end_exceptions(void); struct pmc_mdep *pmc_amd_initialize(void); void pmc_amd_finalize(struct pmc_mdep *_md); struct pmc_mdep *pmc_intel_initialize(void); void pmc_intel_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _MACHINE_PMC_MDEP_H */ diff --git a/sys/arm/include/pmc_mdep.h b/sys/arm/include/pmc_mdep.h index 69cb0c84deca..d7b80abb64b0 100644 --- a/sys/arm/include/pmc_mdep.h +++ b/sys/arm/include/pmc_mdep.h @@ -1,83 +1,81 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ #define PMC_MDEP_CLASS_INDEX_ARMV7 1 /* * On the ARM platform we support the following PMCs. * * ARMV7 ARM Cortex-A processors */ #include union pmc_md_op_pmcallocate { uint64_t __pad[4]; }; /* Logging */ #define PMCLOG_READADDR PMCLOG_READ32 #define PMCLOG_EMITADDR PMCLOG_EMIT32 #ifdef _KERNEL union pmc_md_pmc { struct pmc_md_armv7_pmc pm_armv7; }; -#define PMC_IN_KERNEL_STACK(S,START,END) \ - ((S) >= (START) && (S) < (END)) +#define PMC_IN_KERNEL_STACK(va) kstack_contains(curthread, (va), sizeof(va)) #define PMC_IN_KERNEL(va) INKERNEL((va)) - -#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_pc) #define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_r11) #define PMC_TRAPFRAME_TO_SVC_SP(TF) ((TF)->tf_svc_sp) #define PMC_TRAPFRAME_TO_USR_SP(TF) ((TF)->tf_usr_sp) #define PMC_TRAPFRAME_TO_SVC_LR(TF) ((TF)->tf_svc_lr) #define PMC_TRAPFRAME_TO_USR_LR(TF) ((TF)->tf_usr_lr) /* Build a fake kernel trapframe from current instruction pointer. */ #define PMC_FAKE_TRAPFRAME(TF) \ do { \ (TF)->tf_spsr = PSR_SVC32_MODE; \ __asm __volatile("mov %0, pc" : "=r" ((TF)->tf_pc)); \ __asm __volatile("mov %0, r11" : "=r" ((TF)->tf_r11)); \ } while (0) /* * Prototypes */ struct pmc_mdep *pmc_armv7_initialize(void); void pmc_armv7_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* !_MACHINE_PMC_MDEP_H_ */ diff --git a/sys/arm64/include/pmc_mdep.h b/sys/arm64/include/pmc_mdep.h index d27bdbd5afc1..049bb2b22751 100644 --- a/sys/arm64/include/pmc_mdep.h +++ b/sys/arm64/include/pmc_mdep.h @@ -1,96 +1,95 @@ /*- * Copyright (c) 2009 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ #define PMC_MDEP_CLASS_INDEX_ARMV8 1 #define PMC_MDEP_CLASS_INDEX_DMC620_CD2 2 #define PMC_MDEP_CLASS_INDEX_DMC620_C 3 #define PMC_MDEP_CLASS_INDEX_CMN600 4 /* * On the ARMv8 platform we support the following PMCs. * * ARMV8 ARM Cortex-A53/57/72 processors */ #include #include #include #include #include union pmc_md_op_pmcallocate { struct { uint32_t pm_md_config; uint32_t pm_md_flags; #define PM_MD_RAW_EVENT 0x1 }; struct pmc_md_cmn600_pmu_op_pmcallocate pm_cmn600; struct pmc_md_dmc620_pmu_op_pmcallocate pm_dmc620; uint64_t __pad[4]; }; /* Logging */ #define PMCLOG_READADDR PMCLOG_READ64 #define PMCLOG_EMITADDR PMCLOG_EMIT64 #ifdef _KERNEL union pmc_md_pmc { struct pmc_md_arm64_pmc pm_arm64; struct pmc_md_cmn600_pmc pm_cmn600; struct pmc_md_dmc620_pmc pm_dmc620; }; -#define PMC_IN_KERNEL_STACK(S,START,END) \ - ((S) >= (START) && (S) < (END)) +#define PMC_IN_KERNEL_STACK(va) kstack_contains(curthread, (va), sizeof(va)) #define PMC_IN_KERNEL(va) INKERNEL((va)) -#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) -#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_elr) -#define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_x[29]) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_elr) +#define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_x[29]) /* * Prototypes */ struct pmc_mdep *pmc_arm64_initialize(void); void pmc_arm64_finalize(struct pmc_mdep *_md); /* Optional class for CMN-600 controler's PMU. */ int pmc_cmn600_initialize(struct pmc_mdep *md); void pmc_cmn600_finalize(struct pmc_mdep *_md); int pmc_cmn600_nclasses(void); /* Optional class for DMC-620 controler's PMU. */ int pmc_dmc620_initialize_cd2(struct pmc_mdep *md); void pmc_dmc620_finalize_cd2(struct pmc_mdep *_md); int pmc_dmc620_initialize_c(struct pmc_mdep *md); void pmc_dmc620_finalize_c(struct pmc_mdep *_md); int pmc_dmc620_nclasses(void); #endif /* _KERNEL */ #endif /* !_MACHINE_PMC_MDEP_H_ */ diff --git a/sys/dev/hwpmc/hwpmc_arm.c b/sys/dev/hwpmc/hwpmc_arm.c index c585a6cefd1e..a0e2e6823219 100644 --- a/sys/dev/hwpmc/hwpmc_arm.c +++ b/sys/dev/hwpmc/hwpmc_arm.c @@ -1,168 +1,161 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include -#include -#include #include +#include + +#include +#include #include #include #include #include - -#include -#include -#include +#include /* XXX: Userland code compiled with gcc will need an heuristic * to be correctly detected. */ #ifdef __clang__ #define PC_OFF 1 #define FP_OFF 0 #else #define PC_OFF -1 #define FP_OFF -3 #endif struct pmc_mdep * pmc_md_initialize(void) { #ifdef CPU_CORTEXA if (cpu_class == CPU_CLASS_CORTEXA) return pmc_armv7_initialize(); #endif return NULL; } void pmc_md_finalize(struct pmc_mdep *md) { #ifdef CPU_CORTEXA if (cpu_class == CPU_CLASS_CORTEXA) pmc_armv7_finalize(md); #endif } int pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { - uintptr_t pc, r, stackstart, stackend, fp; - struct thread *td; + uintptr_t pc, ra, fp; int count; KASSERT(TRAPF_USERMODE(tf) == 0,("[arm,%d] not a kernel backtrace", __LINE__)); - td = curthread; pc = PMC_TRAPFRAME_TO_PC(tf); *cc++ = pc; if (maxsamples <= 1) return (1); - stackstart = (uintptr_t) td->td_kstack; - stackend = (uintptr_t) td->td_kstack + td->td_kstack_pages * PAGE_SIZE; fp = PMC_TRAPFRAME_TO_FP(tf); - - if (!PMC_IN_KERNEL(pc) || - !PMC_IN_KERNEL_STACK(fp, stackstart, stackend)) + if (!PMC_IN_KERNEL(pc) || !PMC_IN_KERNEL_STACK(fp)) return (1); for (count = 1; count < maxsamples; count++) { /* Use saved lr as pc. */ - r = fp + PC_OFF * sizeof(uintptr_t); - if (!PMC_IN_KERNEL_STACK(r, stackstart, stackend)) + ra = fp + PC_OFF * sizeof(uintptr_t); + if (!PMC_IN_KERNEL_STACK(ra)) break; - pc = *(uintptr_t *)r; + pc = *(uintptr_t *)ra; if (!PMC_IN_KERNEL(pc)) break; *cc++ = pc; /* Switch to next frame up */ - r = fp + FP_OFF * sizeof(uintptr_t); - if (!PMC_IN_KERNEL_STACK(r, stackstart, stackend)) + ra = fp + FP_OFF * sizeof(uintptr_t); + if (!PMC_IN_KERNEL_STACK(ra)) break; - fp = *(uintptr_t *)r; - if (!PMC_IN_KERNEL_STACK(fp, stackstart, stackend)) + fp = *(uintptr_t *)ra; + if (!PMC_IN_KERNEL_STACK(fp)) break; } return (count); } int pmc_save_user_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t pc, r, oldfp, fp; int count; KASSERT(TRAPF_USERMODE(tf), ("[x86,%d] Not a user trap frame tf=%p", __LINE__, (void *) tf)); pc = PMC_TRAPFRAME_TO_PC(tf); *cc++ = pc; if (maxsamples <= 1) return (1); oldfp = fp = PMC_TRAPFRAME_TO_FP(tf); if (!PMC_IN_USERSPACE(pc) || !PMC_IN_USERSPACE(fp)) return (1); for (count = 1; count < maxsamples; count++) { /* Use saved lr as pc. */ r = fp + PC_OFF * sizeof(uintptr_t); if (copyin((void *)r, &pc, sizeof(pc)) != 0) break; if (!PMC_IN_USERSPACE(pc)) break; *cc++ = pc; /* Switch to next frame up */ oldfp = fp; r = fp + FP_OFF * sizeof(uintptr_t); if (copyin((void *)r, &fp, sizeof(fp)) != 0) break; if (fp < oldfp || !PMC_IN_USERSPACE(fp)) break; } return (count); } diff --git a/sys/dev/hwpmc/hwpmc_arm64_md.c b/sys/dev/hwpmc/hwpmc_arm64_md.c index dd51f7bd18d5..4220badb30a6 100644 --- a/sys/dev/hwpmc/hwpmc_arm64_md.c +++ b/sys/dev/hwpmc/hwpmc_arm64_md.c @@ -1,141 +1,132 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by the University of Cambridge Computer * Laboratory with support from ARM Ltd. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include -#include -#include #include +#include + +#include +#include #include #include #include #include - -#include -#include -#include +#include struct pmc_mdep * pmc_md_initialize(void) { return (pmc_arm64_initialize()); } void pmc_md_finalize(struct pmc_mdep *md) { pmc_arm64_finalize(md); } int -pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, - struct trapframe *tf) +pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { struct unwind_state frame; - uintptr_t stackstart, stackend; - struct thread *td; int count; KASSERT(TRAPF_USERMODE(tf) == 0,("[arm64,%d] not a kernel backtrace", __LINE__)); - td = curthread; frame.pc = PMC_TRAPFRAME_TO_PC(tf); *cc++ = frame.pc; if (maxsamples <= 1) return (1); - stackstart = (uintptr_t) td->td_kstack; - stackend = (uintptr_t) td->td_kstack + td->td_kstack_pages * PAGE_SIZE; frame.fp = PMC_TRAPFRAME_TO_FP(tf); - - if (!PMC_IN_KERNEL(frame.pc) || - !PMC_IN_KERNEL_STACK(frame.fp, stackstart, stackend)) + if (!PMC_IN_KERNEL(frame.pc) || !PMC_IN_KERNEL_STACK(frame.fp)) return (1); for (count = 1; count < maxsamples; count++) { if (!unwind_frame(curthread, &frame)) break; if (!PMC_IN_KERNEL(frame.pc)) break; *cc++ = frame.pc; } return (count); } int pmc_save_user_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t pc, r, oldfp, fp; int count; KASSERT(TRAPF_USERMODE(tf), ("[arm64,%d] Not a user trap frame tf=%p", __LINE__, (void *) tf)); pc = PMC_TRAPFRAME_TO_PC(tf); *cc++ = pc; if (maxsamples <= 1) return (1); oldfp = fp = PMC_TRAPFRAME_TO_FP(tf); if (!PMC_IN_USERSPACE(pc) || !PMC_IN_USERSPACE(fp)) return (1); for (count = 1; count < maxsamples; count++) { /* Use saved lr as pc. */ r = fp + sizeof(uintptr_t); if (copyin((void *)r, &pc, sizeof(pc)) != 0) break; if (!PMC_IN_USERSPACE(pc)) break; *cc++ = pc; /* Switch to next frame up */ oldfp = fp; r = fp; if (copyin((void *)r, &fp, sizeof(fp)) != 0) break; if (fp < oldfp || !PMC_IN_USERSPACE(fp)) break; } return (count); } diff --git a/sys/dev/hwpmc/hwpmc_x86.c b/sys/dev/hwpmc/hwpmc_x86.c index e427139244ad..913c8f756815 100644 --- a/sys/dev/hwpmc/hwpmc_x86.c +++ b/sys/dev/hwpmc/hwpmc_x86.c @@ -1,278 +1,270 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005,2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include +#include #include #include -#include -#include + +#include +#include #include #include -#include -#include -#include +#include /* For x86/apicvar.h */ #include +#include +#include +#include -#include -#include -#include +#include #include "hwpmc_soft.h" /* * Attempt to walk a user call stack using a too-simple algorithm. * In the general case we need unwind information associated with * the executable to be able to walk the user stack. * * We are handed a trap frame laid down at the time the PMC interrupt * was taken. If the application is using frame pointers, the saved * PC value could be: * a. at the beginning of a function before the stack frame is laid * down, * b. just before a 'ret', after the stack frame has been taken off, * c. somewhere else in the function with a valid stack frame being * present, * * If the application is not using frame pointers, this algorithm will * fail to yield an interesting call chain. * * TODO: figure out a way to use unwind information. */ int pmc_save_user_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { int n; uint32_t instr; uintptr_t fp, oldfp, pc, r, sp; KASSERT(TRAPF_USERMODE(tf), ("[x86,%d] Not a user trap frame tf=%p", __LINE__, (void *) tf)); pc = PMC_TRAPFRAME_TO_PC(tf); oldfp = fp = PMC_TRAPFRAME_TO_FP(tf); sp = PMC_TRAPFRAME_TO_USER_SP(tf); *cc++ = pc; n = 1; r = fp + sizeof(uintptr_t); /* points to return address */ if (!PMC_IN_USERSPACE(pc)) return (n); if (copyin((void *) pc, &instr, sizeof(instr)) != 0) return (n); if (PMC_AT_FUNCTION_PROLOGUE_PUSH_BP(instr) || PMC_AT_FUNCTION_EPILOGUE_RET(instr)) { /* ret */ if (copyin((void *) sp, &pc, sizeof(pc)) != 0) return (n); } else if (PMC_AT_FUNCTION_PROLOGUE_MOV_SP_BP(instr)) { sp += sizeof(uintptr_t); if (copyin((void *) sp, &pc, sizeof(pc)) != 0) return (n); } else if (copyin((void *) r, &pc, sizeof(pc)) != 0 || copyin((void *) fp, &fp, sizeof(fp)) != 0) return (n); for (; n < nframes;) { if (pc == 0 || !PMC_IN_USERSPACE(pc)) break; *cc++ = pc; n++; if (fp < oldfp) break; r = fp + sizeof(uintptr_t); /* address of return address */ oldfp = fp; if (copyin((void *) r, &pc, sizeof(pc)) != 0 || copyin((void *) fp, &fp, sizeof(fp)) != 0) break; } return (n); } /* * Walking the kernel call stack. * * We are handed the trap frame laid down at the time the PMC * interrupt was taken. The saved PC could be: * a. in the lowlevel trap handler, meaning that there isn't a C stack * to traverse, * b. at the beginning of a function before the stack frame is laid * down, * c. just before a 'ret', after the stack frame has been taken off, * d. somewhere else in a function with a valid stack frame being * present. * * In case (d), the previous frame pointer is at [%ebp]/[%rbp] and * the return address is at [%ebp+4]/[%rbp+8]. * * For cases (b) and (c), the return address is at [%esp]/[%rsp] and * the frame pointer doesn't need to be changed when going up one * level in the stack. * * For case (a), we check if the PC lies in low-level trap handling * code, and if so we terminate our trace. */ int __nosanitizeaddress __nosanitizememory pmc_save_kernel_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { - int n; + uintptr_t fp, pc, ra, sp; uint32_t instr; - uintptr_t fp, pc, r, sp, stackstart, stackend; - struct thread *td; + int n; KASSERT(TRAPF_USERMODE(tf) == 0,("[x86,%d] not a kernel backtrace", __LINE__)); - td = curthread; pc = PMC_TRAPFRAME_TO_PC(tf); fp = PMC_TRAPFRAME_TO_FP(tf); sp = PMC_TRAPFRAME_TO_KERNEL_SP(tf); *cc++ = pc; - r = fp + sizeof(uintptr_t); /* points to return address */ + ra = fp + sizeof(uintptr_t); /* points to return address */ if (nframes <= 1) return (1); - stackstart = (uintptr_t) td->td_kstack; - stackend = (uintptr_t) td->td_kstack + td->td_kstack_pages * PAGE_SIZE; - - if (PMC_IN_TRAP_HANDLER(pc) || - !PMC_IN_KERNEL(pc) || - !PMC_IN_KERNEL_STACK(r, stackstart, stackend) || - !PMC_IN_KERNEL_STACK(sp, stackstart, stackend) || - !PMC_IN_KERNEL_STACK(fp, stackstart, stackend)) + if (PMC_IN_TRAP_HANDLER(pc) || !PMC_IN_KERNEL(pc) || + !PMC_IN_KERNEL_STACK(ra) || !PMC_IN_KERNEL_STACK(sp) || + !PMC_IN_KERNEL_STACK(fp)) return (1); - instr = *(uint32_t *) pc; + instr = *(uint32_t *)pc; /* * Determine whether the interrupted function was in the * processing of either laying down its stack frame or taking * it off. * * If we haven't started laying down a stack frame, or are * just about to return, then our caller's address is at * *sp, and we don't have a frame to unwind. */ if (PMC_AT_FUNCTION_PROLOGUE_PUSH_BP(instr) || PMC_AT_FUNCTION_EPILOGUE_RET(instr)) pc = *(uintptr_t *) sp; else if (PMC_AT_FUNCTION_PROLOGUE_MOV_SP_BP(instr)) { /* * The code was midway through laying down a frame. * At this point sp[0] has a frame back pointer, * and the caller's address is therefore at sp[1]. */ sp += sizeof(uintptr_t); - if (!PMC_IN_KERNEL_STACK(sp, stackstart, stackend)) + if (!PMC_IN_KERNEL_STACK(sp)) return (1); - pc = *(uintptr_t *) sp; + pc = *(uintptr_t *)sp; } else { /* * Not in the function prologue or epilogue. */ - pc = *(uintptr_t *) r; - fp = *(uintptr_t *) fp; + pc = *(uintptr_t *)ra; + fp = *(uintptr_t *)fp; } for (n = 1; n < nframes; n++) { *cc++ = pc; if (PMC_IN_TRAP_HANDLER(pc)) break; - r = fp + sizeof(uintptr_t); - if (!PMC_IN_KERNEL_STACK(fp, stackstart, stackend) || - !PMC_IN_KERNEL_STACK(r, stackstart, stackend)) + ra = fp + sizeof(uintptr_t); + if (!PMC_IN_KERNEL_STACK(fp) || !PMC_IN_KERNEL_STACK(ra)) break; - pc = *(uintptr_t *) r; - fp = *(uintptr_t *) fp; + pc = *(uintptr_t *)ra; + fp = *(uintptr_t *)fp; } return (n); } /* * Machine dependent initialization for x86 class platforms. */ - struct pmc_mdep * pmc_md_initialize(void) { int i; struct pmc_mdep *md; /* determine the CPU kind */ if (cpu_vendor_id == CPU_VENDOR_AMD || cpu_vendor_id == CPU_VENDOR_HYGON) md = pmc_amd_initialize(); else if (cpu_vendor_id == CPU_VENDOR_INTEL) md = pmc_intel_initialize(); else return (NULL); /* disallow sampling if we do not have an LAPIC */ if (md != NULL && !lapic_enable_pmc()) for (i = 0; i < md->pmd_nclass; i++) { if (i == PMC_CLASS_INDEX_SOFT) continue; md->pmd_classdep[i].pcd_caps &= ~PMC_CAP_INTERRUPT; } return (md); } void pmc_md_finalize(struct pmc_mdep *md) { lapic_disable_pmc(); if (cpu_vendor_id == CPU_VENDOR_AMD || cpu_vendor_id == CPU_VENDOR_HYGON) pmc_amd_finalize(md); else if (cpu_vendor_id == CPU_VENDOR_INTEL) pmc_intel_finalize(md); else KASSERT(0, ("[x86,%d] Unknown vendor", __LINE__)); } diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h index 3d498e006a36..15bdd7a65dc8 100644 --- a/sys/i386/include/pmc_mdep.h +++ b/sys/i386/include/pmc_mdep.h @@ -1,162 +1,160 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2003-2005,2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMC_MDEP_H #define _MACHINE_PMC_MDEP_H 1 #ifdef _KERNEL struct pmc_mdep; #endif /* * On the i386 platform we support the following PMCs. * * TSC The timestamp counter * K7 AMD Athlon XP/MP and other 32 bit processors. * K8 AMD Athlon64 and Opteron PMCs in 32 bit mode. * IAP Intel Core/Core2/Atom programmable PMCs. * IAF Intel fixed-function PMCs. * UCP Intel Uncore programmable PMCs. * UCF Intel Uncore fixed-function PMCs. */ #include /* K7 and K8 */ #include #include #include /* * Intel processors implementing V2 and later of the Intel performance * measurement architecture have PMCs of the following classes: TSC, * IAF, IAP, UCF and UCP. */ #define PMC_MDEP_CLASS_INDEX_TSC 1 #define PMC_MDEP_CLASS_INDEX_K7 2 #define PMC_MDEP_CLASS_INDEX_K8 2 #define PMC_MDEP_CLASS_INDEX_IAP 2 #define PMC_MDEP_CLASS_INDEX_IAF 3 #define PMC_MDEP_CLASS_INDEX_UCP 4 #define PMC_MDEP_CLASS_INDEX_UCF 5 /* * Architecture specific extensions to structures. */ union pmc_md_op_pmcallocate { struct pmc_md_amd_op_pmcallocate pm_amd; struct pmc_md_iap_op_pmcallocate pm_iap; struct pmc_md_ucf_op_pmcallocate pm_ucf; struct pmc_md_ucp_op_pmcallocate pm_ucp; uint64_t __pad[4]; }; /* Logging */ #define PMCLOG_READADDR PMCLOG_READ32 #define PMCLOG_EMITADDR PMCLOG_EMIT32 #ifdef _KERNEL /* MD extension for 'struct pmc' */ union pmc_md_pmc { struct pmc_md_amd_pmc pm_amd; struct pmc_md_iaf_pmc pm_iaf; struct pmc_md_iap_pmc pm_iap; struct pmc_md_ucf_pmc pm_ucf; struct pmc_md_ucp_pmc pm_ucp; }; struct pmc; struct pmc_mdep; #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_eip) #define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_ebp) /* * The layout of the stack frame on entry into the NMI handler depends on * whether a privilege level change (and consequent stack switch) was * required for entry. * * When processing an interrupt when in user mode, the processor switches * stacks, and saves the user mode stack pointer on the kernel stack. The * user mode stack pointer is then available to the interrupt handler * at frame->tf_esp. * * When processing an interrupt while in kernel mode, the processor * continues to use the existing (kernel) stack. Therefore we determine * the stack pointer for the interrupted kernel procedure by adding an * offset to the current frame pointer. */ #define PMC_TRAPFRAME_TO_USER_SP(TF) ((TF)->tf_esp) #define PMC_TRAPFRAME_TO_KERNEL_SP(TF) ((uintptr_t) &((TF)->tf_esp)) -#define PMC_IN_KERNEL_STACK(S,START,END) \ - ((S) >= (START) && (S) < (END)) +#define PMC_IN_KERNEL_STACK(va) kstack_contains(curthread, (va), sizeof(va)) #define PMC_IN_KERNEL(va) INKERNEL(va) - -#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) #define PMC_IN_TRAP_HANDLER(PC) \ ((PC) >= (uintptr_t)start_exceptions + setidt_disp && \ (PC) < (uintptr_t) end_exceptions + setidt_disp) #define PMC_AT_FUNCTION_PROLOGUE_PUSH_BP(I) \ (((I) & 0x00ffffff) == 0xe58955) /* pushl %ebp; movl %esp,%ebp */ #define PMC_AT_FUNCTION_PROLOGUE_MOV_SP_BP(I) \ (((I) & 0x0000ffff) == 0xe589) /* movl %esp,%ebp */ #define PMC_AT_FUNCTION_EPILOGUE_RET(I) \ (((I) & 0xFF) == 0xC3) /* ret */ /* Build a fake kernel trapframe from current instruction pointer. */ #define PMC_FAKE_TRAPFRAME(TF) \ do { \ (TF)->tf_cs = 0; (TF)->tf_eflags = 0; \ __asm __volatile("movl %%ebp,%0" : "=r" ((TF)->tf_ebp)); \ __asm __volatile("movl %%esp,%0" : "=r" ((TF)->tf_esp)); \ __asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_eip)); \ } while (0) /* * Prototypes */ void start_exceptions(void), end_exceptions(void); struct pmc_mdep *pmc_amd_initialize(void); void pmc_amd_finalize(struct pmc_mdep *_md); struct pmc_mdep *pmc_intel_initialize(void); void pmc_intel_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _MACHINE_PMC_MDEP_H */ diff --git a/sys/riscv/include/pmc_mdep.h b/sys/riscv/include/pmc_mdep.h index 10dc5c8b98ed..b192d75ec14b 100644 --- a/sys/riscv/include/pmc_mdep.h +++ b/sys/riscv/include/pmc_mdep.h @@ -1,65 +1,64 @@ /*- * Copyright (c) 2009 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ #define PMC_MDEP_CLASS_INDEX_RISCV 1 /* * On the RISC-V platform we don't support any PMCs yet. */ #include union pmc_md_op_pmcallocate { uint64_t __pad[4]; }; /* Logging */ #define PMCLOG_READADDR PMCLOG_READ64 #define PMCLOG_EMITADDR PMCLOG_EMIT64 #ifdef _KERNEL union pmc_md_pmc { struct pmc_md_riscv_pmc pm_riscv; }; -#define PMC_IN_KERNEL_STACK(S,START,END) \ - ((S) >= (START) && (S) < (END)) -#define PMC_IN_KERNEL(va) INKERNEL((va)) -#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) -#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_ra) -#define PMC_TRAPFRAME_TO_FP(TF) (0) /* stub */ +#define PMC_IN_KERNEL_STACK(va) kstack_contains(curthread, (va), sizeof(va)) +#define PMC_IN_KERNEL(va) INKERNEL((va)) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_ra) +#define PMC_TRAPFRAME_TO_FP(TF) (0) /* stub */ /* * Prototypes */ struct pmc_mdep *pmc_riscv_initialize(void); void pmc_riscv_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* !_MACHINE_PMC_MDEP_H_ */