Index: sys/amd64/amd64/trap.c =================================================================== --- sys/amd64/amd64/trap.c +++ sys/amd64/amd64/trap.c @@ -70,12 +70,6 @@ #include #include #include -#ifdef HWPMC_HOOKS -#include -PMC_SOFT_DEFINE( , , page_fault, all); -PMC_SOFT_DEFINE( , , page_fault, read); -PMC_SOFT_DEFINE( , , page_fault, write); -#endif #include #include @@ -101,6 +95,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg), IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32), IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall); @@ -817,15 +818,7 @@ rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode); if (rv == KERN_SUCCESS) { #ifdef HWPMC_HOOKS - if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { - PMC_SOFT_CALL_TF( , , page_fault, all, frame); - if (ftype == VM_PROT_READ) - PMC_SOFT_CALL_TF( , , page_fault, read, - frame); - else - PMC_SOFT_CALL_TF( , , page_fault, write, - frame); - } + pmc_soft_page_fault(ftype, frame); #endif return (0); } Index: sys/arm/arm/trap-v6.c =================================================================== --- sys/arm/arm/trap-v6.c +++ sys/arm/arm/trap-v6.c @@ -27,6 +27,7 @@ * SUCH DAMAGE. */ +#include "opt_hwpmc_hooks.h" #include "opt_ktrace.h" #include @@ -68,6 +69,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + extern char cachebailout[]; #ifdef DEBUG @@ -513,8 +521,12 @@ pcb->pcb_onfault = onfault; #endif - if (__predict_true(rv == KERN_SUCCESS)) + if (__predict_true(rv == KERN_SUCCESS)) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, tf); +#endif goto out; + } nogo: if (!usermode) { if (td->td_intr_nesting_level == 0 && Index: sys/arm64/arm64/trap.c =================================================================== --- sys/arm64/arm64/trap.c +++ sys/arm64/arm64/trap.c @@ -28,6 +28,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -59,6 +61,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + #ifdef VFP #include #endif @@ -307,7 +316,11 @@ /* Fault in the page. */ error = vm_fault_trap(map, far, ftype, VM_FAULT_NORMAL, &sig, &ucode); - if (error != KERN_SUCCESS) { + if (error == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, frame); +#endif + } else { if (lower) { call_trapsignal(td, sig, ucode, (void *)far, ESR_ELx_EXCEPTION(esr)); Index: sys/i386/i386/trap.c =================================================================== --- sys/i386/i386/trap.c +++ sys/i386/i386/trap.c @@ -71,12 +71,7 @@ #include #include #include -#ifdef HWPMC_HOOKS -#include -PMC_SOFT_DEFINE( , , page_fault, all); -PMC_SOFT_DEFINE( , , page_fault, read); -PMC_SOFT_DEFINE( , , page_fault, write); -#endif + #include #include @@ -109,6 +104,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + void trap(struct trapframe *frame); void syscall(struct trapframe *frame); @@ -851,15 +853,7 @@ rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode); if (rv == KERN_SUCCESS) { #ifdef HWPMC_HOOKS - if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { - PMC_SOFT_CALL_TF( , , page_fault, all, frame); - if (ftype == VM_PROT_READ) - PMC_SOFT_CALL_TF( , , page_fault, read, - frame); - else - PMC_SOFT_CALL_TF( , , page_fault, write, - frame); - } + pmc_soft_page_fault(ftype, frame); #endif return (0); } Index: sys/mips/mips/trap.c =================================================================== --- sys/mips/mips/trap.c +++ sys/mips/mips/trap.c @@ -45,6 +45,7 @@ #include "opt_ddb.h" #include "opt_ktrace.h" +#include "opt_hwpmc_hooks.h" #include #include @@ -96,6 +97,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + #ifdef TRAP_DEBUG int trap_debug = 0; SYSCTL_INT(_machdep, OID_AUTO, trap_debug, CTLFLAG_RW, @@ -672,8 +680,12 @@ va = (vm_offset_t)trapframe->badvaddr; rv = vm_fault_trap(kernel_map, va, ftype, VM_FAULT_NORMAL, NULL, NULL); - if (rv == KERN_SUCCESS) + if (rv == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, trapframe); +#endif return (trapframe->pc); + } if (td->td_pcb->pcb_onfault != NULL) { pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault; td->td_pcb->pcb_onfault = NULL; @@ -727,6 +739,9 @@ #endif if (rv == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, trapframe); +#endif if (!usermode) { return (trapframe->pc); } Index: sys/powerpc/powerpc/trap.c =================================================================== --- sys/powerpc/powerpc/trap.c +++ sys/powerpc/powerpc/trap.c @@ -34,6 +34,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -73,6 +75,13 @@ #include #include +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + /* Below matches setjmp.S */ #define FAULTBUF_LR 21 #define FAULTBUF_R1 1 @@ -837,8 +846,12 @@ * XXXDTRACE: add dtrace_doubletrap_func here? */ - if (rv == KERN_SUCCESS) + if (rv == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, frame); +#endif return (true); + } if (!user && handle_onfault(frame)) return (true); Index: sys/riscv/riscv/trap.c =================================================================== --- sys/riscv/riscv/trap.c +++ sys/riscv/riscv/trap.c @@ -35,6 +35,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -71,6 +73,13 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); +#endif + int (*dtrace_invop_jump_addr)(struct trapframe *); extern register_t fsu_intr_fault; @@ -232,7 +241,11 @@ goto done; error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode); - if (error != KERN_SUCCESS) { + if (error == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + pmc_soft_page_fault(ftype, frame); +#endif + } else { if (usermode) { call_trapsignal(td, sig, ucode, (void *)stval, frame->tf_scause & SCAUSE_CODE); Index: sys/sys/pmckern.h =================================================================== --- sys/sys/pmckern.h +++ sys/sys/pmckern.h @@ -269,4 +269,28 @@ struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); void pmc_soft_ev_release(struct pmc_soft *ps); +#ifdef VM_H +/* Page fault events */ +PMC_SOFT_DECLARE( , , page_fault, all); +PMC_SOFT_DECLARE( , , page_fault, read); +PMC_SOFT_DECLARE( , , page_fault, write); + +/* + * Helper function for the page fault soft events. To be called from the MD + * trap handler. + */ +static __inline void +pmc_soft_page_fault(vm_prot_t ftype, struct trapframe *tf) +{ + if (ftype != VM_PROT_READ && ftype != VM_PROT_WRITE) + return; + + PMC_SOFT_CALL_TF( , , page_fault, all, tf); + if (ftype == VM_PROT_READ) + PMC_SOFT_CALL_TF( , , page_fault, read, tf); + else + PMC_SOFT_CALL_TF( , , page_fault, write, tf); +} +#endif + #endif /* _SYS_PMCKERN_H_ */