Index: sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- sys/dev/hwpmc/hwpmc_mod.c +++ sys/dev/hwpmc/hwpmc_mod.c @@ -78,6 +78,7 @@ #include #include "hwpmc_soft.h" +#include "hwpmc_vm.h" #define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et) #define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et) @@ -104,6 +105,7 @@ #define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)] struct mtx_pool *pmc_mtxpool; +struct mtx_pool *pmc_mtxpool_sleep; static int *pmc_pmcdisp; /* PMC row dispositions */ #define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0) @@ -1467,6 +1469,8 @@ ("[pmc,%d] pmcval outside of expected range cpu=%d " "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__, cpu, ri, newvalue, pm->pm_sc.pm_reloadcount)); + } else if (PMC_TO_MODE(pm) == PMC_MODE_TT) { + /* Nothing */ } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, @@ -1482,7 +1486,8 @@ pcd->pcd_write_pmc(cpu, adjri, newvalue); /* If a sampling mode PMC, reset stalled state. */ - if (PMC_TO_MODE(pm) == PMC_MODE_TS) + if (PMC_TO_MODE(pm) == PMC_MODE_TS || + PMC_TO_MODE(pm) == PMC_MODE_TT) pm->pm_pcpu_state[cpu].pps_stalled = 0; /* Indicate that we desire this to run. */ @@ -1663,6 +1668,8 @@ pm->pm_sc.pm_reloadcount; } mtx_pool_unlock_spin(pmc_mtxpool, pm); + } else if (mode == PMC_MODE_TT) { + /* Nothing */ } else { tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); @@ -1755,8 +1762,11 @@ char *fullpath, *freepath; const struct pmc *pm; struct pmc_owner *po; - const struct pmc_process *pp; + struct pmc_process *pp; + struct proc *p; + bool pause_thread; + pause_thread = false; freepath = fullpath = NULL; MPASS(!in_epoch(global_epoch_preempt)); pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); @@ -1772,19 +1782,43 @@ if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) goto done; + p = td->td_proc; + if ((p->p_flag & P_HWPMC) == 0) + goto done; + /* * Inform sampling PMC owners tracking this process. */ - for (ri = 0; ri < md->pmd_npmc; ri++) - if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && - PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + for (ri = 0; ri < md->pmd_npmc; ri++) { + if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) + continue; + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_TT) pmclog_process_map_in(pm->pm_owner, pid, pkm->pm_address, fullpath); + if (PMC_TO_MODE(pm) == PMC_MODE_TT) + pause_thread = true; + } done: if (freepath) free(freepath, M_TEMP); PMC_EPOCH_EXIT(); + + /* + * pmclog entry with mmap information just scheduled to ship + * to userspace. This not yet received by pmctrace application. + * Put this thread on pause before we continue. Once user process + * receive log entry, it can reconfigure tracing filters, start + * tracing operation and finally wakeup this thread. + */ + if (pp != NULL && pause_thread == true) { + pp->pp_refcnt++; + mtx_lock(pp->pp_tslock); + msleep(pp, pp->pp_tslock, PWAIT, "pmc-mmap", 0); + mtx_unlock(pp->pp_tslock); + pp->pp_refcnt--; + } } @@ -1813,11 +1847,14 @@ if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) return; - for (ri = 0; ri < md->pmd_npmc; ri++) - if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && - PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + for (ri = 0; ri < md->pmd_npmc; ri++) { + if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) + continue; + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_TT) pmclog_process_map_out(pm->pm_owner, pid, pkm->pm_address, pkm->pm_address + pkm->pm_size); + } } /* @@ -1831,7 +1868,8 @@ struct pmckern_map_in *km, *kmbase; MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx)); - KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), + KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_ST, ("[pmc,%d] non-sampling PMC (%p) desires mapping information", __LINE__, (void *) pm)); @@ -2594,8 +2632,8 @@ mtx_lock_spin(&pmc_processhash_mtx); LIST_FOREACH(pp, pph, pp_next) - if (pp->pp_proc == p) - break; + if (pp->pp_proc == p) + break; if ((mode & PMC_FLAG_REMOVE) && pp != NULL) LIST_REMOVE(pp, pp_next); @@ -2605,6 +2643,7 @@ ppnew->pp_proc = p; LIST_INIT(&ppnew->pp_tds); ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew); + ppnew->pp_tslock = mtx_pool_find(pmc_mtxpool_sleep, ppnew); LIST_INSERT_HEAD(pph, ppnew, pp_next); mtx_unlock_spin(&pmc_processhash_mtx); pp = ppnew; @@ -2873,6 +2912,15 @@ */ LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { pp = ptgt->pt_process; + + /* + * Owner of this pmc descriptor is terminated. + * If any tracing target thread that associated with + * this pmc descriptor is sleeping, then release it + * from sleeping. + */ + wakeup(pp); + pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt); @@ -3160,7 +3208,8 @@ * If this is a sampling mode PMC, log mapping information for * the kernel modules that are currently loaded. */ - if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_ST) pmc_log_kernel_mappings(pm); if (PMC_IS_VIRTUAL_MODE(mode)) { @@ -3835,9 +3884,14 @@ mode = pa.pm_mode; cpu = pa.pm_cpu; - if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && - mode != PMC_MODE_TS && mode != PMC_MODE_TC) || - (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) { + if (mode != PMC_MODE_SS && mode != PMC_MODE_TS && + mode != PMC_MODE_SC && mode != PMC_MODE_TC && + mode != PMC_MODE_ST && mode != PMC_MODE_TT) { + error = EINVAL; + break; + } + + if (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max()) { error = EINVAL; break; } @@ -4312,6 +4366,176 @@ } break; + case PMC_OP_LOG_KERNEL_MAP: + { + struct pmc_op_simple sp; + struct pmc *pm; + + if ((error = copyin(arg, &sp, sizeof(sp))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(sp.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST) + break; + + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + pmc_log_kernel_mappings(pm); + } + break; + + case PMC_OP_THREAD_WAKEUP: + { + struct pmc_op_thread_wakeup u; + struct pmc_process *pp; + struct proc *p; + struct pmc *pm; + + if ((error = copyin(arg, &u, sizeof(u))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(u.pm_pmcid, &pm)) != 0) + break; + + /* lookup pid */ + if ((p = pfind(u.pm_pid)) == NULL) { + error = ESRCH; + break; + } + + if ((p->p_flag & P_HWPMC) == 0) + break; + + if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) + wakeup(pp); + + PROC_UNLOCK(p); + } + break; + + case PMC_OP_TRACE_CONFIG: + { + struct pmc_op_trace_config trc; + uint64_t *ranges; + struct pmc *pm; + struct pmc_binding pb; + struct pmc_classdep *pcd; + uint32_t nranges; + uint32_t cpu; + uint32_t ri; + int adjri; + + if ((error = copyin(arg, &trc, sizeof(trc))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(trc.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST && + PMC_TO_MODE(pm) != PMC_MODE_TT) + break; + + /* Can't proceed with PMC that hasn't been started. */ + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + cpu = trc.pm_cpu; + + ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + if (pcd->pcd_trace_config == NULL) + break; + + /* switch to CPU 'cpu' */ + pmc_save_cpu_binding(&pb); + pmc_select_cpu(cpu); + + ranges = trc.ranges; + nranges = trc.nranges; + + mtx_pool_lock_spin(pmc_mtxpool, pm); + error = (*pcd->pcd_trace_config)(cpu, adjri, + pm, ranges, nranges); + mtx_pool_unlock_spin(pmc_mtxpool, pm); + + pmc_restore_cpu_binding(&pb); + } + break; + + /* + * Read a PMC trace buffer ptr. + */ + case PMC_OP_TRACE_READ: + { + struct pmc_op_trace_read trr; + struct pmc_op_trace_read *trr_ret; + struct pmc_binding pb; + struct pmc_classdep *pcd; + struct pmc *pm; + pmc_value_t cycle; + pmc_value_t offset; + uint32_t cpu; + uint32_t ri; + int adjri; + + if ((error = copyin(arg, &trr, sizeof(trr))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(trr.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST && + PMC_TO_MODE(pm) != PMC_MODE_TT) + break; + + /* Can't read a PMC that hasn't been started. */ + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + cpu = trr.pm_cpu; + + ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + + /* switch to CPU 'cpu' */ + pmc_save_cpu_binding(&pb); + pmc_select_cpu(cpu); + + mtx_pool_lock_spin(pmc_mtxpool, pm); + error = (*pcd->pcd_read_trace)(cpu, adjri, + pm, &cycle, &offset); + mtx_pool_unlock_spin(pmc_mtxpool, pm); + + pmc_restore_cpu_binding(&pb); + + trr_ret = (struct pmc_op_trace_read *)arg; + if ((error = copyout(&cycle, &trr_ret->pm_cycle, + sizeof(trr.pm_cycle)))) + break; + if ((error = copyout(&offset, &trr_ret->pm_offset, + sizeof(trr.pm_offset)))) + break; + } + break; /* * Read and/or write a PMC. @@ -4415,7 +4639,7 @@ /* save old value */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, - &oldvalue))) + &oldvalue))) goto error; /* write out new value */ if (prw.pm_flags & PMC_F_NEWVALUE) @@ -5585,6 +5809,12 @@ return (ENOSYS); } + error = pmc_vm_initialize(md); + if (error) { + pmc_md_finalize(md); + return (error); + } + KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, ("[pmc,%d] no classes or pmcs", __LINE__)); @@ -5705,6 +5935,8 @@ /* allocate a pool of spin mutexes */ pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size, MTX_SPIN); + pmc_mtxpool_sleep = mtx_pool_create("pmc-sleep", pmc_mtxpool_size, + MTX_DEF); PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx " "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, @@ -5822,6 +6054,8 @@ if (pmc_mtxpool) mtx_pool_destroy(&pmc_mtxpool); + if (pmc_mtxpool_sleep) + mtx_pool_destroy(&pmc_mtxpool_sleep); mtx_destroy(&pmc_processhash_mtx); taskqgroup_config_gtask_deinit(&free_gtask); @@ -5919,6 +6153,8 @@ } pmclog_shutdown(); + pmc_vm_finalize(); + counter_u64_free(pmc_stats.pm_intr_ignored); counter_u64_free(pmc_stats.pm_intr_processed); counter_u64_free(pmc_stats.pm_intr_bufferfull); Index: sys/sys/pmc.h =================================================================== --- sys/sys/pmc.h +++ sys/sys/pmc.h @@ -110,6 +110,7 @@ __PMC_CPU(INTEL_BROADWELL_XEON, 0x97, "Intel Broadwell Xeon") \ __PMC_CPU(INTEL_SKYLAKE, 0x98, "Intel Skylake") \ __PMC_CPU(INTEL_SKYLAKE_XEON, 0x99, "Intel Skylake Xeon") \ + __PMC_CPU(INTEL_KABYLAKE, 0x9A, "Intel Kabylake") \ __PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \ __PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \ __PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \ @@ -162,7 +163,9 @@ __PMC_CLASS(ARMV8, 0x11, "ARMv8") \ __PMC_CLASS(MIPS74K, 0x12, "MIPS 74K") \ __PMC_CLASS(E500, 0x13, "Freescale e500 class") \ - __PMC_CLASS(BERI, 0x14, "MIPS BERI") + __PMC_CLASS(BERI, 0x14, "MIPS BERI") \ + __PMC_CLASS(PT, 0x15, "Intel PT") \ + __PMC_CLASS(CORESIGHT, 0x16, "ARM Coresight") enum pmc_class { #undef __PMC_CLASS @@ -171,7 +174,7 @@ }; #define PMC_CLASS_FIRST PMC_CLASS_TSC -#define PMC_CLASS_LAST PMC_CLASS_E500 +#define PMC_CLASS_LAST PMC_CLASS_CORESIGHT /* * A PMC can be in the following states: @@ -242,7 +245,9 @@ __PMC_MODE(SS, 0) \ __PMC_MODE(SC, 1) \ __PMC_MODE(TS, 2) \ - __PMC_MODE(TC, 3) + __PMC_MODE(TC, 3) \ + __PMC_MODE(ST, 4) \ + __PMC_MODE(TT, 5) enum pmc_mode { #undef __PMC_MODE @@ -256,11 +261,11 @@ #define PMC_IS_COUNTING_MODE(mode) \ ((mode) == PMC_MODE_SC || (mode) == PMC_MODE_TC) #define PMC_IS_SYSTEM_MODE(mode) \ - ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC) + ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC || (mode) == PMC_MODE_ST) #define PMC_IS_SAMPLING_MODE(mode) \ ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_TS) #define PMC_IS_VIRTUAL_MODE(mode) \ - ((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC) + ((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC || (mode) == PMC_MODE_TT) /* * PMC row disposition @@ -352,7 +357,11 @@ __PMC_OP(PMCSTOP, "Stop a PMC") \ __PMC_OP(WRITELOG, "Write a cookie to the log file") \ __PMC_OP(CLOSELOG, "Close log file") \ - __PMC_OP(GETDYNEVENTINFO, "Get dynamic events list") + __PMC_OP(GETDYNEVENTINFO, "Get dynamic events list") \ + __PMC_OP(LOG_KERNEL_MAP, "Log kernel mappings") \ + __PMC_OP(THREAD_WAKEUP, "Thread wakeup") \ + __PMC_OP(TRACE_READ, "Read trace buffer pointer") \ + __PMC_OP(TRACE_CONFIG, "Setup trace IP ranges") enum pmc_ops { @@ -500,7 +509,6 @@ pmc_value_t pm_value; /* new&returned value */ }; - /* * OP GETPMCINFO * @@ -526,6 +534,40 @@ struct pmc_info pm_pmcs[]; /* space for 'npmc' structures */ }; +/* + * OP THREAD_WAKEUP + * + * Wakeup a sleeping thread. + */ + +struct pmc_op_thread_wakeup { + pmc_id_t pm_pmcid; + pid_t pm_pid; +}; + +/* + * OP TRACE_CONFIG + */ + +#define PMC_FILTER_MAX_IP_RANGES 4 + +struct pmc_op_trace_config { + pmc_id_t pm_pmcid; + uint32_t pm_cpu; /* CPU number or PMC_CPU_ANY */ + uint64_t ranges[2 * PMC_FILTER_MAX_IP_RANGES]; + uint32_t nranges; +}; + +/* + * OP TRACE_READ + */ + +struct pmc_op_trace_read { + pmc_id_t pm_pmcid; + uint32_t pm_cpu; + pmc_value_t pm_cycle; /* returned value */ + pmc_value_t pm_offset; /* returned value */ +}; /* * OP GETCPUINFO @@ -533,7 +575,6 @@ * Retrieve system CPU information. */ - struct pmc_classinfo { enum pmc_class pm_class; /* class id */ uint32_t pm_caps; /* counter capabilities */ @@ -843,6 +884,7 @@ LIST_ENTRY(pmc_process) pp_next; /* hash chain */ LIST_HEAD(,pmc_thread) pp_tds; /* list of threads */ struct mtx *pp_tdslock; /* lock on pp_tds thread list */ + struct mtx *pp_tslock; /* thread sleep lock */ int pp_refcnt; /* reference count */ uint32_t pp_flags; /* flags PMC_PP_* */ struct proc *pp_proc; /* target process */ @@ -1019,6 +1061,12 @@ int (*pcd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); int (*pcd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); + /* tracing */ + int (*pcd_read_trace)(int _cpu, int _ri, struct pmc *_pm, + pmc_value_t *_cycle, pmc_value_t *_offset); + int (*pcd_trace_config)(int _cpu, int _ri, struct pmc *_pm, + uint64_t *ranges, uint32_t nranges); + /* pmc allocation/release */ int (*pcd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t, const struct pmc_op_pmcallocate *_a); @@ -1046,7 +1094,7 @@ * Machine dependent bits needed per CPU type. */ -struct pmc_mdep { +struct pmc_mdep { uint32_t pmd_cputype; /* from enum pmc_cputype */ uint32_t pmd_npmc; /* number of PMCs per CPU */ uint32_t pmd_nclass; /* number of PMC classes present */