Index: sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- sys/dev/hwpmc/hwpmc_mod.c +++ sys/dev/hwpmc/hwpmc_mod.c @@ -81,6 +81,7 @@ PMC_FLAG_NONE = 0x00, /* do nothing */ PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */ PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */ + PMC_FLAG_NOWAIT = 0x04, /* do not wait for mallocs */ }; /* @@ -163,6 +164,21 @@ static LIST_HEAD(, pmc_owner) pmc_ss_owners; +/* + * List of free thread entries. This is protected by the spin + * mutex. + */ +static struct mtx pmc_threadfreelist_mtx; /* spin mutex */ +static LIST_HEAD(, pmc_thread) pmc_threadfreelist; +static int pmc_threadfreelist_entries=0; +#define THREADENTRY_SIZE \ +(sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate))) + +/* + * Callout to manage thread free list entries. + */ +static struct callout pmc_threadfreelist_callout; +static struct mtx pmc_threadfreelist_callout_mtx; /* * A map of row indices to classdep structures. @@ -179,6 +195,8 @@ #endif static int load(struct module *module, int cmd, void *arg); +static void pmc_add_thread_descriptors_from_proc(struct proc *p, + struct pmc_process *pp); static int pmc_attach_process(struct proc *p, struct pmc *pm); static struct pmc *pmc_allocate_pmc_descriptor(void); static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); @@ -193,12 +211,15 @@ int flags); static void pmc_destroy_owner_descriptor(struct pmc_owner *po); static void pmc_destroy_pmc_descriptor(struct pmc *pm); +static void pmc_destroy_process_descriptor(struct pmc_process *pp); static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmc); static struct pmc_process *pmc_find_process_descriptor(struct proc *p, uint32_t mode); +static struct pmc_thread *pmc_find_thread_descriptor(struct pmc_process *pp, + struct thread *td, uint32_t mode); static void pmc_force_context_switch(void); static void pmc_link_target_process(struct pmc *pm, struct pmc_process *pp); @@ -213,6 +234,8 @@ struct proc *p2, int n); static void pmc_process_samples(int cpu, int soft); static void pmc_release_pmc_descriptor(struct pmc *pmc); +static void pmc_process_thread_add(struct thread *td); +static void pmc_process_thread_delete(struct thread *td); static void pmc_remove_owner(struct pmc_owner *po); static void pmc_remove_process_descriptor(struct pmc_process *pp); static void pmc_restore_cpu_binding(struct pmc_binding *pb); @@ -221,6 +244,10 @@ static int pmc_start(struct pmc *pm); static int pmc_stop(struct pmc *pm); static int pmc_syscall_handler(struct thread *td, void *syscall_args); +static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void); +static void pmc_thread_descriptor_pool_drain(void); +static void pmc_thread_descriptor_pool_free(struct pmc_thread *pt); +static void pmc_thread_descriptor_pool_monitor(void *arg); static void pmc_unlink_target_process(struct pmc *pmc, struct pmc_process *pp); static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp); @@ -276,6 +303,24 @@ /* + * kern.hwpmc.threadfreelist_entries -- number of free entries + */ + +SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD, + &pmc_threadfreelist_entries, 0, "number of avalable thread entries"); + + +/* + * kern.hwpmc.threadfreelist_max -- maximum number of free entries + */ + +static int pmc_threadfreelist_max = PMC_THREADLIST_MAX; +SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW, + &pmc_threadfreelist_max, 0, + "maximum number of available thread entries before freeing some"); + + +/* * security.bsd.unprivileged_syspmcs -- allow non-root processes to * allocate system-wide PMCs. * @@ -799,6 +844,9 @@ { int ri; struct pmc_target *pt; +#ifdef INVARIANTS + struct pmc_thread *pt_td; +#endif sx_assert(&pmc_sx, SX_XLOCKED); @@ -842,6 +890,18 @@ pp->pp_refcnt++; +#ifdef INVARIANTS + /* Confirm that the per-thread values at this row index are cleared. */ + if (PMC_TO_MODE(pm) == PMC_MODE_TS) { + mtx_lock_spin(pp->pp_tdslock); + LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) { + KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0, + ("[pmc,%d] pt_pmcval not cleared for pid=%d at " + "ri=%d", __LINE__, pp->pp_proc->p_pid, ri)); + } + mtx_unlock_spin(pp->pp_tdslock); + } +#endif } /* @@ -854,6 +914,7 @@ int ri; struct proc *p; struct pmc_target *ptgt; + struct pmc_thread *pt; sx_assert(&pmc_sx, SX_XLOCKED); @@ -876,6 +937,14 @@ pp->pp_pmcs[ri].pp_pmc = NULL; pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; + /* Clear the per-thread values at this row index. */ + if (PMC_TO_MODE(pm) == PMC_MODE_TS) { + mtx_lock_spin(pp->pp_tdslock); + LIST_FOREACH(pt, &pp->pp_tds, pt_next) + pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0; + mtx_unlock_spin(pp->pp_tdslock); + } + /* Remove owner-specific flags */ if (pm->pm_owner->po_owner == pp->pp_proc) { pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS; @@ -990,6 +1059,11 @@ */ ri = PMC_TO_ROWINDEX(pm); + /* mark process as using HWPMCs */ + PROC_LOCK(p); + p->p_flag |= P_HWPMC; + PROC_UNLOCK(p); + if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) return ENOMEM; @@ -1020,10 +1094,6 @@ if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pmc_log_process_mappings(pm->pm_owner, p); } - /* mark process as using HWPMCs */ - PROC_LOCK(p); - p->p_flag |= P_HWPMC; - PROC_UNLOCK(p); return 0; } @@ -1137,7 +1207,7 @@ pmc_remove_process_descriptor(pp); if (flags & PMC_FLAG_REMOVE) - free(pp, M_PMC); + pmc_destroy_process_descriptor(pp); PROC_LOCK(p); p->p_flag &= ~P_HWPMC; @@ -1214,6 +1284,7 @@ struct pmc_hw *phw; pmc_value_t newvalue; struct pmc_process *pp; + struct pmc_thread *pt = NULL; struct pmc_classdep *pcd; p = td->td_proc; @@ -1276,23 +1347,54 @@ /* * Write out saved value and start the PMC. * - * Sampling PMCs use a per-process value, while + * Sampling PMCs use a per-thread value, while * counting mode PMCs use a per-pmc value that is * inherited across descendants. */ if (PMC_TO_MODE(pm) == PMC_MODE_TS) { + if (pt == NULL) + pt = pmc_find_thread_descriptor(pp, td, + PMC_FLAG_NONE); + + KASSERT(pt != NULL, + ("[pmc,%d] No thread found for td=%p", __LINE__, + td)); + mtx_pool_lock_spin(pmc_mtxpool, pm); /* - * Use the saved value calculated after the most recent - * thread switch out to start this counter. Reset - * the saved count in case another thread from this - * process switches in before any threads switch out. + * If we have a thread descriptor, use the per-thread + * counter in the descriptor. If not, we will use + * a per-process counter. + * + * TODO: Remove the per-process "safety net" once + * we have thoroughly tested that we don't hit the + * above assert. */ - newvalue = PMC_PCPU_SAVED(cpu,ri) = - pp->pp_pmcs[ri].pp_pmcval; - pp->pp_pmcs[ri].pp_pmcval = pm->pm_sc.pm_reloadcount; + if (pt != NULL) { + if (pt->pt_pmcs[ri].pt_pmcval > 0) + newvalue = pt->pt_pmcs[ri].pt_pmcval; + else + newvalue = pm->pm_sc.pm_reloadcount; + } else { + /* + * Use the saved value calculated after the most + * recent time a thread using the shared counter + * switched out. Reset the saved count in case + * another thread from this process switches in + * before any threads switch out. + */ + + newvalue = pp->pp_pmcs[ri].pp_pmcval; + pp->pp_pmcs[ri].pp_pmcval = + pm->pm_sc.pm_reloadcount; + } mtx_pool_unlock_spin(pmc_mtxpool, pm); + KASSERT(newvalue > 0 && newvalue <= + pm->pm_sc.pm_reloadcount, + ("[pmc,%d] pmcval outside of expected range cpu=%d " + "ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__, + cpu, ri, newvalue, pm->pm_sc.pm_reloadcount)); } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, @@ -1345,6 +1447,7 @@ pmc_value_t newvalue; unsigned int adjri, ri; struct pmc_process *pp; + struct pmc_thread *pt = NULL; struct pmc_classdep *pcd; @@ -1440,37 +1543,51 @@ pcd->pcd_read_pmc(cpu, adjri, &newvalue); if (mode == PMC_MODE_TS) { - PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (samp)", - cpu, ri, PMC_PCPU_SAVED(cpu,ri) - newvalue); + PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)", + cpu, ri, newvalue); + + if (pt == NULL) + pt = pmc_find_thread_descriptor(pp, td, + PMC_FLAG_NONE); + + KASSERT(pt != NULL, + ("[pmc,%d] No thread found for td=%p", + __LINE__, td)); + + mtx_pool_lock_spin(pmc_mtxpool, pm); /* - * For sampling process-virtual PMCs, - * newvalue is the number of events to be seen - * until the next sampling interrupt. - * We can just add the events left from this - * invocation to the counter, then adjust - * in case we overflow our range. + * If we have a thread descriptor, save the + * per-thread counter in the descriptor. If not, + * we will update the per-process counter. * - * (Recall that we reload the counter every - * time we use it.) + * TODO: Remove the per-process "safety net" + * once we have thoroughly tested that we + * don't hit the above assert. */ - mtx_pool_lock_spin(pmc_mtxpool, pm); - - pp->pp_pmcs[ri].pp_pmcval += newvalue; - if (pp->pp_pmcs[ri].pp_pmcval > - pm->pm_sc.pm_reloadcount) - pp->pp_pmcs[ri].pp_pmcval -= - pm->pm_sc.pm_reloadcount; - KASSERT(pp->pp_pmcs[ri].pp_pmcval > 0 && - pp->pp_pmcs[ri].pp_pmcval <= - pm->pm_sc.pm_reloadcount, - ("[pmc,%d] pp_pmcval outside of expected " - "range cpu=%d ri=%d pp_pmcval=%jx " - "pm_reloadcount=%jx", __LINE__, cpu, ri, - pp->pp_pmcs[ri].pp_pmcval, - pm->pm_sc.pm_reloadcount)); + if (pt != NULL) + pt->pt_pmcs[ri].pt_pmcval = newvalue; + else { + /* + * For sampling process-virtual PMCs, + * newvalue is the number of events to + * be seen until the next sampling + * interrupt. We can just add the events + * left from this invocation to the + * counter, then adjust in case we + * overflow our range. + * + * (Recall that we reload the counter + * every time we use it.) + */ + pp->pp_pmcs[ri].pp_pmcval += newvalue; + if (pp->pp_pmcs[ri].pp_pmcval > + pm->pm_sc.pm_reloadcount) + pp->pp_pmcs[ri].pp_pmcval -= + pm->pm_sc.pm_reloadcount; + newvalue = pp->pp_pmcs[ri].pp_pmcval; + } mtx_pool_unlock_spin(pmc_mtxpool, pm); - } else { tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); @@ -1514,6 +1631,35 @@ } /* + * A new thread for a process. + */ +static void +pmc_process_thread_add(struct thread *td) +{ + struct pmc_process *pmc; + + pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE); + + if (pmc != NULL) + pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE); +} + +/* + * A thread delete for a process. + */ +static void +pmc_process_thread_delete(struct thread *td) +{ + struct pmc_process *pmc; + + pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE); + + if (pmc != NULL) + pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc, + td, PMC_FLAG_REMOVE)); +} + +/* * A mapping change for a process. */ @@ -1833,13 +1979,16 @@ "MUNMAP", "CALLCHAIN-NMI", "CALLCHAIN-SOFT", - "SOFTSAMPLING" + "SOFTSAMPLING", + "THR-CREATE", + "THR-EXIT", }; #endif static int pmc_hook_handler(struct thread *td, int function, void *arg) { + int cpu; PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, pmc_hooknames[function], arg); @@ -1954,7 +2103,7 @@ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); - free(pp, M_PMC); + pmc_destroy_process_descriptor(pp); break; } @@ -1991,9 +2140,10 @@ * had already processed the interrupt). We don't * lose the interrupt sample. */ - CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask); - pmc_process_samples(PCPU_GET(cpuid), PMC_HR); - pmc_process_samples(PCPU_GET(cpuid), PMC_SR); + cpu = PCPU_GET(cpuid); + CPU_CLR_ATOMIC(cpu, &pmc_cpumask); + pmc_process_samples(cpu, PMC_HR); + pmc_process_samples(cpu, PMC_SR); break; case PMC_FN_MMAP: @@ -2036,6 +2186,16 @@ pmc_soft_intr((struct pmckern_soft *) arg); break; + case PMC_FN_THR_CREATE: + pmc_process_thread_add(td); + break; + + case PMC_FN_THR_EXIT: + KASSERT(td == curthread, ("[pmc,%d] td != curthread", + __LINE__)); + pmc_process_thread_delete(td); + break; + default: #ifdef HWPMC_DEBUG KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); @@ -2087,6 +2247,205 @@ } /* + * Allocate a thread descriptor from the free pool. + * + * NOTE: This *can* return NULL. + */ +static struct pmc_thread * +pmc_thread_descriptor_pool_alloc() +{ + struct pmc_thread *pt; + + mtx_lock_spin(&pmc_threadfreelist_mtx); + if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) { + LIST_REMOVE(pt, pt_next); + pmc_threadfreelist_entries--; + } + mtx_unlock_spin(&pmc_threadfreelist_mtx); + + return pt; +} + +/* + * Add a thread descriptor to the free pool. We use this instead of free() + * to maintain a cache of free entries. Additionally, we can safely call + * this function when we cannot call free(), such as in a critical section. + * + */ +static void +pmc_thread_descriptor_pool_free(struct pmc_thread *pt) +{ + + if (pt == NULL) + return; + + memset(pt, 0, THREADENTRY_SIZE); + mtx_lock_spin(&pmc_threadfreelist_mtx); + LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next); + pmc_threadfreelist_entries++; + mtx_unlock_spin(&pmc_threadfreelist_mtx); +} + +/* + * A callout to manage the free list. + */ +static void +pmc_thread_descriptor_pool_monitor(void *arg __unused) +{ + struct pmc_thread *pt; + int delta; + + mtx_lock(&pmc_threadfreelist_callout_mtx); + + /* If the callout was reset or stopped, take no action. */ + if (callout_pending(&pmc_threadfreelist_callout) || + !callout_active(&pmc_threadfreelist_callout)) + goto unlock; + + /* Deactivate the callout. */ + callout_deactivate(&pmc_threadfreelist_callout); + + /* Determine what changes, if any, we need to make. */ + mtx_lock_spin(&pmc_threadfreelist_mtx); + delta = pmc_threadfreelist_entries - pmc_threadfreelist_max; + mtx_unlock_spin(&pmc_threadfreelist_mtx); + + /* If there are entries to free, free them. */ + while (delta > 0) { + if ((pt = pmc_thread_descriptor_pool_alloc()) != NULL) + free(pt, M_PMC); + else + break; + } + + /* Reschedule this function to run every second. */ + callout_reset(&pmc_threadfreelist_callout, hz, + pmc_thread_descriptor_pool_monitor, NULL); + +unlock: + mtx_unlock(&pmc_threadfreelist_callout_mtx); + return; +} + +/* + * Drain the thread free pool, freeing all allocations. + */ +static void +pmc_thread_descriptor_pool_drain() +{ + struct pmc_thread *pt, *next; + + LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) { + LIST_REMOVE(pt, pt_next); + free(pt, M_PMC); + } +} + +/* + * find the descriptor corresponding to thread 'td', adding or removing it + * as specified by 'mode'. + * + * Note that this supports additional mode flags in addition to those + * supported by pmc_find_process_descriptor(): + * PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs. + * This makes it safe to call while holding certain other locks. + */ + +static struct pmc_thread * +pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td, + uint32_t mode) +{ + struct pmc_thread *pt = NULL, *ptnew = NULL; + int wait_flag; + + KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__)); + + /* + * Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to + * acquiring the lock. + */ + if (mode & PMC_FLAG_ALLOCATE) { + if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) { + wait_flag = (mode & PMC_FLAG_NOWAIT) ? M_NOWAIT : + M_WAITOK; + ptnew = malloc(THREADENTRY_SIZE, M_PMC, + wait_flag|M_ZERO); + } + } + + mtx_lock_spin(pp->pp_tdslock); + + LIST_FOREACH(pt, &pp->pp_tds, pt_next) + if (pt->pt_td == td) + break; + + if ((mode & PMC_FLAG_REMOVE) && pt != NULL) + LIST_REMOVE(pt, pt_next); + + if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) { + pt = ptnew; + ptnew = NULL; + pt->pt_td = td; + LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next); + } + + mtx_unlock_spin(pp->pp_tdslock); + + if (ptnew != NULL) { + free(ptnew, M_PMC); + } + + return pt; +} + +/* + * Try to add thread descriptors for each thread in a process. + */ + +static void +pmc_add_thread_descriptors_from_proc(struct proc *p, struct pmc_process *pp) +{ + struct thread *curtd, **tdlist; + struct pmc_thread *pt; + int i, tdcnt, tdlistsz=32; + + KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked", + __LINE__)); + + restart: + tdcnt = 0; + tdlist = malloc(sizeof(struct thread) * tdlistsz, M_TEMP, M_WAITOK); + + PROC_LOCK(p); + + /* + * Try to add each thread to the list without sleeping. If unable, + * add to a queue to retry after dropping the process lock. + */ + FOREACH_THREAD_IN_PROC(p, curtd) + if (pmc_find_thread_descriptor(pp, curtd, + PMC_FLAG_ALLOCATE|PMC_FLAG_NOWAIT) == NULL && + tdcnt < tdlistsz) + tdlist[tdcnt++] = curtd; + + PROC_UNLOCK(p); + + /* Retry the threads we missed. */ + for (i=0; i < tdcnt; i++) { + pt = pmc_find_thread_descriptor(pp, tdlist[i], PMC_FLAG_ALLOCATE); + KASSERT(pt != NULL, ("[pmc,%d] error adding thread", __LINE__)); + } + + free(tdlist, M_TEMP); + + /* Handle an overflow. */ + if (tdcnt == tdlistsz) { + tdlistsz <<= 1; + goto restart; + } +} + +/* * find the descriptor corresponding to process 'p', adding or removing it * as specified by 'mode'. */ @@ -2104,7 +2463,7 @@ ppnew = NULL; /* - * Pre-allocate memory in the FIND_ALLOCATE case since we + * Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we * cannot call malloc(9) once we hold a spin lock. */ if (mode & PMC_FLAG_ALLOCATE) @@ -2122,13 +2481,20 @@ if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL && ppnew != NULL) { ppnew->pp_proc = p; + LIST_INIT(&ppnew->pp_tds); + ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew); LIST_INSERT_HEAD(pph, ppnew, pp_next); + mtx_unlock_spin(&pmc_processhash_mtx); pp = ppnew; ppnew = NULL; + + /* Add thread descriptors for this process' current threads. */ + pmc_add_thread_descriptors_from_proc(p, pp); } - mtx_unlock_spin(&pmc_processhash_mtx); + else + mtx_unlock_spin(&pmc_processhash_mtx); - if (pp != NULL && ppnew != NULL) + if (ppnew != NULL) free(ppnew, M_PMC); return pp; @@ -2150,6 +2516,24 @@ mtx_unlock_spin(&pmc_processhash_mtx); } +/* + * destroy a process descriptor. + */ + +static void +pmc_destroy_process_descriptor(struct pmc_process *pp) +{ + struct pmc_thread *pmc_td; + + mtx_lock_spin(pp->pp_tdslock); + while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) { + LIST_REMOVE(pmc_td, pt_next); + pmc_thread_descriptor_pool_free(pmc_td); + } + mtx_unlock_spin(pp->pp_tdslock); + free(pp, M_PMC); +} + /* * find an owner descriptor corresponding to proc 'p' @@ -2374,7 +2758,7 @@ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); - free(pp, M_PMC); + pmc_destroy_process_descriptor(pp); } } @@ -4514,15 +4898,21 @@ CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate); if (!CPU_ISSET(cpu, &pm->pm_stalled)) { (void) pcd->pcd_stop_pmc(cpu, adjri); - pcd->pcd_read_pmc(cpu, adjri, - &newvalue); - tmp = newvalue - - PMC_PCPU_SAVED(cpu,ri); - - mtx_pool_lock_spin(pmc_mtxpool, pm); - pm->pm_gv.pm_savedvalue += tmp; - pp->pp_pmcs[ri].pp_pmcval += tmp; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + + if (PMC_TO_MODE(pm) == PMC_MODE_TC) { + pcd->pcd_read_pmc(cpu, adjri, + &newvalue); + tmp = newvalue - + PMC_PCPU_SAVED(cpu,ri); + + mtx_pool_lock_spin(pmc_mtxpool, + pm); + pm->pm_gv.pm_savedvalue += tmp; + pp->pp_pmcs[ri].pp_pmcval += + tmp; + mtx_pool_unlock_spin( + pmc_mtxpool, pm); + } } } @@ -4631,6 +5021,13 @@ if (do_descendants == 0) /* nothing to do */ goto done; + /* + * Now mark the new process as being tracked by this driver. + */ + PROC_LOCK(newproc); + newproc->p_flag |= P_HWPMC; + PROC_UNLOCK(newproc); + /* allocate a descriptor for the new process */ if ((ppnew = pmc_find_process_descriptor(newproc, PMC_FLAG_ALLOCATE)) == NULL) @@ -4655,13 +5052,6 @@ newproc->p_pid); } - /* - * Now mark the new process as being tracked by this driver. - */ - PROC_LOCK(newproc); - newproc->p_flag |= P_HWPMC; - PROC_UNLOCK(newproc); - done: sx_xunlock(&pmc_sx); } @@ -4979,6 +5369,22 @@ "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, pmc_processhash, pmc_processhashmask); + /* Initialize a spin mutex for the thread free list. */ + mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf", + MTX_SPIN); + + /* + * Initialize the callout to monitor the thread free list. + * This callout will also handle the initial population of the list. + */ + mtx_init(&pmc_threadfreelist_callout_mtx, "pmc-threadcallout", + "pmc-leaf", MTX_DEF); + mtx_lock(&pmc_threadfreelist_callout_mtx); + callout_init(&pmc_threadfreelist_callout, TRUE); + callout_reset(&pmc_threadfreelist_callout, 1, + pmc_thread_descriptor_pool_monitor, NULL); + mtx_unlock(&pmc_threadfreelist_callout_mtx); + /* register process {exit,fork,exec} handlers */ pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit, pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY); @@ -5074,6 +5480,11 @@ } /* reclaim allocated data structures */ + callout_drain(&pmc_threadfreelist_callout); + mtx_destroy(&pmc_threadfreelist_callout_mtx); + mtx_destroy(&pmc_threadfreelist_mtx); + pmc_thread_descriptor_pool_drain(); + if (pmc_mtxpool) mtx_pool_destroy(&pmc_mtxpool); Index: sys/kern/kern_thr.c =================================================================== --- sys/kern/kern_thr.c +++ sys/kern/kern_thr.c @@ -29,6 +29,7 @@ #include "opt_compat.h" #include "opt_posix.h" +#include "opt_hwpmc_hooks.h" #include #include #include @@ -53,6 +54,9 @@ #include #include #include +#ifdef HWPMC_HOOKS +#include +#endif #include @@ -260,6 +264,12 @@ vm_domain_policy_localcopy(&newtd->td_vm_dom_policy, &td->td_vm_dom_policy); +#ifdef HWPMC_HOOKS + if (PMC_PROC_IS_USING_PMCS(p)) { + PROC_UNLOCK(p); + PMC_CALL_HOOK(newtd, PMC_FN_THR_CREATE, NULL); + } else +#endif PROC_UNLOCK(p); tidhash_add(newtd); Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -526,8 +526,10 @@ * If this thread is part of a process that is being tracked by hwpmc(4), * inform the module of the thread's impending exit. */ - if (PMC_PROC_IS_USING_PMCS(td->td_proc)) + if (PMC_PROC_IS_USING_PMCS(td->td_proc)) { PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); + PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL); + } #endif PROC_UNLOCK(p); PROC_STATLOCK(p); Index: sys/sys/pmc.h =================================================================== --- sys/sys/pmc.h +++ sys/sys/pmc.h @@ -624,6 +624,7 @@ #define PMC_LOG_BUFFER_SIZE 4 #define PMC_NLOGBUFFERS 1024 #define PMC_NSAMPLES 1024 +#define PMC_THREADLIST_MAX 64 #define PMC_CALLCHAIN_DEPTH 32 #define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "." @@ -760,6 +761,25 @@ #define PMC_TO_ROWINDEX(P) PMC_ID_TO_ROWINDEX((P)->pm_id) #define PMC_TO_CPU(P) PMC_ID_TO_CPU((P)->pm_id) +/* + * struct pmc_threadpmcstate + * + * Record per-PMC, per-thread state. + */ +struct pmc_threadpmcstate { + pmc_value_t pt_pmcval; /* per-thread reload count */ +}; + +/* + * struct pmc_thread + * + * Record a 'target' thread being profiled. + */ +struct pmc_thread { + LIST_ENTRY(pmc_thread) pt_next; /* linked list */ + struct thread *pt_td; /* target thread */ + struct pmc_threadpmcstate pt_pmcs[]; /* per-PMC state */ +}; /* * struct pmc_process @@ -782,9 +802,11 @@ struct pmc_process { LIST_ENTRY(pmc_process) pp_next; /* hash chain */ + LIST_HEAD(,pmc_thread) pp_tds; /* list of threads */ + struct mtx *pp_tdslock; /* lock on pp_tds thread list */ int pp_refcnt; /* reference count */ uint32_t pp_flags; /* flags PMC_PP_* */ - struct proc *pp_proc; /* target thread */ + struct proc *pp_proc; /* target process */ struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */ }; Index: sys/sys/pmckern.h =================================================================== --- sys/sys/pmckern.h +++ sys/sys/pmckern.h @@ -58,6 +58,8 @@ #define PMC_FN_USER_CALLCHAIN 9 #define PMC_FN_USER_CALLCHAIN_SOFT 10 #define PMC_FN_SOFT_SAMPLING 11 +#define PMC_FN_THR_CREATE 12 +#define PMC_FN_THR_EXIT 13 #define PMC_HR 0 /* Hardware ring buffer */ #define PMC_SR 1 /* Software ring buffer */