Differential D7350 Diff 18848 sys/dev/hwpmc/hwpmc_mod.c

Changeset View

Standalone View

sys/dev/hwpmc/hwpmc_mod.c

Show First 20 Lines • Show All 189 Lines • ▼ Show 20 Lines
*/		*/

#ifdef HWPMC_DEBUG		#ifdef HWPMC_DEBUG
static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);		static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
static int pmc_debugflags_parse(char newstr, char fence);		static int pmc_debugflags_parse(char newstr, char fence);
#endif		#endif

static int load(struct module module, int cmd, void arg);		static int load(struct module module, int cmd, void arg);
		static int pmc_add_normal_sample(int cpu, int ring, struct pmc *pm,
		gnnUnsubmitted Not Done Inline Actions Silly request but can you remove "normal" here because it makes me think of normalization of samples of data. I think that just pmc_add_sample() would be fine. gnn: Silly request but can you remove "normal" here because it makes me think of normalization of…
		struct trapframe *tf, int inuserspace, uint32_t count);
static void pmc_add_thread_descriptors_from_proc(struct proc *p,		static void pmc_add_thread_descriptors_from_proc(struct proc *p,
struct pmc_process *pp);		struct pmc_process *pp);
		static void pmc_add_useronly_sample(int cpu, struct pmc *pm);
static int pmc_attach_process(struct proc p, struct pmc pm);		static int pmc_attach_process(struct proc p, struct pmc pm);
static struct pmc *pmc_allocate_pmc_descriptor(void);		static struct pmc *pmc_allocate_pmc_descriptor(void);
static struct pmc_owner pmc_allocate_owner_descriptor(struct proc p);		static struct pmc_owner pmc_allocate_owner_descriptor(struct proc p);
static int pmc_attach_one_process(struct proc p, struct pmc pm);		static int pmc_attach_one_process(struct proc p, struct pmc pm);
static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,		static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
int cpu);		int cpu);
static int pmc_can_attach(struct pmc pm, struct proc p);		static int pmc_can_attach(struct pmc pm, struct proc p);
static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);		static int pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
static void pmc_cleanup(void);		static void pmc_cleanup(void);
static int pmc_detach_process(struct proc p, struct pmc pm);		static int pmc_detach_process(struct proc p, struct pmc pm);
static int pmc_detach_one_process(struct proc p, struct pmc pm,		static int pmc_detach_one_process(struct proc p, struct pmc pm,
int flags);		int flags);
static void pmc_destroy_owner_descriptor(struct pmc_owner *po);		static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
static void pmc_destroy_pmc_descriptor(struct pmc *pm);		static void pmc_destroy_pmc_descriptor(struct pmc *pm);
static void pmc_destroy_process_descriptor(struct pmc_process *pp);		static void pmc_destroy_process_descriptor(struct pmc_process *pp);
static struct pmc_owner pmc_find_owner_descriptor(struct proc p);		static struct pmc_owner pmc_find_owner_descriptor(struct proc p);
Show All 15 Lines
static void pmc_process_csw_out(struct thread *td);		static void pmc_process_csw_out(struct thread *td);
static void pmc_process_exit(void arg, struct proc p);		static void pmc_process_exit(void arg, struct proc p);
static void pmc_process_fork(void arg, struct proc p1,		static void pmc_process_fork(void arg, struct proc p1,
struct proc *p2, int n);		struct proc *p2, int n);
static void pmc_process_samples(int cpu, int soft);		static void pmc_process_samples(int cpu, int soft);
static void pmc_release_pmc_descriptor(struct pmc *pmc);		static void pmc_release_pmc_descriptor(struct pmc *pmc);
static void pmc_process_thread_add(struct thread *td);		static void pmc_process_thread_add(struct thread *td);
static void pmc_process_thread_delete(struct thread *td);		static void pmc_process_thread_delete(struct thread *td);
		static void pmc_process_thread_userret(struct thread *td);
static void pmc_remove_owner(struct pmc_owner *po);		static void pmc_remove_owner(struct pmc_owner *po);
static void pmc_remove_process_descriptor(struct pmc_process *pp);		static void pmc_remove_process_descriptor(struct pmc_process *pp);
static void pmc_restore_cpu_binding(struct pmc_binding *pb);		static void pmc_restore_cpu_binding(struct pmc_binding *pb);
static void pmc_save_cpu_binding(struct pmc_binding *pb);		static void pmc_save_cpu_binding(struct pmc_binding *pb);
static void pmc_select_cpu(int cpu);		static void pmc_select_cpu(int cpu);
static int pmc_start(struct pmc *pm);		static int pmc_start(struct pmc *pm);
static int pmc_stop(struct pmc *pm);		static int pmc_stop(struct pmc *pm);
static int pmc_syscall_handler(struct thread td, void syscall_args);		static int pmc_syscall_handler(struct thread td, void syscall_args);
▲ Show 20 Lines • Show All 646 Lines • ▼ Show 20 Lines
#ifdef INVARIANTS		#ifdef INVARIANTS
/* Confirm that the per-thread values at this row index are cleared. */		/* Confirm that the per-thread values at this row index are cleared. */
if (PMC_TO_MODE(pm) == PMC_MODE_TS) {		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
mtx_lock_spin(pp->pp_tdslock);		mtx_lock_spin(pp->pp_tdslock);
LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {		LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,		KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
("[pmc,%d] pt_pmcval not cleared for pid=%d at "		("[pmc,%d] pt_pmcval not cleared for pid=%d at "
"ri=%d", __LINE__, pp->pp_proc->p_pid, ri));		"ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
		KASSERT(pt_td->pt_pmcs[ri].pt_pendusamples == 0,
		("[pmc,%d] pt_pendusamples not cleared for pid=%d "
		"at ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
}		}
mtx_unlock_spin(pp->pp_tdslock);		mtx_unlock_spin(pp->pp_tdslock);
}		}
#endif		#endif
}		}

/*		/*
* Removes the association between a target process and a PMC.		* Removes the association between a target process and a PMC.
Show All 26 Lines	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
ri, pm, pp->pp_pmcs[ri].pp_pmc));		ri, pm, pp->pp_pmcs[ri].pp_pmc));

pp->pp_pmcs[ri].pp_pmc = NULL;		pp->pp_pmcs[ri].pp_pmc = NULL;
pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;		pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;

/* Clear the per-thread values at this row index. */		/* Clear the per-thread values at this row index. */
if (PMC_TO_MODE(pm) == PMC_MODE_TS) {		if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
mtx_lock_spin(pp->pp_tdslock);		mtx_lock_spin(pp->pp_tdslock);
LIST_FOREACH(pt, &pp->pp_tds, pt_next)		LIST_FOREACH(pt, &pp->pp_tds, pt_next) {
pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;		pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
		pt->pt_pmcs[ri].pt_pendusamples = 0;
		}
mtx_unlock_spin(pp->pp_tdslock);		mtx_unlock_spin(pp->pp_tdslock);
}		}

/* Remove owner-specific flags */		/* Remove owner-specific flags */
if (pm->pm_owner->po_owner == pp->pp_proc) {		if (pm->pm_owner->po_owner == pp->pp_proc) {
pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;		pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;		pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
}		}
▲ Show 20 Lines • Show All 402 Lines • ▼ Show 20 Lines	if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
if (pt == NULL)		if (pt == NULL)
pt = pmc_find_thread_descriptor(pp, td,		pt = pmc_find_thread_descriptor(pp, td,
PMC_FLAG_NONE);		PMC_FLAG_NONE);

KASSERT(pt != NULL,		KASSERT(pt != NULL,
("[pmc,%d] No thread found for td=%p", __LINE__,		("[pmc,%d] No thread found for td=%p", __LINE__,
td));		td));

		phw->phw_pmcthread = pt;

mtx_pool_lock_spin(pmc_mtxpool, pm);		mtx_pool_lock_spin(pmc_mtxpool, pm);

/*		/*
* If we have a thread descriptor, use the per-thread		* If we have a thread descriptor, use the per-thread
* counter in the descriptor. If not, we will use		* counter in the descriptor. If not, we will use
* a per-process counter.		* a per-process counter.
*		*
* TODO: Remove the per-process "safety net" once		* TODO: Remove the per-process "safety net" once
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines
pmc_process_csw_out(struct thread *td)		pmc_process_csw_out(struct thread *td)
{		{
int cpu;		int cpu;
int64_t tmp;		int64_t tmp;
struct pmc *pm;		struct pmc *pm;
struct proc *p;		struct proc *p;
enum pmc_mode mode;		enum pmc_mode mode;
struct pmc_cpu *pc;		struct pmc_cpu *pc;
		struct pmc_hw *phw;
pmc_value_t newvalue;		pmc_value_t newvalue;
unsigned int adjri, ri;		unsigned int adjri, ri;
struct pmc_process *pp;		struct pmc_process *pp;
struct pmc_thread *pt = NULL;		struct pmc_thread *pt = NULL;
struct pmc_classdep *pcd;		struct pmc_classdep *pcd;


/*		/*
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	for (ri = 0; ri < md->pmd_npmc; ri++) {
mode = PMC_TO_MODE(pm);		mode = PMC_TO_MODE(pm);
if (!PMC_IS_VIRTUAL_MODE(mode))		if (!PMC_IS_VIRTUAL_MODE(mode))
continue; /* not a process virtual PMC */		continue; /* not a process virtual PMC */

KASSERT(PMC_TO_ROWINDEX(pm) == ri,		KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] ri mismatch pmc(%d) ri(%d)",		("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
__LINE__, PMC_TO_ROWINDEX(pm), ri));		__LINE__, PMC_TO_ROWINDEX(pm), ri));

		phw = pc->pc_hwpmcs[ri];

/*		/*
* Change desired state, and then stop if not stalled.		* Change desired state, and then stop if not stalled.
* This two-step dance should avoid race conditions where		* This two-step dance should avoid race conditions where
* an interrupt re-enables the PMC after this code has		* an interrupt re-enables the PMC after this code has
* already checked the pm_stalled flag.		* already checked the pm_stalled flag.
*/		*/
CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);		CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);
if (!CPU_ISSET(cpu, &pm->pm_stalled))		if (!CPU_ISSET(cpu, &pm->pm_stalled))
Show All 19 Lines	if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {

pcd->pcd_read_pmc(cpu, adjri, &newvalue);		pcd->pcd_read_pmc(cpu, adjri, &newvalue);

if (mode == PMC_MODE_TS) {		if (mode == PMC_MODE_TS) {
PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",		PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
cpu, ri, newvalue);		cpu, ri, newvalue);

if (pt == NULL)		if (pt == NULL)
pt = pmc_find_thread_descriptor(pp, td,		pt = phw->phw_pmcthread;
PMC_FLAG_NONE);

KASSERT(pt != NULL,		KASSERT(pt != NULL,
("[pmc,%d] No thread found for td=%p",		("[pmc,%d] No thread found for td=%p",
__LINE__, td));		__LINE__, td));

mtx_pool_lock_spin(pmc_mtxpool, pm);		mtx_pool_lock_spin(pmc_mtxpool, pm);

/*		/*
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Lines	pmc_process_thread_delete(struct thread *td)
pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);		pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);

if (pmc != NULL)		if (pmc != NULL)
pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,		pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
td, PMC_FLAG_REMOVE));		td, PMC_FLAG_REMOVE));
}		}

/*		/*
		* A userret() call for a thread.
		*/
		static void
		pmc_process_thread_userret(struct thread *td)
		{
		struct pmc *pm;
		struct pmc_process *pp;
		struct pmc_thread *pmc_td;
		struct pmc_classdep *pcd;
		uint32_t num_samples;
		int adjri, cpu, needast, ri;

		needast = 0;

		/* Find our process and thread. */
		pp = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);

		if (pp == NULL)
		return;

		pmc_td = pmc_find_thread_descriptor(pp, td, PMC_FLAG_NONE);

		if (pmc_td == NULL)
		return;

		/* We may touch td->td_flags and td->td_pflags. */
		thread_lock(td);

		/* Don't switch CPUs while we are processing samples. */
		critical_enter();

		cpu = td->td_oncpu;

		KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
		("[pmc,%d] wierd CPU id %d", __LINE__, cpu));

		/*
		* Loop through the current PMCs, looking for any that care about
		* our process.
		*/
		for (ri = 0; ri < md->pmd_npmc; ri++) {
		pcd = pmc_ri_to_classdep(md, ri, &adjri);
		KASSERT(pcd != NULL,
		("[pmc,%d] null pcd ri=%d", __LINE__, ri));
		(void) (*pcd->pcd_get_config)(cpu,adjri,&pm);

		if (pm == NULL \|\| (pm->pm_flags & PMC_F_USERCALLCHAIN) == 0)
		continue;

		#ifdef INVARIANTS
		{
		struct pmc_target *pt;
		int found = FALSE;

		LIST_FOREACH(pt, &pm->pm_targets, pt_next) {
		if (pt->pt_process->pp_proc == td->td_proc) {
		found = TRUE;
		break;
		}
		}

		KASSERT(found,
		("[pmc,%d] called to process user call chain "
		"for td=%p and found pm=%p not for our process",
		__LINE__, td, pm));
		}
		#endif

		/* Determine how many (if any) samples we owe. */
		num_samples = atomic_readandclear_32(
		&pmc_td->pt_pmcs[ri].pt_pendusamples);
		if (num_samples == 0)
		continue;

		/* Record a sample. */
		if (pmc_add_normal_sample(cpu, PMC_UR, pm, NULL, TRUE,
		num_samples) == 0)
		needast++;
		}

		critical_exit();

		if (needast > 0)
		curthread->td_flags \|= TDF_ASTPENDING;

		thread_unlock(td);
		}

		/*
* A mapping change for a process.		* A mapping change for a process.
*/		*/

static void		static void
pmc_process_mmap(struct thread td, struct pmckern_map_in pkm)		pmc_process_mmap(struct thread td, struct pmckern_map_in pkm)
{		{
int ri;		int ri;
pid_t pid;		pid_t pid;
▲ Show 20 Lines • Show All 306 Lines • ▼ Show 20 Lines	const char *pmc_hooknames[] = {
"UNUSED2",		"UNUSED2",
"MMAP",		"MMAP",
"MUNMAP",		"MUNMAP",
"CALLCHAIN-NMI",		"CALLCHAIN-NMI",
"CALLCHAIN-SOFT",		"CALLCHAIN-SOFT",
"SOFTSAMPLING",		"SOFTSAMPLING",
"THR-CREATE",		"THR-CREATE",
"THR-EXIT",		"THR-EXIT",
		"THR-USERRET",
};		};
#endif		#endif

static int		static int
pmc_hook_handler(struct thread td, int function, void arg)		pmc_hook_handler(struct thread td, int function, void arg)
{		{
int cpu;		int cpu;
		#ifdef INVARIANTS
		int ncallchains;
		#endif

PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,		PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
pmc_hooknames[function], arg);		pmc_hooknames[function], arg);

switch (function)		switch (function)
{		{

/*		/*
▲ Show 20 Lines • Show All 139 Lines • ▼ Show 20 Lines	case PMC_FN_DO_SAMPLES:
* may find nothing to do if "pmc_process_samples()"		* may find nothing to do if "pmc_process_samples()"
* had already processed the interrupt). We don't		* had already processed the interrupt). We don't
* lose the interrupt sample.		* lose the interrupt sample.
*/		*/
cpu = PCPU_GET(cpuid);		cpu = PCPU_GET(cpuid);
CPU_CLR_ATOMIC(cpu, &pmc_cpumask);		CPU_CLR_ATOMIC(cpu, &pmc_cpumask);
pmc_process_samples(cpu, PMC_HR);		pmc_process_samples(cpu, PMC_HR);
pmc_process_samples(cpu, PMC_SR);		pmc_process_samples(cpu, PMC_SR);
		pmc_process_samples(cpu, PMC_UR);
break;		break;

case PMC_FN_MMAP:		case PMC_FN_MMAP:
sx_assert(&pmc_sx, SX_LOCKED);		sx_assert(&pmc_sx, SX_LOCKED);
pmc_process_mmap(td, (struct pmckern_map_in *) arg);		pmc_process_mmap(td, (struct pmckern_map_in *) arg);
break;		break;

case PMC_FN_MUNMAP:		case PMC_FN_MUNMAP:
sx_assert(&pmc_sx, SX_LOCKED);		sx_assert(&pmc_sx, SX_LOCKED);
pmc_process_munmap(td, (struct pmckern_map_out *) arg);		pmc_process_munmap(td, (struct pmckern_map_out *) arg);
break;		break;

case PMC_FN_USER_CALLCHAIN:		case PMC_FN_USER_CALLCHAIN:
/*		/*
* Record a call chain.		* Record a call chain.
*/		*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));		__LINE__));

pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,		cpu = PCPU_GET(cpuid);
		#ifdef INVARIANTS
		ncallchains =
		#endif
		pmc_capture_user_callchain(cpu, PMC_HR,
(struct trapframe *) arg);		(struct trapframe *) arg);

		KASSERT(ncallchains > 0,
		("[pmc,%d] cpu %d didn't find a sample to collect",
		__LINE__, cpu));

		KASSERT(td->td_pinned == 1,
		("[pmc,%d] invalid td_pinned value", __LINE__));

		sched_unpin(); /* Can migrate safely now. */

td->td_pflags &= ~TDP_CALLCHAIN;		td->td_pflags &= ~TDP_CALLCHAIN;
break;		break;

case PMC_FN_USER_CALLCHAIN_SOFT:		case PMC_FN_USER_CALLCHAIN_SOFT:
/*		/*
* Record a call chain.		* Record a call chain.
*/		*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));		__LINE__));
pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR,
		cpu = PCPU_GET(cpuid);
		#ifdef INVARIANTS
		ncallchains =
		#endif
		pmc_capture_user_callchain(cpu, PMC_SR,
(struct trapframe *) arg);		(struct trapframe *) arg);
		#ifdef INVARIANTS
		ncallchains +=
		#endif
		pmc_capture_user_callchain(cpu, PMC_UR,
		(struct trapframe *) arg);

		KASSERT(ncallchains > 0,
		("[pmc,%d] cpu %d didn't find a sample to collect",
		__LINE__, cpu));

		KASSERT(td->td_pinned == 1,
		("[pmc,%d] invalid td_pinned value", __LINE__));

		sched_unpin(); /* Can migrate safely now. */

td->td_pflags &= ~TDP_CALLCHAIN;		td->td_pflags &= ~TDP_CALLCHAIN;
break;		break;

case PMC_FN_SOFT_SAMPLING:		case PMC_FN_SOFT_SAMPLING:
/*		/*
* Call soft PMC sampling intr.		* Call soft PMC sampling intr.
*/		*/
pmc_soft_intr((struct pmckern_soft *) arg);		pmc_soft_intr((struct pmckern_soft *) arg);
break;		break;

case PMC_FN_THR_CREATE:		case PMC_FN_THR_CREATE:
pmc_process_thread_add(td);		pmc_process_thread_add(td);
break;		break;

case PMC_FN_THR_EXIT:		case PMC_FN_THR_EXIT:
KASSERT(td == curthread, ("[pmc,%d] td != curthread",		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));		__LINE__));
pmc_process_thread_delete(td);		pmc_process_thread_delete(td);
break;		break;

		case PMC_FN_THR_USERRET:
		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
		__LINE__));
		pmc_process_thread_userret(td);
		break;

default:		default:
#ifdef HWPMC_DEBUG		#ifdef HWPMC_DEBUG
KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));		KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
#endif		#endif
break;		break;

}		}

▲ Show 20 Lines • Show All 1,499 Lines • ▼ Show 20 Lines	if (PMC_IS_SYSTEM_MODE(mode)) {
}		}
}		}

/*		/*
* Look for valid values for 'pm_flags'		* Look for valid values for 'pm_flags'
*/		*/

if ((pa.pm_flags & ~(PMC_F_DESCENDANTS \| PMC_F_LOG_PROCCSW \|		if ((pa.pm_flags & ~(PMC_F_DESCENDANTS \| PMC_F_LOG_PROCCSW \|
PMC_F_LOG_PROCEXIT \| PMC_F_CALLCHAIN)) != 0) {		PMC_F_LOG_PROCEXIT \| PMC_F_CALLCHAIN \|
		PMC_F_USERCALLCHAIN)) != 0) {
error = EINVAL;		error = EINVAL;
break;		break;
}		}

		/* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
		if ((pa.pm_flags & (PMC_F_CALLCHAIN \| PMC_F_USERCALLCHAIN)) ==
		PMC_F_USERCALLCHAIN) {
		error = EINVAL;
		break;
		}

		/* PMC_F_USERCALLCHAIN is only valid with PMC_MODE_TS */
		if ((pa.pm_flags & PMC_F_USERCALLCHAIN) &&
		mode != PMC_MODE_TS) {
		error = EINVAL;
		break;
		}

/* process logging options are not allowed for system PMCs */		/* process logging options are not allowed for system PMCs */
if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &		if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
(PMC_F_LOG_PROCCSW \| PMC_F_LOG_PROCEXIT))) {		(PMC_F_LOG_PROCCSW \| PMC_F_LOG_PROCEXIT))) {
error = EINVAL;		error = EINVAL;
break;		break;
}		}

/*		/*
▲ Show 20 Lines • Show All 711 Lines • ▼ Show 20 Lines	pmc_post_callchain_callback(void)
* capture completes.		* capture completes.
*/		*/
sched_pin();		sched_pin();

return;		return;
}		}

/*		/*
* Interrupt processing.
*
* Find a free slot in the per-cpu array of samples and capture the		* Find a free slot in the per-cpu array of samples and capture the
* current callchain there. If a sample was successfully added, a bit		* current callchain there. If a sample was successfully added, a bit
* is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook		* is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
* needs to be invoked from the clock handler.		* needs to be invoked from the clock handler.
*		*
* This function is meant to be called from an NMI handler. It cannot		* This function is meant to be called from an NMI handler. It cannot
* use any of the locking primitives supplied by the OS.		* use any of the locking primitives supplied by the OS.
*/		*/

int		static int
pmc_process_interrupt(int cpu, int ring, struct pmc pm, struct trapframe tf,		pmc_add_normal_sample(int cpu, int ring, struct pmc pm, struct trapframe tf,
int inuserspace)		int inuserspace, uint32_t count)
{		{
int error, callchaindepth;		int error, callchaindepth;
struct thread *td;		struct thread *td;
struct pmc_sample *ps;		struct pmc_sample *ps;
struct pmc_samplebuffer *psb;		struct pmc_samplebuffer *psb;

error = 0;		error = 0;

▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines	if (!inuserspace) {
callchaindepth, tf);		callchaindepth, tf);
} else {		} else {
pmc_post_callchain_callback();		pmc_post_callchain_callback();
callchaindepth = PMC_SAMPLE_INUSE;		callchaindepth = PMC_SAMPLE_INUSE;
}		}
}		}

ps->ps_nsamples = callchaindepth; /* mark entry as in use */		ps->ps_nsamples = callchaindepth; /* mark entry as in use */
		ps->ps_count = count;

/* increment write pointer, modulo ring buffer size */		/* increment write pointer, modulo ring buffer size */
ps++;		ps++;
if (ps == psb->ps_fence)		if (ps == psb->ps_fence)
psb->ps_write = psb->ps_samples;		psb->ps_write = psb->ps_samples;
else		else
psb->ps_write = ps;		psb->ps_write = ps;

done:		done:
/* mark CPU as needing processing */		/* mark CPU as needing processing */
if (callchaindepth != PMC_SAMPLE_INUSE)		if (callchaindepth != PMC_SAMPLE_INUSE)
CPU_SET_ATOMIC(cpu, &pmc_cpumask);		CPU_SET_ATOMIC(cpu, &pmc_cpumask);

return (error);		return (error);
}		}

/*		/*
		* Interrupt processing for a user call chain while in kernel space.
		*
		* Find the thread descriptor and increment the number of samples we
		* have deferred while in kernel space. Just before the return to user-
		* space, we will capture the callchain to record these.
		*
		* This function is meant to be called from an NMI handler. It cannot
		* use any of the locking primitives supplied by the OS. However, we
		* can use the pmc_thread pointer we saved during context switch in.
		*/

		static void
		pmc_add_useronly_sample(int cpu, struct pmc *pm)
		{
		struct pmc_thread *pmc_td;
		struct thread *td;
		char *status;
		uint32_t num_outstanding = 0;

		status = "(uspc) ";

		/* Ignore user call chain requests without a valid process. */
		if ((td = curthread) == NULL \|\| td->td_proc == NULL) {
		PMCDBG4(SAM,INT,1,"(notd) cpu=%d pm=%p um=%d n=%u", cpu, pm, 0,
		num_outstanding);
		return;
		}

		/* Get the saved PMC thread descriptor. */
		pmc_td = pmc_pcpu[cpu]->pc_hwpmcs[PMC_TO_ROWINDEX(pm)]->phw_pmcthread;

		if (pmc_td == NULL) {
		PMCDBG4(SAM,INT,1,"(nopmctd) cpu=%d pm=%p um=%d n=%u", cpu, pm,
		0, num_outstanding);
		return;
		}

		/* Increment the number of pending user-space samples. */
		num_outstanding = atomic_fetchadd_32(
		&pmc_td->pt_pmcs[PMC_TO_ROWINDEX(pm)].pt_pendusamples, 1) + 1;

		PMCDBG4(SAM,INT,1,"(uspc) cpu=%d pm=%p um=%d n=%u", cpu, pm, 0,
		num_outstanding);

		return;
		}

		/*
		* Interrupt processing.
		*
		* This function is meant to be called from an NMI handler. It cannot
		* use any of the locking primitives supplied by the OS.
		*/

		int
		pmc_process_interrupt(int cpu, int ring, struct pmc pm, struct trapframe tf,
		int inuserspace)
		{
		if ((pm->pm_flags & PMC_F_USERCALLCHAIN) != 0 && !inuserspace) {
		pmc_add_useronly_sample(cpu, pm);
		return (0);
		}
		return (pmc_add_normal_sample(cpu, ring, pm, tf, inuserspace, 1));
		}

		/*
* Capture a user call chain. This function will be called from ast()		* Capture a user call chain. This function will be called from ast()
* before control returns to userland and before the process gets		* before control returns to userland and before the process gets
* rescheduled.		* rescheduled.
*/		*/

static void		static int
pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)		pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
{		{
struct pmc *pm;		struct pmc *pm;
struct thread *td;		struct thread *td;
struct pmc_sample ps, ps_end;		struct pmc_sample ps, ps_end;
struct pmc_samplebuffer *psb;		struct pmc_samplebuffer *psb;
#ifdef INVARIANTS		#ifdef INVARIANTS
int ncallchains;		int ncallchains;
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines
#endif		#endif

next:		next:
/* increment the pointer, modulo sample ring size */		/* increment the pointer, modulo sample ring size */
if (++ps == psb->ps_fence)		if (++ps == psb->ps_fence)
ps = psb->ps_samples;		ps = psb->ps_samples;
} while (ps != ps_end);		} while (ps != ps_end);

KASSERT(ncallchains > 0,
("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
cpu));

KASSERT(td->td_pinned == 1,
("[pmc,%d] invalid td_pinned value", __LINE__));
sched_unpin(); /* Can migrate safely now. */

/* mark CPU as needing processing */		/* mark CPU as needing processing */
CPU_SET_ATOMIC(cpu, &pmc_cpumask);		CPU_SET_ATOMIC(cpu, &pmc_cpumask);

return;		#ifdef INVARIANTS
		/* The return value only matters when INVARIANTS is defined. */
		return (ncallchains);
		#else
		return (0);
		#endif
}		}

/*		/*
* Process saved PC samples.		* Process saved PC samples.
*/		*/

static void		static void
pmc_process_samples(int cpu, int ring)		pmc_process_samples(int cpu, int ring)
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
break;		break;
}		}

PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,		PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
pm, ps->ps_nsamples, ps->ps_flags,		pm, ps->ps_nsamples, ps->ps_flags,
(int) (psb->ps_write - psb->ps_samples),		(int) (psb->ps_write - psb->ps_samples),
(int) (psb->ps_read - psb->ps_samples));		(int) (psb->ps_read - psb->ps_samples));

		while (ps->ps_count-- != 0) {
/*		/*
* If this is a process-mode PMC that is attached to		* If this is a process-mode PMC that is attached to
* its owner, and if the PC is in user mode, update		* its owner, and if the PC is in user mode, update
* profiling statistics like timer-based profiling		* profiling statistics like timer-based profiling
* would have done.		* would have done.
		*
		* Otherwise, this is either a sampling-mode PMC that
		* is attached to a different process than its owner,
		* or a system-wide sampling PMC. Dispatch a log
		* entry to the PMC's owner process.
*/		*/
if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {		if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
if (ps->ps_flags & PMC_CC_F_USERSPACE) {		if (ps->ps_flags & PMC_CC_F_USERSPACE) {
td = FIRST_THREAD_IN_PROC(po->po_owner);		td = FIRST_THREAD_IN_PROC(po->po_owner);
addupc_intr(td, ps->ps_pc[0], 1);		addupc_intr(td, ps->ps_pc[0], 1);
}		}
goto entrydone;		} else
		pmclog_process_callchain(pm, ps);
}		}

/*
* Otherwise, this is either a sampling mode PMC that
* is attached to a different process than its owner,
* or a system-wide sampling PMC. Dispatch a log
* entry to the PMC's owner process.
*/
pmclog_process_callchain(pm, ps);

entrydone:		entrydone:
ps->ps_nsamples = 0; /* mark entry as free */		ps->ps_nsamples = 0; /* mark entry as free */
atomic_subtract_rel_int(&pm->pm_runcount, 1);		atomic_subtract_rel_int(&pm->pm_runcount, 1);

/* increment read pointer, modulo sample size */		/* increment read pointer, modulo sample size */
if (++ps == psb->ps_fence)		if (++ps == psb->ps_fence)
psb->ps_read = psb->ps_samples;		psb->ps_read = psb->ps_samples;
else		else
▲ Show 20 Lines • Show All 588 Lines • ▼ Show 20 Lines	for (cpu = 0; cpu < maxcpu; cpu++) {
sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *		sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, M_WAITOK\|M_ZERO);		sizeof(uintptr_t), M_PMC, M_WAITOK\|M_ZERO);

for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains +		ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth);		(n * pmc_callchaindepth);

pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;		pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;

		sb = malloc(sizeof(struct pmc_samplebuffer) +
		pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
		M_WAITOK\|M_ZERO);
		sb->ps_read = sb->ps_write = sb->ps_samples;
		sb->ps_fence = sb->ps_samples + pmc_nsamples;

		KASSERT(pmc_pcpu[cpu] != NULL,
		("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));

		sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
		sizeof(uintptr_t), M_PMC, M_WAITOK\|M_ZERO);

		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
		ps->ps_pc = sb->ps_callchains +
		(n * pmc_callchaindepth);

		pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
}		}

/* allocate space for the row disposition array */		/* allocate space for the row disposition array */
pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,		pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
M_PMC, M_WAITOK\|M_ZERO);		M_PMC, M_WAITOK\|M_ZERO);

/* mark all PMCs as available */		/* mark all PMCs as available */
for (n = 0; n < (int) md->pmd_npmc; n++)		for (n = 0; n < (int) md->pmd_npmc; n++)
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines	for (cpu = 0; cpu < maxcpu; cpu++) {
if (!pmc_cpu_is_active(cpu))		if (!pmc_cpu_is_active(cpu))
continue;		continue;
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,		("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
cpu));		cpu));
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,		("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
cpu));		cpu));
		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
		("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
		cpu));
free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);		free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);		free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);		free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);		free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
		free(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
		free(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
free(pmc_pcpu[cpu], M_PMC);		free(pmc_pcpu[cpu], M_PMC);
}		}

free(pmc_pcpu, M_PMC);		free(pmc_pcpu, M_PMC);
pmc_pcpu = NULL;		pmc_pcpu = NULL;

free(pmc_pcpu_saved, M_PMC);		free(pmc_pcpu_saved, M_PMC);
pmc_pcpu_saved = NULL;		pmc_pcpu_saved = NULL;
▲ Show 20 Lines • Show All 63 Lines • Show Last 20 Lines