D15155.id41776.diff
No OneTemporary
Actions

Size

30 KB

Referenced Files

None

Subscribers

None

D15155.id41776.diff
View Options

	Index: sys/dev/hwpmc/hwpmc_amd.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_amd.c
	+++ sys/dev/hwpmc/hwpmc_amd.c
	@@ -694,8 +694,10 @@
	wrmsr(evsel, config);
	}

	- atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (retval)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	PMCDBG1(MDP,INT,2, "retval=%d", retval);
	return (retval);
	Index: sys/dev/hwpmc/hwpmc_core.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_core.c
	+++ sys/dev/hwpmc/hwpmc_core.c
	@@ -2831,8 +2831,10 @@
	if (found_interrupt)
	lapic_reenable_pmc();

	- atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (found_interrupt)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	return (found_interrupt);
	}
	@@ -2955,8 +2957,10 @@
	if (found_interrupt)
	lapic_reenable_pmc();

	- atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (found_interrupt)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	return (found_interrupt);
	}
	Index: sys/dev/hwpmc/hwpmc_e500.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_e500.c
	+++ sys/dev/hwpmc/hwpmc_e500.c
	@@ -616,8 +616,10 @@
	e500_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
	}

	- atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (retval)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	/* Re-enable PERF exceptions. */
	if (retval)
	Index: sys/dev/hwpmc/hwpmc_logging.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_logging.c
	+++ sys/dev/hwpmc/hwpmc_logging.c
	@@ -50,7 +50,9 @@
	#include <sys/pmckern.h>
	#include <sys/pmclog.h>
	#include <sys/proc.h>
	+#include <sys/sched.h>
	#include <sys/signalvar.h>
	+#include <sys/smp.h>
	#include <sys/syscallsubr.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>
	@@ -116,8 +118,10 @@
	/* reserve LEN bytes of space and initialize the entry header */
	#define _PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do { \
	uint32_t *_le; \
	- int _len = roundup((LEN), sizeof(uint32_t)); \
	+ int _len = roundup((LEN), sizeof(uint32_t)); \
	+ spinlock_enter(); \
	if ((_le = pmclog_reserve((PO), _len)) == NULL) { \
	+ spinlock_exit(); \
	ACTION; \
	} \
	*_le = _PMCLOG_TO_HEADER(TYPE,_len); \
	@@ -139,7 +143,13 @@
	#define PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0)

	#define PMCLOG_DESPATCH(PO) \
	- pmclog_release((PO)); \
	+ pmclog_release((PO)); \
	+ spinlock_exit(); \
	+ } while (0)
	+
	+#define PMCLOG_DESPATCH_SYNC(PO) \
	+ pmclog_schedule_io((PO)); \
	+ spinlock_exit(); \
	} while (0)


	@@ -191,6 +201,7 @@
	static void pmclog_release(struct pmc_owner *po);
	static uint32_t pmclog_reserve(struct pmc_owner po, int length);
	static void pmclog_schedule_io(struct pmc_owner *po);
	+static void pmclog_schedule_all(struct pmc_owner *po);
	static void pmclog_stop_kthread(struct pmc_owner *po);

	/*
	@@ -206,9 +217,7 @@
	{
	struct pmclog_buffer *plb;

	- mtx_assert(&po->po_mtx, MA_OWNED);
	-
	- KASSERT(po->po_curbuf == NULL,
	+ KASSERT(po->po_curbuf[curcpu] == NULL,
	("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));

	mtx_lock_spin(&pmc_bufferlist_mtx);
	@@ -227,12 +236,12 @@
	plb->plb_base, plb->plb_fence));
	#endif

	- po->po_curbuf = plb;
	+ po->po_curbuf[curcpu] = plb;

	/* update stats */
	- atomic_add_int(&pmc_stats.pm_buffer_requests, 1);
	+ counter_u64_add(pmc_stats.pm_buffer_requests, 1);
	if (plb == NULL)
	- atomic_add_int(&pmc_stats.pm_buffer_requests_failed, 1);
	+ counter_u64_add(pmc_stats.pm_buffer_requests_failed, 1);

	return (plb ? 0 : ENOMEM);
	}
	@@ -460,18 +469,21 @@
	static void
	pmclog_release(struct pmc_owner *po)
	{
	- KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
	+ struct pmclog_buffer *plb;
	+
	+ spinlock_enter();
	+ plb = po->po_curbuf[curcpu];
	+ KASSERT(plb->plb_ptr >= plb->plb_base,
	("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
	- po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
	- KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
	+ po, plb->plb_ptr, plb->plb_base));
	+ KASSERT(plb->plb_ptr <= plb->plb_fence,
	("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
	- po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
	+ po, plb->plb_ptr, plb->plb_fence));

	/* schedule an I/O if we've filled a buffer */
	- if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence)
	+ if (plb->plb_ptr >= plb->plb_fence)
	pmclog_schedule_io(po);
	-
	- mtx_unlock_spin(&po->po_mtx);
	+ spinlock_exit();

	PMCDBG1(LOG,REL,1, "po=%p", po);
	}
	@@ -492,36 +504,32 @@
	uintptr_t newptr, oldptr;
	uint32_t *lh;
	struct timespec ts;
	+ struct pmclog_buffer plb, *pplb;

	PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length);

	KASSERT(length % sizeof(uint32_t) == 0,
	("[pmclog,%d] length not a multiple of word size", __LINE__));

	- mtx_lock_spin(&po->po_mtx);
	-
	/* No more data when shutdown in progress. */
	- if (po->po_flags & PMC_PO_SHUTDOWN) {
	- mtx_unlock_spin(&po->po_mtx);
	+ if (po->po_flags & PMC_PO_SHUTDOWN)
	return (NULL);
	- }

	- if (po->po_curbuf == NULL)
	- if (pmclog_get_buffer(po) != 0) {
	- mtx_unlock_spin(&po->po_mtx);
	- return (NULL);
	- }
	+ pplb = &po->po_curbuf[curcpu];
	+ if (*pplb == NULL && pmclog_get_buffer(po) != 0)
	+ goto fail;

	- KASSERT(po->po_curbuf != NULL,
	+ KASSERT(*pplb != NULL,
	("[pmclog,%d] po=%p no current buffer", __LINE__, po));

	- KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base &&
	- po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
	+ plb = *pplb;
	+ KASSERT(plb->plb_ptr >= plb->plb_base &&
	+ plb->plb_ptr <= plb->plb_fence,
	("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
	- __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
	- po->po_curbuf->plb_fence));
	+ __LINE__, po, plb->plb_ptr, plb->plb_base,
	+ plb->plb_fence));

	- oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
	+ oldptr = (uintptr_t) plb->plb_ptr;
	newptr = oldptr + length;

	KASSERT(oldptr != (uintptr_t) NULL,
	@@ -531,8 +539,8 @@
	* If we have space in the current buffer, return a pointer to
	* available space with the PO structure locked.
	*/
	- if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) {
	- po->po_curbuf->plb_ptr = (char *) newptr;
	+ if (newptr <= (uintptr_t) plb->plb_fence) {
	+ plb->plb_ptr = (char *) newptr;
	goto done;
	}

	@@ -542,24 +550,23 @@
	*/
	pmclog_schedule_io(po);

	- if (pmclog_get_buffer(po) != 0) {
	- mtx_unlock_spin(&po->po_mtx);
	- return (NULL);
	- }
	+ if (pmclog_get_buffer(po) != 0)
	+ goto fail;

	- KASSERT(po->po_curbuf != NULL,
	+ plb = *pplb;
	+ KASSERT(plb != NULL,
	("[pmclog,%d] po=%p no current buffer", __LINE__, po));

	- KASSERT(po->po_curbuf->plb_ptr != NULL,
	+ KASSERT(plb->plb_ptr != NULL,
	("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__));

	- KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base &&
	- po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
	+ KASSERT(plb->plb_ptr == plb->plb_base &&
	+ plb->plb_ptr <= plb->plb_fence,
	("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
	- __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
	- po->po_curbuf->plb_fence));
	+ __LINE__, po, plb->plb_ptr, plb->plb_base,
	+ plb->plb_fence));

	- oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
	+ oldptr = (uintptr_t) plb->plb_ptr;

	done:
	lh = (uint32_t *) oldptr;
	@@ -568,6 +575,8 @@
	*lh++ = ts.tv_sec & 0xFFFFFFFF;
	*lh++ = ts.tv_nsec & 0xFFFFFFF;
	return ((uint32_t *) oldptr);
	+ fail:
	+ return (NULL);
	}

	/*
	@@ -579,26 +588,28 @@
	static void
	pmclog_schedule_io(struct pmc_owner *po)
	{
	- KASSERT(po->po_curbuf != NULL,
	- ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
	+ struct pmclog_buffer *plb;

	- KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
	+ plb = po->po_curbuf[curcpu];
	+ po->po_curbuf[curcpu] = NULL;
	+ KASSERT(plb != NULL,
	+ ("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
	+ KASSERT(plb->plb_ptr >= plb->plb_base,
	("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
	- po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
	- KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
	+ po, plb->plb_ptr, plb->plb_base));
	+ KASSERT(plb->plb_ptr <= plb->plb_fence,
	("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
	- po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
	+ po, plb->plb_ptr, plb->plb_fence));

	PMCDBG1(LOG,SIO, 1, "po=%p", po);

	- mtx_assert(&po->po_mtx, MA_OWNED);
	-
	/*
	* Add the current buffer to the tail of the buffer list and
	* wakeup the helper.
	*/
	- TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next);
	- po->po_curbuf = NULL;
	+ mtx_lock_spin(&po->po_mtx);
	+ TAILQ_INSERT_TAIL(&po->po_logbuffers, plb, plb_next);
	+ mtx_unlock_spin(&po->po_mtx);
	wakeup_one(po);
	}

	@@ -673,7 +684,7 @@
	sizeof(struct pmclog_initialize));
	PMCLOG_EMIT32(PMC_VERSION);
	PMCLOG_EMIT32(md->pmd_cputype);
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);

	return (0);

	@@ -726,14 +737,21 @@
	TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
	mtx_unlock_spin(&pmc_bufferlist_mtx);
	}
	-
	- /* return the 'current' buffer to the global pool */
	- if ((lb = po->po_curbuf) != NULL) {
	- PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
	- mtx_lock_spin(&pmc_bufferlist_mtx);
	- TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
	- mtx_unlock_spin(&pmc_bufferlist_mtx);
	+ for (int i = 0; i < mp_ncpus; i++) {
	+ thread_lock(curthread);
	+ sched_bind(curthread, i);
	+ thread_unlock(curthread);
	+ /* return the 'current' buffer to the global pool */
	+ if ((lb = po->po_curbuf[curcpu]) != NULL) {
	+ PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
	+ mtx_lock_spin(&pmc_bufferlist_mtx);
	+ TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
	+ mtx_unlock_spin(&pmc_bufferlist_mtx);
	+ }
	}
	+ thread_lock(curthread);
	+ sched_unbind(curthread);
	+ thread_unlock(curthread);

	/* drop a reference to the fd */
	if (po->po_file != NULL) {
	@@ -754,7 +772,6 @@
	pmclog_flush(struct pmc_owner *po)
	{
	int error;
	- struct pmclog_buffer *lb;

	PMCDBG1(LOG,FLS,1, "po=%p", po);

	@@ -776,23 +793,43 @@
	goto error;
	}

	- /*
	- * Schedule the current buffer if any and not empty.
	- */
	- mtx_lock_spin(&po->po_mtx);
	- lb = po->po_curbuf;
	- if (lb && lb->plb_ptr != lb->plb_base) {
	- pmclog_schedule_io(po);
	- } else
	- error = ENOBUFS;
	- mtx_unlock_spin(&po->po_mtx);
	-
	+ pmclog_schedule_all(po);
	error:
	mtx_unlock(&pmc_kthread_mtx);

	return (error);
	}

	+static void
	+pmclog_schedule_one_cond(void *arg)
	+{
	+ struct pmc_owner *po = arg;
	+ struct pmclog_buffer *plb;
	+
	+ spinlock_enter();
	+ plb = po->po_curbuf[curcpu];
	+ if (plb && plb->plb_ptr != plb->plb_base)
	+ pmclog_schedule_io(po);
	+ spinlock_exit();
	+}
	+
	+static void
	+pmclog_schedule_all(struct pmc_owner *po)
	+{
	+ /*
	+ * Schedule the current buffer if any and not empty.
	+ */
	+ for (int i = 0; i < mp_ncpus; i++) {
	+ thread_lock(curthread);
	+ sched_bind(curthread, i);
	+ thread_unlock(curthread);
	+ pmclog_schedule_one_cond(po);
	+ }
	+ thread_lock(curthread);
	+ sched_unbind(curthread);
	+ thread_unlock(curthread);
	+}
	+
	int
	pmclog_close(struct pmc_owner *po)
	{
	@@ -806,19 +843,14 @@
	/*
	* Schedule the current buffer.
	*/
	- mtx_lock_spin(&po->po_mtx);
	- if (po->po_curbuf)
	- pmclog_schedule_io(po);
	- else
	- wakeup_one(po);
	- mtx_unlock_spin(&po->po_mtx);
	+ pmclog_schedule_all(po);
	+ wakeup_one(po);

	/*
	* Initiate shutdown: no new data queued,
	* thread will close file on last block.
	*/
	po->po_flags \|= PMC_PO_SHUTDOWN;
	-
	mtx_unlock(&pmc_kthread_mtx);

	return (0);
	@@ -851,7 +883,7 @@
	pmclog_process_closelog(struct pmc_owner *po)
	{
	PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog));
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);
	}

	void
	@@ -915,14 +947,14 @@
	else
	PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
	pmc_soft_ev_release(ps);
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);
	} else {
	PMCLOG_RESERVE(po, PMCALLOCATE,
	sizeof(struct pmclog_pmcallocate));
	PMCLOG_EMIT32(pm->pm_id);
	PMCLOG_EMIT32(pm->pm_event);
	PMCLOG_EMIT32(pm->pm_flags);
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);
	}
	}

	@@ -943,7 +975,7 @@
	PMCLOG_EMIT32(pm->pm_id);
	PMCLOG_EMIT32(pid);
	PMCLOG_EMITSTRING(path, pathlen);
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);
	}

	void
	@@ -958,7 +990,7 @@
	PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach));
	PMCLOG_EMIT32(pm->pm_id);
	PMCLOG_EMIT32(pid);
	- PMCLOG_DESPATCH(po);
	+ PMCLOG_DESPATCH_SYNC(po);
	}

	/*
	Index: sys/dev/hwpmc/hwpmc_mod.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_mod.c
	+++ sys/dev/hwpmc/hwpmc_mod.c
	@@ -138,7 +138,7 @@
	pmc_kld_unload_tag;

	/* Module statistics */
	-struct pmc_op_getdriverstats pmc_stats;
	+struct pmc_driverstats pmc_stats;

	/* Machine/processor dependent operations */
	static struct pmc_mdep *md;
	@@ -1260,7 +1260,7 @@
	continue;

	/* increment PMC runcount */
	- atomic_add_rel_int(&pm->pm_runcount, 1);
	+ counter_u64_add(pm->pm_runcount, 1);

	/* configure the HWPMC we are going to use. */
	pcd = pmc_ri_to_classdep(md, ri, &adjri);
	@@ -1311,10 +1311,10 @@

	/* If a sampling mode PMC, reset stalled state. */
	if (PMC_TO_MODE(pm) == PMC_MODE_TS)
	- CPU_CLR_ATOMIC(cpu, &pm->pm_stalled);
	+ pm->pm_pcpu_state[cpu].pps_stalled = 0;

	/* Indicate that we desire this to run. */
	- CPU_SET_ATOMIC(cpu, &pm->pm_cpustate);
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 1;

	/* Start the PMC. */
	pcd->pcd_start_pmc(cpu, adjri);
	@@ -1417,12 +1417,12 @@
	* an interrupt re-enables the PMC after this code has
	* already checked the pm_stalled flag.
	*/
	- CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);
	- if (!CPU_ISSET(cpu, &pm->pm_stalled))
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	+ if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
	pcd->pcd_stop_pmc(cpu, adjri);

	/* reduce this PMC's runcount */
	- atomic_subtract_rel_int(&pm->pm_runcount, 1);
	+ counter_u64_add(pm->pm_runcount, -1);

	/*
	* If this PMC is associated with this process,
	@@ -1537,7 +1537,7 @@
	/* Inform owners of all system-wide sampling PMCs. */
	LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	- pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
	+ pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);

	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
	goto done;
	@@ -1993,7 +1993,7 @@
	* had already processed the interrupt). We don't
	* lose the interrupt sample.
	*/
	- CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask);
	+ DPCPU_SET(pmc_sampled, 0);
	pmc_process_samples(PCPU_GET(cpuid), PMC_HR);
	pmc_process_samples(PCPU_GET(cpuid), PMC_SR);
	break;
	@@ -2191,7 +2191,8 @@
	struct pmc *pmc;

	pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK\|M_ZERO);
	-
	+ pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
	+ pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK\|M_ZERO);
	PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);

	return pmc;
	@@ -2212,10 +2213,12 @@
	("[pmc,%d] destroying pmc with targets", __LINE__));
	KASSERT(pm->pm_owner == NULL,
	("[pmc,%d] destroying pmc attached to an owner", __LINE__));
	- KASSERT(pm->pm_runcount == 0,
	- ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
	- pm->pm_runcount));
	+ KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
	+ ("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
	+ counter_u64_fetch(pm->pm_runcount)));

	+ counter_u64_free(pm->pm_runcount);
	+ free(pm->pm_pcpu_state, M_PMC);
	free(pm, M_PMC);
	}

	@@ -2231,13 +2234,13 @@
	* Loop (with a forced context switch) till the PMC's runcount
	* comes down to zero.
	*/
	- while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
	+ while (counter_u64_fetch(pm->pm_runcount) > 0) {
	#ifdef HWPMC_DEBUG
	maxloop--;
	KASSERT(maxloop > 0,
	- ("[pmc,%d] (ri%d, rc%d) waiting too long for "
	+ ("[pmc,%d] (ri%d, rc%ld) waiting too long for "
	"pmc to be free", __LINE__,
	- PMC_TO_ROWINDEX(pm), pm->pm_runcount));
	+ PMC_TO_ROWINDEX(pm), counter_u64_fetch(pm->pm_runcount)));
	#endif
	pmc_force_context_switch();
	}
	@@ -2295,9 +2298,9 @@
	pmc_select_cpu(cpu);

	/* switch off non-stalled CPUs */
	- CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	if (pm->pm_state == PMC_STATE_RUNNING &&
	- !CPU_ISSET(cpu, &pm->pm_stalled)) {
	+ pm->pm_pcpu_state[cpu].pps_stalled == 0) {

	phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];

	@@ -2735,10 +2738,10 @@
	pm->pm_sc.pm_initial)) == 0) {
	/* If a sampling mode PMC, reset stalled state. */
	if (PMC_IS_SAMPLING_MODE(mode))
	- CPU_CLR_ATOMIC(cpu, &pm->pm_stalled);
	+ pm->pm_pcpu_state[cpu].pps_stalled = 0;

	/* Indicate that we desire this to run. Start it. */
	- CPU_SET_ATOMIC(cpu, &pm->pm_cpustate);
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 1;
	error = pcd->pcd_start_pmc(cpu, adjri);
	}
	critical_exit();
	@@ -2802,7 +2805,7 @@
	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	- CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	critical_enter();
	if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
	error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
	@@ -2884,7 +2887,7 @@
	pmc_op_to_name[op], arg);

	error = 0;
	- atomic_add_int(&pmc_stats.pm_syscalls, 1);
	+ counter_u64_add(pmc_stats.pm_syscalls, 1);

	switch (op) {

	@@ -3063,8 +3066,16 @@
	case PMC_OP_GETDRIVERSTATS:
	{
	struct pmc_op_getdriverstats gms;
	-
	- bcopy(&pmc_stats, &gms, sizeof(gms));
	+#define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
	+ CFETCH(gms, pmc_stats, pm_intr_ignored);
	+ CFETCH(gms, pmc_stats, pm_intr_processed);
	+ CFETCH(gms, pmc_stats, pm_intr_bufferfull);
	+ CFETCH(gms, pmc_stats, pm_syscalls);
	+ CFETCH(gms, pmc_stats, pm_syscall_errors);
	+ CFETCH(gms, pmc_stats, pm_buffer_requests);
	+ CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
	+ CFETCH(gms, pmc_stats, pm_log_sweeps);
	+#undef CFETCH
	error = copyout(&gms, arg, sizeof(gms));
	}
	break;
	@@ -4040,7 +4051,7 @@
	sx_xunlock(&pmc_sx);
	done_syscall:
	if (error)
	- atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
	+ counter_u64_add(pmc_stats.pm_syscall_errors, 1);

	return (error);
	}
	@@ -4115,8 +4126,8 @@

	ps = psb->ps_write;
	if (ps->ps_nsamples) { /* in use, reader hasn't caught up */
	- CPU_SET_ATOMIC(cpu, &pm->pm_stalled);
	- atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
	+ pm->pm_pcpu_state[cpu].pps_stalled = 1;
	+ counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
	PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
	cpu, pm, (void *) tf, inuserspace,
	(int) (psb->ps_write - psb->ps_samples),
	@@ -4133,11 +4144,11 @@
	(int) (psb->ps_write - psb->ps_samples),
	(int) (psb->ps_read - psb->ps_samples));

	- KASSERT(pm->pm_runcount >= 0,
	- ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
	- pm->pm_runcount));
	+ KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
	+ ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	+ counter_u64_fetch(pm->pm_runcount)));

	- atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */
	+ counter_u64_add(pm->pm_runcount, 1); /* hold onto PMC */

	ps->ps_pmc = pm;
	if ((td = curthread) && td->td_proc)
	@@ -4180,7 +4191,7 @@
	done:
	/* mark CPU as needing processing */
	if (callchaindepth != PMC_SAMPLE_INUSE)
	- CPU_SET_ATOMIC(cpu, &pmc_cpumask);
	+ DPCPU_SET(pmc_sampled, 1);

	return (error);
	}
	@@ -4244,8 +4255,8 @@
	("[pmc,%d] Retrieving callchain for PMC that doesn't "
	"want it", __LINE__));

	- KASSERT(pm->pm_runcount > 0,
	- ("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount));
	+ KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	+ ("[pmc,%d] runcount %ld", __LINE__, counter_u64_fetch(pm->pm_runcount)));

	/*
	* Retrieve the callchain and mark the sample buffer
	@@ -4275,9 +4286,7 @@
	sched_unpin(); /* Can migrate safely now. */

	/* mark CPU as needing processing */
	- CPU_SET_ATOMIC(cpu, &pmc_cpumask);
	-
	- return;
	+ DPCPU_SET(pmc_sampled, 1);
	}

	/*
	@@ -4309,9 +4318,9 @@

	pm = ps->ps_pmc;

	- KASSERT(pm->pm_runcount > 0,
	- ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
	- pm->pm_runcount));
	+ KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	+ ("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	+ counter_u64_fetch(pm->pm_runcount)));

	po = pm->pm_owner;

	@@ -4326,7 +4335,7 @@
	/* If there is a pending AST wait for completion */
	if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
	/* Need a rescan at a later time. */
	- CPU_SET_ATOMIC(cpu, &pmc_cpumask);
	+ DPCPU_SET(pmc_sampled, 1);
	break;
	}

	@@ -4359,7 +4368,7 @@

	entrydone:
	ps->ps_nsamples = 0; /* mark entry as free */
	- atomic_subtract_rel_int(&pm->pm_runcount, 1);
	+ counter_u64_add(pm->pm_runcount, -1);

	/* increment read pointer, modulo sample size */
	if (++ps == psb->ps_fence)
	@@ -4368,7 +4377,7 @@
	psb->ps_read = ps;
	}

	- atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
	+ counter_u64_add(pmc_stats.pm_log_sweeps, 1);

	/* Do not re-enable stalled PMCs if we failed to process any samples */
	if (n == 0)
	@@ -4390,11 +4399,11 @@
	if (pm == NULL \|\| /* !cfg'ed */
	pm->pm_state != PMC_STATE_RUNNING \|\| /* !active */
	!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) \|\| /* !sampling */
	- !CPU_ISSET(cpu, &pm->pm_cpustate) \|\| /* !desired */
	- !CPU_ISSET(cpu, &pm->pm_stalled)) /* !stalled */
	+ !pm->pm_pcpu_state[cpu].pps_cpustate \|\| /* !desired */
	+ !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
	continue;

	- CPU_CLR_ATOMIC(cpu, &pm->pm_stalled);
	+ pm->pm_pcpu_state[cpu].pps_stalled = 0;
	(*pcd->pcd_start_pmc)(cpu, adjri);
	}
	}
	@@ -4513,9 +4522,9 @@
	("[pmc,%d] pm %p != pp_pmcs[%d] %p",
	__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));

	- KASSERT(pm->pm_runcount > 0,
	- ("[pmc,%d] bad runcount ri %d rc %d",
	- __LINE__, ri, pm->pm_runcount));
	+ KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	+ ("[pmc,%d] bad runcount ri %d rc %ld",
	+ __LINE__, ri, counter_u64_fetch(pm->pm_runcount)));

	/*
	* Change desired state, and then stop if not
	@@ -4524,9 +4533,9 @@
	* the PMC after this code has already checked
	* the pm_stalled flag.
	*/
	- if (CPU_ISSET(cpu, &pm->pm_cpustate)) {
	- CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate);
	- if (!CPU_ISSET(cpu, &pm->pm_stalled)) {
	+ if (pm->pm_pcpu_state[cpu].pps_cpustate) {
	+ pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	+ if (!pm->pm_pcpu_state[cpu].pps_stalled) {
	(void) pcd->pcd_stop_pmc(cpu, adjri);
	pcd->pcd_read_pmc(cpu, adjri,
	&newvalue);
	@@ -4540,9 +4549,9 @@
	}
	}

	- atomic_subtract_rel_int(&pm->pm_runcount,1);
	+ counter_u64_add(pm->pm_runcount, -1);

	- KASSERT((int) pm->pm_runcount >= 0,
	+ KASSERT((int) counter_u64_fetch(pm->pm_runcount) >= 0,
	("[pmc,%d] runcount is %d", __LINE__, ri));

	(void) pcd->pcd_config_pmc(cpu, adjri, NULL);
	@@ -4895,6 +4904,14 @@

	maxcpu = pmc_cpu_max();

	+ pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK\|M_ZERO);
	+ pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK\|M_ZERO);
	/* allocate space for the per-cpu array */
	pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
	M_WAITOK\|M_ZERO);
	@@ -5048,7 +5065,8 @@
	PMCDBG0(MOD,INI,0, "cleanup");

	/* switch off sampling */
	- CPU_ZERO(&pmc_cpumask);
	+ CPU_FOREACH(cpu)
	+ DPCPU_ID_SET(cpu, pmc_sampled, 0);
	pmc_intr = NULL;

	sx_xlock(&pmc_sx);
	@@ -5164,6 +5182,14 @@
	free(pmc_pcpu[cpu], M_PMC);
	}

	+ counter_u64_free(pmc_stats.pm_intr_ignored);
	+ counter_u64_free(pmc_stats.pm_intr_processed);
	+ counter_u64_free(pmc_stats.pm_intr_bufferfull);
	+ counter_u64_free(pmc_stats.pm_syscalls);
	+ counter_u64_free(pmc_stats.pm_syscall_errors);
	+ counter_u64_free(pmc_stats.pm_buffer_requests);
	+ counter_u64_free(pmc_stats.pm_buffer_requests_failed);
	+ counter_u64_free(pmc_stats.pm_log_sweeps);
	free(pmc_pcpu, M_PMC);
	pmc_pcpu = NULL;

	Index: sys/dev/hwpmc/hwpmc_mpc7xxx.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_mpc7xxx.c
	+++ sys/dev/hwpmc/hwpmc_mpc7xxx.c
	@@ -710,9 +710,10 @@
	/* reload count. */
	mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
	}
	-
	- atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (retval)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	/* Re-enable PERF exceptions. */
	if (retval)
	Index: sys/dev/hwpmc/hwpmc_piv.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_piv.c
	+++ sys/dev/hwpmc/hwpmc_piv.c
	@@ -1545,8 +1545,10 @@
	if (did_interrupt)
	lapic_reenable_pmc();

	- atomic_add_int(did_interrupt ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (did_interrupt)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	return (did_interrupt);
	}
	Index: sys/dev/hwpmc/hwpmc_ppc970.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_ppc970.c
	+++ sys/dev/hwpmc/hwpmc_ppc970.c
	@@ -528,8 +528,10 @@
	ppc970_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
	}

	- atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (retval)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	/* Re-enable PERF exceptions. */
	if (retval)
	Index: sys/dev/hwpmc/hwpmc_ppro.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_ppro.c
	+++ sys/dev/hwpmc/hwpmc_ppro.c
	@@ -725,8 +725,10 @@
	if (retval)
	lapic_reenable_pmc();

	- atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (retval)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	/* restart counters that can be restarted */
	P6_SYNC_CTR_STATE(pc);
	Index: sys/dev/hwpmc/hwpmc_soft.c
	===================================================================
	--- sys/dev/hwpmc/hwpmc_soft.c
	+++ sys/dev/hwpmc/hwpmc_soft.c
	@@ -441,9 +441,10 @@
	} else
	pc->soft_values[ri]++;
	}
	-
	- atomic_add_int(processed ? &pmc_stats.pm_intr_processed :
	- &pmc_stats.pm_intr_ignored, 1);
	+ if (processed)
	+ counter_u64_add(pmc_stats.pm_intr_processed, 1);
	+ else
	+ counter_u64_add(pmc_stats.pm_intr_ignored, 1);

	return (processed);
	}
	Index: sys/sys/pmc.h
	===================================================================
	--- sys/sys/pmc.h
	+++ sys/sys/pmc.h
	@@ -36,7 +36,7 @@
	#define _SYS_PMC_H_

	#include <dev/hwpmc/pmc_events.h>
	-
	+#include <sys/counter.h>
	#include <machine/pmc_mdep.h>
	#include <machine/profile.h>

	@@ -553,6 +553,19 @@
	* Retrieve pmc(4) driver-wide statistics.
	*/

	+struct pmc_driverstats {
	+ counter_u64_t pm_intr_ignored; /* #interrupts ignored */
	+ counter_u64_t pm_intr_processed; /* #interrupts processed */
	+ counter_u64_t pm_intr_bufferfull; /* #interrupts with ENOSPC */
	+ counter_u64_t pm_syscalls; /* #syscalls */
	+ counter_u64_t pm_syscall_errors; /* #syscalls with errors */
	+ counter_u64_t pm_buffer_requests; /* #buffer requests */
	+ counter_u64_t pm_buffer_requests_failed; /* #failed buffer requests */
	+ counter_u64_t pm_log_sweeps; /* #sample buffer processing
	+ passes */
	+};
	+
	+
	struct pmc_op_getdriverstats {
	unsigned int pm_intr_ignored; /* #interrupts ignored */
	unsigned int pm_intr_processed; /* #interrupts processed */
	@@ -625,9 +638,9 @@

	#define PMC_HASH_SIZE 1024
	#define PMC_MTXPOOL_SIZE 2048
	-#define PMC_LOG_BUFFER_SIZE 4
	+#define PMC_LOG_BUFFER_SIZE 512
	#define PMC_NLOGBUFFERS 1024
	-#define PMC_NSAMPLES 1024
	+#define PMC_NSAMPLES 4096
	#define PMC_CALLCHAIN_DEPTH 32

	#define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "."
	@@ -701,7 +714,10 @@
	* field is '0'.
	*
	*/
	-
	+struct pmc_pcpu_state {
	+ uint8_t pps_stalled;
	+ uint8_t pps_cpustate;
	+} __aligned(CACHE_LINE_SIZE);
	struct pmc {
	LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */
	LIST_ENTRY(pmc) pm_next; /* owner's list */
	@@ -735,13 +751,13 @@
	pmc_value_t pm_initial; /* counting PMC modes */
	} pm_sc;

	- volatile cpuset_t pm_stalled; /* marks stalled sampling PMCs */
	+ struct pmc_pcpu_state *pm_pcpu_state;
	volatile cpuset_t pm_cpustate; /* CPUs where PMC should be active */
	uint32_t pm_caps; /* PMC capabilities */
	enum pmc_event pm_event; /* event being measured */
	uint32_t pm_flags; /* additional flags PMC_F_... */
	struct pmc_owner pm_owner; / owner thread state */
	- int pm_runcount; /* #cpus currently on */
	+ counter_u64_t pm_runcount; /* #cpus currently on */
	enum pmc_state pm_state; /* current PMC state */
	uint32_t pm_overflowcnt; /* count overflow interrupts */

	@@ -816,11 +832,11 @@
	struct proc po_owner; / owner proc */
	uint32_t po_flags; /* (k) flags PMC_PO_* */
	struct proc po_kthread; / (k) helper kthread */
	- struct pmclog_buffer po_curbuf; / current log buffer */
	struct file po_file; / file reference */
	int po_error; /* recorded error */
	short po_sscount; /* # SS PMCs owned */
	short po_logprocmaps; /* global mappings done */
	+ struct pmclog_buffer po_curbuf[MAXCPU]; / current log buffer */
	};

	#define PMC_PO_OWNS_LOGFILE 0x00000001 /* has a log file */
	@@ -1012,7 +1028,7 @@
	extern struct pmc_cpu **pmc_pcpu;

	/* driver statistics */
	-extern struct pmc_op_getdriverstats pmc_stats;
	+extern struct pmc_driverstats pmc_stats;

	#if defined(HWPMC_DEBUG)
	#include <sys/ktr.h>
	Index: sys/sys/pmckern.h
	===================================================================
	--- sys/sys/pmckern.h
	+++ sys/sys/pmckern.h
	@@ -165,7 +165,7 @@
	extern struct sx pmc_sx;

	/* Per-cpu flags indicating availability of sampling data */
	-extern volatile cpuset_t pmc_cpumask;
	+DPCPU_DECLARE(uint8_t, pmc_sampled);

	/* Count of system-wide sampling PMCs in existence */
	extern volatile int pmc_ss_count;
	@@ -220,7 +220,7 @@
	#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0)

	/* Check if a CPU has recorded samples. */
	-#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(CPU_ISSET(C, &pmc_cpumask)))
	+#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(DPCPU_ID_GET((C), pmc_sampled)))

	/*
	* Helper functions.

File Metadata

Mime Type: text/plain
Expires: Mon, Oct 13, 3:50 AM (13 h, 33 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 23655711
Default Alt Text: D15155.id41776.diff (30 KB)

D15155.id41776.diffNo OneTemporaryActions

D15155.id41776.diffView Options

File Metadata

Event Timeline

D15155.id41776.diff
No OneTemporary
Actions

D15155.id41776.diff
View Options