Page MenuHomeFreeBSD
Files F170939133

D59370.diff
No OneTemporary
Actions

D59370.diff
View Options

diff --git a/sys/dev/hwpmc/hwpmc_logging.c b/sys/dev/hwpmc/hwpmc_logging.c
--- a/sys/dev/hwpmc/hwpmc_logging.c
+++ b/sys/dev/hwpmc/hwpmc_logging.c
@@ -64,6 +64,8 @@
#include <machine/clock.h>
#endif
+#include "hwpmc_pmu.h"
+
#define curdomain PCPU_GET(domain)
/*
@@ -903,7 +905,7 @@
return (0);
}
-void
+bool
pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
{
int n, recordlen;
@@ -917,7 +919,10 @@
ps->ps_nsamples * sizeof(uintfptr_t);
po = pm->pm_owner;
flags = PMC_CALLCHAIN_TO_CPUFLAGS(ps->ps_cpu,ps->ps_flags);
- PMCLOG_RESERVE_SAFE(po, PMCLOG_TYPE_CALLCHAIN, recordlen, ps->ps_tsc);
+ if (pmc_test_callchain_log_should_fail(pm))
+ return (false);
+ _PMCLOG_RESERVE_SAFE(po, PMCLOG_TYPE_CALLCHAIN, recordlen,
+ return (false), ps->ps_tsc);
PMCLOG_EMIT32(ps->ps_pid);
PMCLOG_EMIT32(ps->ps_tid);
PMCLOG_EMIT32(pm->pm_handle);
@@ -925,6 +930,7 @@
for (n = 0; n < ps->ps_nsamples; n++)
PMCLOG_EMITADDR(ps->ps_pc[n]);
PMCLOG_DESPATCH_SAFE(po);
+ return (true);
}
void
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -198,6 +198,13 @@
#ifdef HWPMC_DEBUG
static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
static int pmc_debugflags_parse(char *newstr, char *fence);
+static int pmc_test_gate_sysctl_handler(SYSCTL_HANDLER_ARGS);
+static int pmc_test_handle_failpoint_sysctl_handler(SYSCTL_HANDLER_ARGS);
+static int pmc_test_group_state_sysctl_handler(SYSCTL_HANDLER_ARGS);
+static int pmc_test_inject_kernel_sample_sysctl_handler(
+ SYSCTL_HANDLER_ARGS);
+static int pmc_test_residual_state_sysctl_handler(SYSCTL_HANDLER_ARGS);
+static int pmc_test_sample_counts_sysctl_handler(SYSCTL_HANDLER_ARGS);
#endif
static void pmc_multipart_add(struct pmc_sample *ps, int type,
@@ -340,14 +347,723 @@
"cpu version string");
#ifdef HWPMC_DEBUG
+struct pmc_test_gate {
+ u_int ptg_request;
+ u_int ptg_ack;
+};
+
+struct pmc_test_handle_failpoint {
+ u_int pth_request;
+ u_int pth_ack;
+};
+
+struct pmc_test_sample_counts {
+ uint64_t ptsc_handle;
+ uint32_t ptsc_pid;
+ uint32_t ptsc_tid;
+ uint64_t ptsc_accepted;
+ uint64_t ptsc_emitted;
+ uint64_t ptsc_dropped;
+ uint64_t ptsc_run_refs;
+};
+
+struct pmc_test_group_state {
+ uint64_t ptgs_handle;
+ uint32_t ptgs_running;
+ uint32_t ptgs_assigned;
+ uint32_t ptgs_sys_listed;
+ uint32_t ptgs_sscounted;
+ uint32_t ptgs_nevents;
+ uint32_t ptgs_running_members;
+ uint32_t ptgs_stopped_members;
+ uint32_t ptgs_allocated_members;
+};
+
+enum pmc_test_residual_operation {
+ PMC_TEST_RESIDUAL_GET_SAVED = 1,
+ PMC_TEST_RESIDUAL_GET_LIVE,
+ PMC_TEST_RESIDUAL_SET_LIVE,
+ PMC_TEST_RESIDUAL_SET_SAVED
+};
+
+enum pmc_test_residual_kind {
+ PMC_TEST_RESIDUAL_UNINITIALIZED,
+ PMC_TEST_RESIDUAL_VALID,
+ PMC_TEST_RESIDUAL_OVERFLOW_PENDING
+};
+
+struct pmc_test_residual_query {
+ uint64_t ptrq_handle;
+ uint64_t ptrq_value;
+ uint32_t ptrq_pid;
+ uint32_t ptrq_tid;
+ uint32_t ptrq_operation;
+ uint32_t ptrq_state;
+ uint32_t ptrq_assigned;
+ uint32_t ptrq_reserved;
+};
+
struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
char pmc_debugstr[PMC_DEBUG_STRSIZE];
+static counter_u64_t pmc_test_counters[PMC_TEST_COUNTER_COUNT];
+static u_int pmc_test_failpoints[PMC_TEST_FAILPOINT_COUNT];
+static struct pmc_test_gate pmc_test_hold_resident;
+static struct pmc_test_gate pmc_test_hold_evicted;
+static struct pmc_test_gate pmc_test_pause_descendants;
+static struct pmc_test_gate pmc_test_descendants_exit;
+static struct pmc_test_gate pmc_test_pause_system_start;
+static struct pmc_test_gate pmc_test_pause_sample;
+static struct pmc_test_handle_failpoint pmc_test_fail_callchain_log;
+static u_int pmc_test_pause_descendants_targets;
+static u_int pmc_test_pause_system_start_count;
+static u_int pmc_test_pause_sample_schedule_out_ack;
+
TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
sizeof(pmc_debugstr));
SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE,
0, 0, pmc_debugflags_sysctl_handler, "A",
"debug flags");
+
+SYSCTL_NODE(_kern_hwpmc, OID_AUTO, test, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+ "HWPMC deterministic test support");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_pmc_targets, CTLFLAG_RD,
+ &pmc_test_counters[PMC_TEST_LIVE_PMC_TARGETS],
+ "live pmc_target objects");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_group_targets, CTLFLAG_RD,
+ &pmc_test_counters[PMC_TEST_LIVE_GROUP_TARGETS],
+ "live pmu_group_target objects");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_target_processes,
+ CTLFLAG_RD, &pmc_test_counters[PMC_TEST_LIVE_TARGET_PROCESSES],
+ "live target pmc_process descriptors");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_residual_entries,
+ CTLFLAG_RD, &pmc_test_counters[PMC_TEST_LIVE_RESIDUAL_ENTRIES],
+ "live saved per-thread residual entries");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_rotation_refs, CTLFLAG_RD,
+ &pmc_test_counters[PMC_TEST_LIVE_ROTATION_REFS],
+ "active rotation references");
+SYSCTL_COUNTER_U64(_kern_hwpmc_test, OID_AUTO, live_run_refs, CTLFLAG_RD,
+ &pmc_test_counters[PMC_TEST_LIVE_RUN_REFS],
+ "nonzero PMC run-count references");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, fail_group_target_alloc_after,
+ CTLFLAG_RW, &pmc_test_failpoints[PMC_TEST_FAIL_GROUP_TARGET_ALLOC], 0,
+ "allow N group-target allocations, then fail one; UINT_MAX disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, fail_target_link_alloc_after,
+ CTLFLAG_RW, &pmc_test_failpoints[PMC_TEST_FAIL_TARGET_LINK_ALLOC], 0,
+ "allow N target-link allocations, then fail one; UINT_MAX disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, fail_attach_authorization_after,
+ CTLFLAG_RW, &pmc_test_failpoints[PMC_TEST_FAIL_ATTACH_AUTHORIZATION], 0,
+ "allow N target authorizations, then fail one; UINT_MAX disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, fail_system_start_after,
+ CTLFLAG_RW, &pmc_test_failpoints[PMC_TEST_FAIL_SYSTEM_START], 0,
+ "allow N system-group member starts, then fail one; UINT_MAX disables");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, hold_group_resident,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &pmc_test_hold_resident, 0,
+ pmc_test_gate_sysctl_handler, "IU",
+ "group ID to keep resident; zero disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, hold_group_resident_ack, CTLFLAG_RD,
+ &pmc_test_hold_resident.ptg_ack, 0,
+ "group ID observed resident and held");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, hold_group_evicted,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &pmc_test_hold_evicted, 0,
+ pmc_test_gate_sysctl_handler, "IU",
+ "group ID to keep evicted; zero disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, hold_group_evicted_ack, CTLFLAG_RD,
+ &pmc_test_hold_evicted.ptg_ack, 0,
+ "group ID observed evicted and held");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, pause_descendants_attach,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &pmc_test_pause_descendants, 0,
+ pmc_test_gate_sysctl_handler, "IU",
+ "group ID to pause after descendant enumeration; zero releases");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_descendants_attach_ack,
+ CTLFLAG_RD, &pmc_test_pause_descendants.ptg_ack, 0,
+ "group ID paused after descendant enumeration");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_descendants_target_count,
+ CTLFLAG_RD, &pmc_test_pause_descendants_targets, 0,
+ "number of targets in the paused descendants snapshot");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, descendants_exit_pid,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &pmc_test_descendants_exit, 0,
+ pmc_test_gate_sysctl_handler, "IU",
+ "pid whose process-exit entry should be acknowledged; zero disables");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, descendants_exit_ack, CTLFLAG_RD,
+ &pmc_test_descendants_exit.ptg_ack, 0,
+ "pid observed after the process-exit P_HWPMC snapshot");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, pause_system_start_after_first,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+ &pmc_test_pause_system_start, 0, pmc_test_gate_sysctl_handler, "IU",
+ "group ID to pause after its first system member starts; zero releases");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_system_start_after_first_ack,
+ CTLFLAG_RD, &pmc_test_pause_system_start.ptg_ack, 0,
+ "group ID paused after its first system member started");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_system_start_member_count,
+ CTLFLAG_RD, &pmc_test_pause_system_start_count, 0,
+ "number of system-group members started at the pause");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, pause_sample_worker,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &pmc_test_pause_sample, 0,
+ pmc_test_gate_sysctl_handler, "IU",
+ "group ID whose accepted sample worker is paused; zero releases");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_sample_worker_ack, CTLFLAG_RD,
+ &pmc_test_pause_sample.ptg_ack, 0,
+ "group ID with an accepted sample held before worker consumption");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, pause_sample_schedule_out_ack,
+ CTLFLAG_RD, &pmc_test_pause_sample_schedule_out_ack, 0,
+ "paused-sample group ID observed in system schedule-out");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, fail_callchain_log_handle,
+ CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+ &pmc_test_fail_callchain_log, 0,
+ pmc_test_handle_failpoint_sysctl_handler, "IU",
+ "PMC handle whose next callchain log reservation fails");
+SYSCTL_UINT(_kern_hwpmc_test, OID_AUTO, fail_callchain_log_handle_ack,
+ CTLFLAG_RD, &pmc_test_fail_callchain_log.pth_ack, 0,
+ "PMC handle whose callchain log reservation was failed");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, sample_counts,
+ CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
+ pmc_test_sample_counts_sysctl_handler, "S,pmc_test_sample_counts",
+ "write a PMC handle and read accepted/emitted/dropped sample counts");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, inject_kernel_sample,
+ CTLTYPE_U64 | CTLFLAG_WR | CTLFLAG_MPSAFE, 0, 0,
+ pmc_test_inject_kernel_sample_sysctl_handler, "QU",
+ "write a sampling PMC handle to emit one kernel-mode callchain record");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, group_state,
+ CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
+ pmc_test_group_state_sysctl_handler, "S,pmc_test_group_state",
+ "write a leader handle and read logical and resident group state");
+SYSCTL_PROC(_kern_hwpmc_test, OID_AUTO, residual_state,
+ CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
+ pmc_test_residual_state_sysctl_handler, "S,pmc_test_residual_query",
+ "write a sampling handle, PID, TID, and operation to inspect or inject "
+ "test residual state");
+
+static int
+pmc_test_gate_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_test_gate *gate;
+ u_int value;
+ int error;
+
+ gate = arg1;
+ value = atomic_load_acq_int(&gate->ptg_request);
+ error = sysctl_handle_32(oidp, &value, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+
+ atomic_store_rel_int(&gate->ptg_ack, 0);
+ if (gate == &pmc_test_pause_descendants)
+ atomic_store_rel_int(&pmc_test_pause_descendants_targets, 0);
+ if (gate == &pmc_test_pause_system_start)
+ atomic_store_rel_int(&pmc_test_pause_system_start_count, 0);
+ if (gate == &pmc_test_pause_sample)
+ atomic_store_rel_int(&pmc_test_pause_sample_schedule_out_ack, 0);
+ atomic_store_rel_int(&gate->ptg_request, value);
+ wakeup(&gate->ptg_request);
+ return (0);
+}
+
+static int
+pmc_test_handle_failpoint_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_test_handle_failpoint *failpoint;
+ u_int value;
+ int error;
+
+ failpoint = arg1;
+ value = atomic_load_acq_int(&failpoint->pth_request);
+ error = sysctl_handle_32(oidp, &value, 0, req);
+ if (error != 0 || req->newptr == NULL)
+ return (error);
+
+ atomic_store_rel_int(&failpoint->pth_ack, PMC_ID_INVALID);
+ atomic_store_rel_int(&failpoint->pth_request, value);
+ return (0);
+}
+
+void
+pmc_test_counter_add(enum pmc_test_counter counter, int64_t delta)
+{
+
+ KASSERT((u_int)counter < PMC_TEST_COUNTER_COUNT,
+ ("invalid hwpmc test counter %d", counter));
+ counter_u64_add(pmc_test_counters[counter], delta);
+}
+
+bool
+pmc_test_should_fail(enum pmc_test_failpoint failpoint)
+{
+ u_int value;
+
+ KASSERT((u_int)failpoint < PMC_TEST_FAILPOINT_COUNT,
+ ("invalid hwpmc test failpoint %d", failpoint));
+ if ((u_int)failpoint >= PMC_TEST_FAILPOINT_COUNT)
+ return (false);
+
+ for (;;) {
+ value = atomic_load_acq_int(&pmc_test_failpoints[failpoint]);
+ if (value == UINT_MAX)
+ return (false);
+ if (value == 0) {
+ if (atomic_cmpset_int(&pmc_test_failpoints[failpoint], 0,
+ UINT_MAX))
+ return (true);
+ continue;
+ }
+ if (atomic_cmpset_int(&pmc_test_failpoints[failpoint], value,
+ value - 1))
+ return (false);
+ }
+}
+
+static bool
+pmc_test_group_gate_match(struct pmc_test_gate *gate, pmu_group_t *pg,
+ bool assigned)
+{
+ u_int group_id;
+
+ if (pg == NULL || pg->pg_assigned != assigned)
+ return (false);
+ group_id = atomic_load_acq_int(&gate->ptg_request);
+ if (group_id == 0 || group_id != pg->pg_id)
+ return (false);
+ atomic_store_rel_int(&gate->ptg_ack, group_id);
+ return (true);
+}
+
+bool
+pmc_test_group_hold_resident(pmu_group_t *pg)
+{
+
+ return (pmc_test_group_gate_match(&pmc_test_hold_resident, pg, true));
+}
+
+bool
+pmc_test_group_hold_evicted(pmu_group_t *pg)
+{
+
+ return (pmc_test_group_gate_match(&pmc_test_hold_evicted, pg, false));
+}
+
+void
+pmc_test_descendants_attach_pause(pmu_group_t *pg, u_int ntargets)
+{
+ u_int group_id;
+ int remaining;
+
+ if (pg == NULL)
+ return;
+ group_id = atomic_load_acq_int(&pmc_test_pause_descendants.ptg_request);
+ if (group_id == 0 || group_id != pg->pg_id)
+ return;
+
+ atomic_store_rel_int(&pmc_test_pause_descendants_targets, ntargets);
+ atomic_store_rel_int(&pmc_test_pause_descendants.ptg_ack, group_id);
+ remaining = 30 * hz;
+ while (atomic_load_acq_int(
+ &pmc_test_pause_descendants.ptg_request) == group_id &&
+ remaining > 0) {
+ (void)tsleep(&pmc_test_pause_descendants.ptg_request, 0,
+ "pmctatt", hz);
+ remaining -= hz;
+ }
+ if (remaining == 0)
+ (void)atomic_cmpset_int(&pmc_test_pause_descendants.ptg_request,
+ group_id, 0);
+}
+
+void
+pmc_test_descendants_exit_observed(struct proc *p)
+{
+ u_int pid;
+
+ pid = atomic_load_acq_int(&pmc_test_descendants_exit.ptg_request);
+ if (pid != 0 && pid == (u_int)p->p_pid)
+ atomic_store_rel_int(&pmc_test_descendants_exit.ptg_ack, pid);
+}
+
+void
+pmc_test_system_start_pause(pmu_group_t *pg, u_int nstarted)
+{
+ u_int group_id;
+ int remaining;
+
+ sx_assert(&pmc_sx, SX_XLOCKED);
+ if (pg == NULL || nstarted != 1)
+ return;
+ group_id =
+ atomic_load_acq_int(&pmc_test_pause_system_start.ptg_request);
+ if (group_id == 0 || group_id != pg->pg_id)
+ return;
+
+ atomic_store_rel_int(&pmc_test_pause_system_start_count, nstarted);
+ atomic_store_rel_int(&pmc_test_pause_system_start.ptg_ack, group_id);
+ remaining = 30 * hz;
+ while (atomic_load_acq_int(
+ &pmc_test_pause_system_start.ptg_request) == group_id &&
+ remaining > 0) {
+ (void)sx_sleep(&pmc_test_pause_system_start.ptg_request, &pmc_sx,
+ 0, "pmctsst", hz);
+ remaining -= hz;
+ }
+ if (remaining == 0)
+ (void)atomic_cmpset_int(
+ &pmc_test_pause_system_start.ptg_request, group_id, 0);
+}
+
+void
+pmc_test_sample_accepted(struct pmc *pm, uint32_t pid, uint32_t tid)
+{
+ pmu_event_t *pe;
+ pmu_group_t *pg;
+ u_int group_id;
+
+ pe = pmu_event_from_pmc(pm);
+ if (pe == NULL || pe->pe_group == NULL)
+ return;
+ pg = pe->pe_group;
+ group_id = atomic_load_acq_int(&pmc_test_pause_sample.ptg_request);
+ if (group_id != 0 && group_id == pg->pg_id) {
+ atomic_store_rel_int(&pe->pe_test_sample_pid, pid);
+ atomic_store_rel_int(&pe->pe_test_sample_tid, tid);
+ atomic_store_rel_int(&pmc_test_pause_sample.ptg_ack, group_id);
+ }
+}
+
+bool
+pmc_test_sample_worker_paused(struct pmc *pm)
+{
+ pmu_group_t *pg;
+ u_int group_id;
+
+ pg = pmu_group_from_pmc(pm);
+ if (pg == NULL)
+ return (false);
+ group_id = atomic_load_acq_int(&pmc_test_pause_sample.ptg_request);
+ return (group_id != 0 && group_id == pg->pg_id &&
+ atomic_load_acq_int(&pmc_test_pause_sample.ptg_ack) == group_id);
+}
+
+void
+pmc_test_sample_schedule_out(pmu_group_t *pg)
+{
+ u_int group_id;
+
+ if (pg == NULL)
+ return;
+ group_id = atomic_load_acq_int(&pmc_test_pause_sample.ptg_request);
+ if (group_id != 0 && group_id == pg->pg_id &&
+ atomic_load_acq_int(&pmc_test_pause_sample.ptg_ack) == group_id)
+ atomic_store_rel_int(&pmc_test_pause_sample_schedule_out_ack,
+ group_id);
+}
+
+bool
+pmc_test_callchain_log_should_fail(struct pmc *pm)
+{
+ u_int handle;
+
+ handle = pm->pm_handle;
+ if (handle == PMC_ID_INVALID ||
+ atomic_load_acq_int(&pmc_test_fail_callchain_log.pth_request) !=
+ handle)
+ return (false);
+ if (!atomic_cmpset_int(&pmc_test_fail_callchain_log.pth_request,
+ handle, PMC_ID_INVALID))
+ return (false);
+ atomic_store_rel_int(&pmc_test_fail_callchain_log.pth_ack, handle);
+ return (true);
+}
+
+void
+pmc_test_sample_dropped(struct pmc *pm)
+{
+ pmu_event_t *pe;
+
+ pe = pmu_event_from_pmc(pm);
+ if (pe != NULL)
+ counter_u64_add(pe->pe_samples_dropped, 1);
+}
+
+void
+pmc_test_sample_done(struct pmc *pm, bool emitted)
+{
+ pmu_event_t *pe;
+
+ pe = pmu_event_from_pmc(pm);
+ if (pe == NULL)
+ return;
+ counter_u64_add(emitted ? pe->pe_samples_emitted :
+ pe->pe_samples_dropped, 1);
+}
+
+static int
+pmc_test_inject_kernel_sample_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_sample sample;
+ struct pmc *pm;
+ uintptr_t pc;
+ uint64_t handle;
+ int error;
+
+ if (req->newptr == NULL || req->newlen != sizeof(handle))
+ return (EINVAL);
+ error = SYSCTL_IN(req, &handle, sizeof(handle));
+ if (error != 0)
+ return (error);
+ if ((uint64_t)(pmc_id_t)handle != handle)
+ return (EINVAL);
+
+ sx_xlock(&pmc_sx);
+ error = pmc_find_pmc((pmc_id_t)handle, &pm);
+ if (error == 0 &&
+ (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) ||
+ pm->pm_state != PMC_STATE_RUNNING ||
+ pm->pm_owner == NULL ||
+ (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) == 0))
+ error = EINVAL;
+ if (error == 0) {
+ pc = (uintptr_t)pmc_test_inject_kernel_sample_sysctl_handler;
+ memset(&sample, 0, sizeof(sample));
+ sample.ps_nsamples = 1;
+ sample.ps_tid = 0;
+ sample.ps_pid = -1;
+ sample.ps_td = curthread;
+ sample.ps_pmc = pm;
+ sample.ps_pc = &pc;
+ spinlock_enter();
+ sample.ps_cpu = curcpu;
+ sample.ps_tsc = pmc_rdtsc();
+ if (!pmclog_process_callchain(pm, &sample))
+ error = ENOMEM;
+ spinlock_exit();
+ }
+ sx_xunlock(&pmc_sx);
+ return (error);
+}
+
+static int
+pmc_test_sample_counts_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_test_sample_counts counts;
+ pmu_event_t *pe;
+ struct pmc *pm;
+ uint64_t handle;
+ int error;
+
+ if (req->newptr == NULL || req->newlen != sizeof(handle))
+ return (EINVAL);
+ error = SYSCTL_IN(req, &handle, sizeof(handle));
+ if (error != 0)
+ return (error);
+ if ((uint64_t)(pmc_id_t)handle != handle)
+ return (EINVAL);
+
+ memset(&counts, 0, sizeof(counts));
+ sx_xlock(&pmc_sx);
+ error = pmc_find_pmc((pmc_id_t)handle, &pm);
+ if (error == 0) {
+ pe = pmu_event_from_pmc(pm);
+ if (pe == NULL)
+ error = EINVAL;
+ else {
+ counts.ptsc_handle = pm->pm_handle;
+ counts.ptsc_pid =
+ atomic_load_acq_int(&pe->pe_test_sample_pid);
+ counts.ptsc_tid =
+ atomic_load_acq_int(&pe->pe_test_sample_tid);
+ counts.ptsc_accepted = counter_u64_fetch(pe->pe_samples);
+ counts.ptsc_emitted =
+ counter_u64_fetch(pe->pe_samples_emitted);
+ counts.ptsc_dropped =
+ counter_u64_fetch(pe->pe_samples_dropped);
+ counts.ptsc_run_refs =
+ counter_u64_fetch(pm->pm_runcount);
+ }
+ }
+ sx_xunlock(&pmc_sx);
+ if (error != 0)
+ return (error);
+ return (SYSCTL_OUT(req, &counts, sizeof(counts)));
+}
+
+static int
+pmc_test_group_state_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_test_group_state state;
+ pmu_event_t *pe;
+ pmu_group_t *pg;
+ struct pmc *pm;
+ uint64_t handle;
+ int error;
+
+ if (req->newptr == NULL || req->newlen != sizeof(handle))
+ return (EINVAL);
+ error = SYSCTL_IN(req, &handle, sizeof(handle));
+ if (error != 0)
+ return (error);
+ if ((uint64_t)(pmc_id_t)handle != handle)
+ return (EINVAL);
+
+ memset(&state, 0, sizeof(state));
+ sx_xlock(&pmc_sx);
+ error = pmc_find_pmc((pmc_id_t)handle, &pm);
+ if (error == 0) {
+ pg = pmu_group_from_pmc(pm);
+ if (pg == NULL || pg->pg_leader == NULL ||
+ pg->pg_leader->pe_pmc != pm)
+ error = EINVAL;
+ else {
+ state.ptgs_handle = pm->pm_handle;
+ state.ptgs_running = pg->pg_running;
+ state.ptgs_assigned = pg->pg_assigned;
+ state.ptgs_sys_listed = pg->pg_sys_listed;
+ state.ptgs_sscounted = pg->pg_sscounted;
+ state.ptgs_nevents = pg->pg_nevents;
+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
+ if (pe->pe_pmc->pm_state == PMC_STATE_RUNNING)
+ state.ptgs_running_members++;
+ else if (pe->pe_pmc->pm_state ==
+ PMC_STATE_STOPPED)
+ state.ptgs_stopped_members++;
+ else if (pe->pe_pmc->pm_state ==
+ PMC_STATE_ALLOCATED)
+ state.ptgs_allocated_members++;
+ }
+ }
+ }
+ sx_xunlock(&pmc_sx);
+ if (error != 0)
+ return (error);
+ return (SYSCTL_OUT(req, &state, sizeof(state)));
+}
+
+static int
+pmc_test_residual_state_sysctl_handler(SYSCTL_HANDLER_ARGS)
+{
+ struct pmc_test_residual_query query;
+ struct pmu_group_target *pgt;
+ struct pmc_process *pp;
+ struct pmc_thread *pt;
+ pmu_event_t *pe;
+ pmu_group_t *pg;
+ struct pmc *pm;
+ pmc_value_t value;
+ int error, ri;
+
+ if (req->newptr == NULL || req->newlen != sizeof(query))
+ return (EINVAL);
+ error = SYSCTL_IN(req, &query, sizeof(query));
+ if (error != 0)
+ return (error);
+ if ((uint64_t)(pmc_id_t)query.ptrq_handle != query.ptrq_handle ||
+ query.ptrq_pid == 0 || query.ptrq_tid == 0 ||
+ query.ptrq_operation < PMC_TEST_RESIDUAL_GET_SAVED ||
+ query.ptrq_operation > PMC_TEST_RESIDUAL_SET_SAVED)
+ return (EINVAL);
+
+ query.ptrq_assigned = 0;
+ query.ptrq_reserved = 0;
+ sx_xlock(&pmc_sx);
+ error = pmc_find_pmc((pmc_id_t)query.ptrq_handle, &pm);
+ if (error != 0)
+ goto out;
+ pe = pmu_event_from_pmc(pm);
+ pg = pmu_group_from_pmc(pm);
+ if (pe == NULL || pg == NULL || PMC_TO_MODE(pm) != PMC_MODE_TS) {
+ error = EINVAL;
+ goto out;
+ }
+
+ pp = NULL;
+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next) {
+ if ((uint32_t)pgt->pgt_pp->pp_proc->p_pid == query.ptrq_pid) {
+ pp = pgt->pgt_pp;
+ break;
+ }
+ }
+ if (pp == NULL) {
+ error = ESRCH;
+ goto out;
+ }
+
+ pt = NULL;
+ mtx_lock_spin(pp->pp_tdslock);
+ LIST_FOREACH(pt, &pp->pp_tds, pt_next) {
+ if ((uint32_t)pt->pt_td->td_tid == query.ptrq_tid)
+ break;
+ }
+ if (pt == NULL) {
+ mtx_unlock_spin(pp->pp_tdslock);
+ error = ESRCH;
+ goto out;
+ }
+
+ query.ptrq_assigned = pg->pg_assigned;
+ if (query.ptrq_operation == PMC_TEST_RESIDUAL_SET_LIVE ||
+ query.ptrq_operation == PMC_TEST_RESIDUAL_SET_SAVED) {
+ switch (query.ptrq_state) {
+ case PMC_TEST_RESIDUAL_UNINITIALIZED:
+ case PMC_TEST_RESIDUAL_OVERFLOW_PENDING:
+ if (query.ptrq_value != 0) {
+ error = EINVAL;
+ goto unlock;
+ }
+ break;
+ case PMC_TEST_RESIDUAL_VALID:
+ if (query.ptrq_value == 0 ||
+ query.ptrq_value > pm->pm_sc.pm_reloadcount) {
+ error = EINVAL;
+ goto unlock;
+ }
+ break;
+ default:
+ error = EINVAL;
+ goto unlock;
+ }
+ }
+ if (query.ptrq_operation == PMC_TEST_RESIDUAL_GET_SAVED ||
+ query.ptrq_operation == PMC_TEST_RESIDUAL_SET_SAVED) {
+ uint8_t state;
+
+ mtx_unlock_spin(pp->pp_tdslock);
+ if (query.ptrq_operation == PMC_TEST_RESIDUAL_SET_SAVED)
+ error = pmu_event_set_thread_residual(pm, pp,
+ (lwpid_t)query.ptrq_tid, query.ptrq_value,
+ (uint8_t)query.ptrq_state);
+ if (error == 0)
+ error = pmu_event_get_thread_residual(pm, pp,
+ (lwpid_t)query.ptrq_tid, &value, &state);
+ if (error != 0)
+ goto out;
+ query.ptrq_value = value;
+ query.ptrq_state = state;
+ goto out;
+ } else {
+ if (!pg->pg_assigned || PMC_ROW_IS_UNASSIGNED(pm)) {
+ error = EBUSY;
+ goto unlock;
+ }
+ ri = PMC_TO_ROWINDEX(pm);
+ if (pp->pp_pmcs[ri].pp_pmc != pm) {
+ error = EBUSY;
+ goto unlock;
+ }
+ if (query.ptrq_operation == PMC_TEST_RESIDUAL_SET_LIVE)
+ pt->pt_pmcs[ri].pt_pmcval = query.ptrq_value;
+ value = pt->pt_pmcs[ri].pt_pmcval;
+ mtx_unlock_spin(pp->pp_tdslock);
+ }
+
+ query.ptrq_value = value;
+ query.ptrq_state = value > 0 &&
+ value <= pm->pm_sc.pm_reloadcount ?
+ PMC_TEST_RESIDUAL_VALID : PMC_TEST_RESIDUAL_UNINITIALIZED;
+ goto out;
+unlock:
+ mtx_unlock_spin(pp->pp_tdslock);
+out:
+ sx_xunlock(&pmc_sx);
+ if (error != 0)
+ return (error);
+ return (SYSCTL_OUT(req, &query, sizeof(query)));
+}
#endif
/*
@@ -939,14 +1655,21 @@
{
counter_u64_add(pm->pm_runcount, delta);
+ pmc_test_counter_add(PMC_TEST_LIVE_RUN_REFS, delta);
}
struct pmc_target *
-pmc_target_object_alloc(int malloc_flags)
+pmc_target_object_alloc(int malloc_flags, bool inject_failure)
{
+ struct pmc_target *pt;
- return (malloc(sizeof(struct pmc_target), M_PMC,
- malloc_flags | M_ZERO));
+ if (inject_failure &&
+ pmc_test_should_fail(PMC_TEST_FAIL_TARGET_LINK_ALLOC))
+ return (NULL);
+ pt = malloc(sizeof(*pt), M_PMC, malloc_flags | M_ZERO);
+ if (pt != NULL)
+ pmc_test_counter_add(PMC_TEST_LIVE_PMC_TARGETS, 1);
+ return (pt);
}
void
@@ -955,6 +1678,7 @@
if (pt == NULL)
return;
+ pmc_test_counter_add(PMC_TEST_LIVE_PMC_TARGETS, -1);
free(pt, M_PMC);
}
@@ -1038,7 +1762,8 @@
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pm != NULL && pp != NULL,
("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
- pt = pmc_target_object_alloc(M_WAITOK);
+ pt = pmc_target_object_alloc(M_WAITOK,
+ pmu_group_from_pmc(pm) != NULL);
if (pt == NULL)
return (ENOMEM);
pmc_link_target_process_preallocated(pm, pp, pt);
@@ -1228,7 +1953,7 @@
KASSERT(pp->pp_pmcs[ri].pp_pmc == NULL,
("[pmc,%d] rotation attach into busy slot ri=%d", __LINE__, ri));
- ptgt = pmc_target_object_alloc(M_WAITOK);
+ ptgt = pmc_target_object_alloc(M_WAITOK, false);
KASSERT(ptgt != NULL, ("[pmc,%d] target allocation failed", __LINE__));
ptgt->pt_process = pp;
LIST_INSERT_HEAD(&pm->pm_targets, ptgt, pt_next);
@@ -1471,6 +2196,9 @@
sx_assert(&pmc_sx, SX_XLOCKED);
pg = pmu_group_from_pmc(pm);
+ if (pg != NULL &&
+ pmc_test_should_fail(PMC_TEST_FAIL_ATTACH_AUTHORIZATION))
+ return (EPERM);
/*
* Deferred group member. Mark it attached, and register it on
@@ -1644,7 +2372,8 @@
break;
}
PROC_UNLOCK(target);
- if (!pmc_can_attach(pm, target)) {
+ if (pmc_test_should_fail(PMC_TEST_FAIL_ATTACH_AUTHORIZATION) ||
+ !pmc_can_attach(pm, target)) {
error = EPERM;
break;
}
@@ -1681,7 +2410,7 @@
link_index = (size_t)i * pg->pg_nevents +
event_index++;
links[link_index] =
- pmc_target_object_alloc(M_WAITOK);
+ pmc_target_object_alloc(M_WAITOK, true);
if (links[link_index] == NULL) {
error = ENOMEM;
break;
@@ -1698,6 +2427,7 @@
KASSERT(i == ntargets,
("[pmc,%d] prepared %u of %u group targets", __LINE__, i,
ntargets));
+ pmc_test_descendants_attach_pause(pg, ntargets);
/* Linearization point. Every fallible operation is done above. */
for (i = 0; i < ntargets; i++)
@@ -1760,6 +2490,7 @@
pmu_group_t *pg;
int error;
struct proc *target, *top;
+ u_int ntargets __pmcdbg_used;
sx_assert(&pmc_sx, SX_XLOCKED);
@@ -1787,6 +2518,12 @@
sx_slock(&proctree_lock);
top = p;
+ ntargets = 0;
+ for (target = top; target != NULL;
+ target = pmc_next_descendant(top, target))
+ ntargets++;
+ pmc_test_descendants_attach_pause(NULL, ntargets);
+
error = 0;
for (target = top; target != NULL;
target = pmc_next_descendant(top, target)) {
@@ -3756,6 +4493,7 @@
mtx_unlock_spin(&pmc_processhash_mtx);
pp = ppnew;
ppnew = NULL;
+ pmc_test_counter_add(PMC_TEST_LIVE_TARGET_PROCESSES, 1);
/* Add thread descriptors for this process' current threads. */
pmc_add_thread_descriptors_from_proc(p, pp);
@@ -3801,6 +4539,7 @@
LIST_REMOVE(pmc_td, pt_next);
pmc_thread_descriptor_pool_free(pmc_td);
}
+ pmc_test_counter_add(PMC_TEST_LIVE_TARGET_PROCESSES, -1);
free(pp, M_PMC);
}
@@ -6732,6 +7471,17 @@
* Allocate space for a sample buffer.
*/
cpu = curcpu;
+ /*
+ * The test group accepts one sample. After that, force the next
+ * interrupt down the buffer-full or stalled path. This holds
+ * the queue at one entry while this group's worker is paused.
+ */
+ if (pmc_test_sample_worker_paused(pm)) {
+ pm->pm_pcpu_state[cpu].pps_stalled = 1;
+ callchaindepth = 1;
+ error = ENOMEM;
+ goto done;
+ }
psb = pmc_pcpu[cpu]->pc_sb[ring];
inuserspace = TRAPF_USERMODE(tf);
ps = PMC_PROD_SAMPLE(psb);
@@ -6810,6 +7560,7 @@
psb->ps_prodidx++;
if (pm->pm_pmu != NULL) {
counter_u64_add(pm->pm_pmu->pe_samples, 1);
+ pmc_test_sample_accepted(pm, ps->ps_pid, ps->ps_tid);
}
done:
/* mark CPU as needing processing */
@@ -6936,6 +7687,7 @@
* reference.
*/
if (nsamples == 0) {
+ pmc_test_sample_dropped(pm);
pmc_runcount_add(pm, -1);
}
}
@@ -6974,6 +7726,7 @@
struct pmc_samplebuffer *psb;
uint64_t delta __diagused;
int adjri, n;
+ bool emitted __pmcdbg_used;
KASSERT(PCPU_GET(cpuid) == cpu,
("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
@@ -6991,6 +7744,11 @@
/* skip non-running samples */
pm = ps->ps_pmc;
+ emitted = false;
+ if (pmc_test_sample_worker_paused(pm)) {
+ DPCPU_SET(pmc_sampled, 1);
+ break;
+ }
if (pm->pm_state != PMC_STATE_RUNNING &&
atomic_load_acq_int(&pm->pm_rotation_drain) == 0)
goto entrydone;
@@ -7050,12 +7808,14 @@
if (ps->ps_flags & PMC_CC_F_USERSPACE) {
td = FIRST_THREAD_IN_PROC(po->po_owner);
addupc_intr(td, ps->ps_pc[0], 1);
+ emitted = true;
}
} else {
- pmclog_process_callchain(pm, ps);
+ emitted = pmclog_process_callchain(pm, ps);
}
entrydone:
+ pmc_test_sample_done(pm, emitted);
ps->ps_nsamples = 0; /* mark entry as free */
KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
("[pmc,%d] pm=%p runcount %ju", __LINE__, pm,
@@ -7128,6 +7888,7 @@
PROC_LOCK(p);
is_using_hwpmcs = (p->p_flag & P_HWPMC) != 0;
PROC_UNLOCK(p);
+ pmc_test_descendants_exit_observed(p);
/*
* Log a sysexit event to all SS PMC owners.
@@ -7274,6 +8035,7 @@
"process_exit: pid=%d pp=%p draining pmu groups",
p->p_pid, pp);
pmu_pp_destroy(pp);
+ pmc_test_counter_add(PMC_TEST_LIVE_TARGET_PROCESSES, -1);
free(pp, M_PMC);
out:
@@ -7631,7 +8393,14 @@
pmc_stats.pm_group_fork_attach_failures = counter_u64_alloc(M_WAITOK);
#ifdef HWPMC_DEBUG
- /* parse debug flags first */
+ for (c = 0; c < PMC_TEST_COUNTER_COUNT; c++)
+ pmc_test_counters[c] = counter_u64_alloc(M_WAITOK);
+ for (c = 0; c < PMC_TEST_FAILPOINT_COUNT; c++)
+ pmc_test_failpoints[c] = UINT_MAX;
+ pmc_test_fail_callchain_log.pth_request = PMC_ID_INVALID;
+ pmc_test_fail_callchain_log.pth_ack = PMC_ID_INVALID;
+
+ /* Parse debug flags first. */
if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
pmc_debugstr, sizeof(pmc_debugstr))) {
pmc_debugflags_parse(pmc_debugstr, pmc_debugstr +
@@ -8043,6 +8812,10 @@
counter_u64_free(pmc_stats.pm_merges);
counter_u64_free(pmc_stats.pm_overwrites);
counter_u64_free(pmc_stats.pm_group_fork_attach_failures);
+#ifdef HWPMC_DEBUG
+ for (c = 0; c < PMC_TEST_COUNTER_COUNT; c++)
+ counter_u64_free(pmc_test_counters[c]);
+#endif
sx_xunlock(&pmc_sx); /* We are done */
}
diff --git a/sys/dev/hwpmc/hwpmc_pmu.h b/sys/dev/hwpmc/hwpmc_pmu.h
--- a/sys/dev/hwpmc/hwpmc_pmu.h
+++ b/sys/dev/hwpmc/hwpmc_pmu.h
@@ -28,6 +28,55 @@
PMC_FLAG_NOWAIT = 0x04, /* do not wait for mallocs */
};
+#ifdef HWPMC_DEBUG
+/* These counters are for tests only. They exist only in HWPMC_DEBUG builds. */
+enum pmc_test_counter {
+ PMC_TEST_LIVE_PMC_TARGETS,
+ PMC_TEST_LIVE_GROUP_TARGETS,
+ PMC_TEST_LIVE_TARGET_PROCESSES,
+ PMC_TEST_LIVE_RESIDUAL_ENTRIES,
+ PMC_TEST_LIVE_ROTATION_REFS,
+ PMC_TEST_LIVE_RUN_REFS,
+ PMC_TEST_COUNTER_COUNT
+};
+
+enum pmc_test_failpoint {
+ PMC_TEST_FAIL_GROUP_TARGET_ALLOC,
+ PMC_TEST_FAIL_TARGET_LINK_ALLOC,
+ PMC_TEST_FAIL_ATTACH_AUTHORIZATION,
+ PMC_TEST_FAIL_SYSTEM_START,
+ PMC_TEST_FAILPOINT_COUNT
+};
+
+void pmc_test_counter_add(enum pmc_test_counter counter, int64_t delta);
+bool pmc_test_should_fail(enum pmc_test_failpoint failpoint);
+bool pmc_test_group_hold_resident(pmu_group_t *pg);
+bool pmc_test_group_hold_evicted(pmu_group_t *pg);
+void pmc_test_descendants_attach_pause(pmu_group_t *pg, u_int ntargets);
+void pmc_test_descendants_exit_observed(struct proc *p);
+void pmc_test_system_start_pause(pmu_group_t *pg, u_int nstarted);
+void pmc_test_sample_accepted(struct pmc *pm, uint32_t pid, uint32_t tid);
+bool pmc_test_sample_worker_paused(struct pmc *pm);
+void pmc_test_sample_schedule_out(pmu_group_t *pg);
+bool pmc_test_callchain_log_should_fail(struct pmc *pm);
+void pmc_test_sample_dropped(struct pmc *pm);
+void pmc_test_sample_done(struct pmc *pm, bool emitted);
+#else
+#define pmc_test_counter_add(counter, delta) do { } while (0)
+#define pmc_test_should_fail(failpoint) (false)
+#define pmc_test_group_hold_resident(pg) (false)
+#define pmc_test_group_hold_evicted(pg) (false)
+#define pmc_test_descendants_attach_pause(pg, n) do { } while (0)
+#define pmc_test_descendants_exit_observed(p) do { } while (0)
+#define pmc_test_system_start_pause(pg, n) do { } while (0)
+#define pmc_test_sample_accepted(pm, pid, tid) do { } while (0)
+#define pmc_test_sample_worker_paused(pm) (false)
+#define pmc_test_sample_schedule_out(pg) do { } while (0)
+#define pmc_test_callchain_log_should_fail(pm) (false)
+#define pmc_test_sample_dropped(pm) do { } while (0)
+#define pmc_test_sample_done(pm, emitted) do { } while (0)
+#endif
+
/*
* Scheduling constraint from the class backend.
* pc_allowed_rows is a bitmask of valid class row indices.
@@ -296,7 +345,8 @@
void pmc_rotation_drain_cpu(struct pmc *pm, int cpu);
void pmc_rotation_detach(struct pmc *pm, struct pmc_process *pp);
void pmc_rotation_attach(struct pmc *pm, struct pmc_process *pp);
-struct pmc_target *pmc_target_object_alloc(int malloc_flags);
+struct pmc_target *pmc_target_object_alloc(int malloc_flags,
+ bool inject_failure);
void pmc_target_object_free(struct pmc_target *pt);
void pmc_link_target_process_preallocated(struct pmc *pm,
struct pmc_process *pp, struct pmc_target *pt);
diff --git a/sys/dev/hwpmc/hwpmc_pmu.c b/sys/dev/hwpmc/hwpmc_pmu.c
--- a/sys/dev/hwpmc/hwpmc_pmu.c
+++ b/sys/dev/hwpmc/hwpmc_pmu.c
@@ -66,12 +66,6 @@
static void pmu_pp_schedule_out(struct pmc_process *pp, pmu_group_t *pg,
bool drain_samples);
static void pmu_pp_kick_rotate(struct pmc_process *pp);
-static void pmu_syscpu_rotate_thread(void *arg);
-static int pmu_sys_schedule_in(int cpu, pmu_group_t *pg);
-static void pmu_sys_schedule_out(int cpu, pmu_group_t *pg);
-static void pmu_sys_stop_rows(int cpu, pmu_group_t *pg, u_int nmembers);
-static void pmu_syscpu_kick_rotate(int cpu);
-static void pmu_syscpu_rotate_one(int cpu);
static void pmu_pp_unlink_group(pmu_group_t *pg, struct pmc_process *pp);
static void pmu_group_attach_siblings(pmu_group_t *pg,
struct pmc_process *pp);
@@ -98,6 +92,12 @@
};
static struct pmu_syscpu pmu_syscpu[MAXCPU];
+static void pmu_syscpu_rotate_thread(void *arg);
+static int pmu_sys_schedule_in(int cpu, pmu_group_t *pg);
+static void pmu_sys_schedule_out(int cpu, pmu_group_t *pg);
+static void pmu_sys_stop_rows(int cpu, pmu_group_t *pg, u_int nmembers);
+static void pmu_syscpu_kick_rotate(int cpu);
+static void pmu_syscpu_rotate_one(int cpu);
void
pmu_group_accounting_initialize(void)
@@ -460,9 +460,12 @@
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pg != NULL && pp != NULL,
("[pmu] invalid target allocation pg=%p pp=%p", pg, pp));
+ if (pmc_test_should_fail(PMC_TEST_FAIL_GROUP_TARGET_ALLOC))
+ return (NULL);
pgt = malloc(sizeof(*pgt), M_PMU, malloc_flags | M_ZERO);
if (pgt == NULL)
return (NULL);
+ pmc_test_counter_add(PMC_TEST_LIVE_GROUP_TARGETS, 1);
pgt->pgt_pp = pp;
pgt->pgt_group = pg;
pgt->pgt_explicit = explicit;
@@ -501,6 +504,7 @@
if (pgt == NULL)
return;
+ pmc_test_counter_add(PMC_TEST_LIVE_GROUP_TARGETS, -1);
free(pgt, M_PMU);
}
@@ -565,7 +569,7 @@
if (pp->pp_pmcs[ri].pp_pmc != NULL)
goto fail;
links[nlinks] =
- pmc_target_object_alloc(M_NOWAIT);
+ pmc_target_object_alloc(M_NOWAIT, true);
if (links[nlinks] == NULL)
goto fail;
nlinks++;
@@ -934,6 +938,7 @@
if (held_pp != NULL) {
mtx_lock_spin(&held_pp->pp_pmu_lock);
held_pp->pp_pmu_refs++;
+ pmc_test_counter_add(PMC_TEST_LIVE_ROTATION_REFS, 1);
mtx_unlock_spin(&held_pp->pp_pmu_lock);
pmu_pp_stop_rotate(held_pp, "muxrel");
}
@@ -955,6 +960,7 @@
KASSERT(held_pp->pp_pmu_refs > 0,
("[pmu] process reference underflow"));
held_pp->pp_pmu_refs--;
+ pmc_test_counter_add(PMC_TEST_LIVE_ROTATION_REFS, -1);
pp_unhashed = held_pp->pp_pmu_unhashed;
mtx_unlock_spin(&held_pp->pp_pmu_lock);
wakeup(&held_pp->pp_pmu_refs);
@@ -1153,6 +1159,7 @@
pg->pg_account_placement_admit = pg->pg_assigned;
mtx_pool_unlock_spin(pmc_mtxpool, pg);
+ (void)pmc_test_group_hold_resident(pg);
pmu_pp_kick_rotate(pp);
return (0);
}
@@ -1478,6 +1485,7 @@
if (!keep) {
if (discard != NULL) {
LIST_REMOVE(discard, ptr_next);
+ pmc_test_counter_add(PMC_TEST_LIVE_RESIDUAL_ENTRIES, -1);
}
} else if (discard != NULL) {
discard->ptr_value = value;
@@ -1490,6 +1498,7 @@
ptr->ptr_value = value;
ptr->ptr_state = state;
LIST_INSERT_HEAD(&pp->pp_pmu_residuals, ptr, ptr_next);
+ pmc_test_counter_add(PMC_TEST_LIVE_RESIDUAL_ENTRIES, 1);
ptr = NULL;
}
mtx_unlock_spin(pp->pp_tdslock);
@@ -1716,6 +1725,7 @@
}
if (ptr != NULL) {
LIST_REMOVE(ptr, ptr_next);
+ pmc_test_counter_add(PMC_TEST_LIVE_RESIDUAL_ENTRIES, -1);
}
mtx_unlock_spin(pp->pp_tdslock);
if (ptr == NULL)
@@ -1737,6 +1747,7 @@
}
if (ptr != NULL) {
LIST_REMOVE(ptr, ptr_next);
+ pmc_test_counter_add(PMC_TEST_LIVE_RESIDUAL_ENTRIES, -1);
}
mtx_unlock_spin(pp->pp_tdslock);
if (ptr == NULL)
@@ -1787,6 +1798,8 @@
if (pg == NULL || pg->pg_assigned)
return (0);
+ if (pmc_test_group_hold_evicted(pg))
+ return (ENOSPC);
/* Use the target process only. */
p = pmu_group_target_proc(pg);
if (p == NULL)
@@ -1818,6 +1831,8 @@
pg->pg_account_placement_admit = true;
mtx_pool_unlock_spin(pmc_mtxpool, pg);
pmu_group_kick_placement(pg);
+ if (pg->pg_running)
+ (void)pmc_test_group_hold_resident(pg);
return (0);
}
@@ -1874,6 +1889,7 @@
/* Phase 4: release the hardware rows. */
pmu_unassign_group(pg, 0);
+ (void)pmc_test_group_hold_evicted(pg);
}
/*
@@ -2170,7 +2186,8 @@
pg = LIST_FIRST(gl);
(*vseen)++;
*vpg = LIST_NEXT(pg, pg_proc_next);
- if (pg->pg_assigned && pg->pg_defer_ok)
+ if (pg->pg_assigned && pg->pg_defer_ok &&
+ !pmc_test_group_hold_resident(pg))
return (pg);
}
return (NULL);
@@ -2337,6 +2354,8 @@
if (pg == NULL || pg->pg_assigned)
return (0);
+ if (pmc_test_group_hold_evicted(pg))
+ return (ENOSPC);
/* Check the owner process's allocation permission. */
owner = pg->pg_owner != NULL ? pg->pg_owner->po_owner : NULL;
@@ -2365,6 +2384,9 @@
nstarted = 0;
TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
+ if (pmc_test_should_fail(PMC_TEST_FAIL_SYSTEM_START))
+ error = EIO;
+ else
error = hwpmc_pmu_sys_start_row(cpu, pe->pe_pmc);
if (error != 0) {
PMCDBG3(PMC, OPS, 1,
@@ -2375,8 +2397,11 @@
return (error);
}
nstarted++;
+ pmc_test_system_start_pause(pg, nstarted);
}
pmu_sys_residency_mark(pg, true);
+ if (pg->pg_sys_listed)
+ (void)pmc_test_group_hold_resident(pg);
PMCDBG3(PMC, OPS, 4, "sys_schedule_in: gid=%u cpu=%d nevents=%u IN",
pg->pg_id, cpu, pg->pg_nevents);
return (0);
@@ -2470,6 +2495,8 @@
("[pmu] group %u stopped member mismatch %u/%u", pg->pg_id,
member, nmembers));
+ if (nmembers != 0)
+ pmc_test_sample_schedule_out(pg);
pmu_sys_group_drain_samples(cpu, pg, nmembers);
}
@@ -2485,6 +2512,7 @@
pmu_sys_stop_rows(cpu, pg, pg->pg_nevents);
pmu_sys_residency_mark(pg, false);
pmu_unassign_group(pg, cpu);
+ (void)pmc_test_group_hold_evicted(pg);
PMCDBG2(PMC, OPS, 4, "sys_schedule_out: gid=%u cpu=%d OUT",
pg->pg_id, cpu);
}
@@ -2582,6 +2610,7 @@
TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
+ (void)pmc_test_group_hold_resident(pg);
pmu_syscpu_kick_rotate(cpu);
return (0);
}
diff --git a/sys/sys/pmclog.h b/sys/sys/pmclog.h
--- a/sys/sys/pmclog.h
+++ b/sys/sys/pmclog.h
@@ -324,7 +324,7 @@
void pmclog_initialize(void);
int pmclog_proc_create(struct thread *td, void **handlep);
void pmclog_proc_ignite(void *handle, struct pmc_owner *po);
-void pmclog_process_callchain(struct pmc *_pm, struct pmc_sample *_ps);
+bool pmclog_process_callchain(struct pmc *_pm, struct pmc_sample *_ps);
void pmclog_process_closelog(struct pmc_owner *po);
void pmclog_process_dropnotify(struct pmc_owner *po);
void pmclog_process_map_in(struct pmc_owner *po, pid_t pid,

File Metadata

Mime Type
text/plain
Expires
Tue, Sep 8, 6:01 PM (3 m, 13 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
38484976
Default Alt Text
D59370.diff (40 KB)

Event Timeline