Page MenuHomeFreeBSD
Files F168382028

D59230.diff
No OneTemporary
Actions

D59230.diff
View Options

diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -46,6 +46,7 @@
#define EXTERR_CATEGORY EXTERR_CAT_HWPMC_AMD
#include <sys/exterrvar.h>
+#include <machine/clock.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
@@ -53,6 +54,49 @@
#define OVERFLOW_WAIT_COUNT 50
+/* Rescale reload by APERF/MPERF to keep a wall-clock period. */
+DPCPU_DEFINE_STATIC(uint64_t, amd_reload_last_aperf);
+DPCPU_DEFINE_STATIC(uint64_t, amd_reload_last_mperf);
+
+static void
+amd_reload_freq_baseline_reset(int cpu)
+{
+
+ DPCPU_ID_SET(cpu, amd_reload_last_aperf, rdmsr(MSR_APERF));
+ DPCPU_ID_SET(cpu, amd_reload_last_mperf, rdmsr(MSR_MPERF));
+}
+
+static uint64_t
+amd_reload_freq_adjust(int cpu, uint64_t base_reload)
+{
+ uint64_t aperf, mperf, daperf, dmperf, adjusted, eff_mhz;
+
+ if (!tsc_perf_stat) {
+ pmc_sample_set_freq(cpu, 0, false);
+ return (base_reload);
+ }
+
+ aperf = rdmsr(MSR_APERF);
+ mperf = rdmsr(MSR_MPERF);
+ daperf = aperf - DPCPU_ID_GET(cpu, amd_reload_last_aperf);
+ dmperf = mperf - DPCPU_ID_GET(cpu, amd_reload_last_mperf);
+ DPCPU_ID_SET(cpu, amd_reload_last_aperf, aperf);
+ DPCPU_ID_SET(cpu, amd_reload_last_mperf, mperf);
+
+ if (dmperf == 0 || daperf == 0) {
+ pmc_sample_set_freq(cpu, 0, false);
+ return (base_reload);
+ }
+
+ eff_mhz = (tsc_freq * daperf) / dmperf / 1000000;
+ pmc_sample_set_freq(cpu, (uint16_t)MIN(eff_mhz, UINT16_MAX), true);
+
+ adjusted = (base_reload * daperf) / dmperf;
+ adjusted = MAX(adjusted, MAX(1, base_reload / 4));
+ adjusted = MIN(adjusted, 4 * base_reload);
+ return (adjusted);
+}
+
DPCPU_DEFINE_STATIC(uint32_t, nmi_counter);
/* AMD K8 PMCs */
@@ -539,6 +583,9 @@
PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config);
+ if (PMC_IS_KERNEL_SAMPLER(pm) && tsc_perf_stat)
+ amd_reload_freq_baseline_reset(cpu);
+
wrmsr(pd->pm_evsel, config);
return (0);
}
@@ -652,6 +699,8 @@
evsel = amd_pmcdesc[i].pm_evsel;
perfctr = amd_pmcdesc[i].pm_perfctr;
v = pm->pm_sc.pm_reloadcount;
+ if (PMC_IS_KERNEL_SAMPLER(pm))
+ v = amd_reload_freq_adjust(cpu, v);
config = rdmsr(evsel);
KASSERT((config & ~AMD_PMC_ENABLE) ==
diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c
--- a/sys/dev/hwpmc/hwpmc_core.c
+++ b/sys/dev/hwpmc/hwpmc_core.c
@@ -32,11 +32,13 @@
#include <sys/param.h>
#include <sys/bus.h>
+#include <sys/pcpu.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>
+#include <machine/clock.h>
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <machine/cpu.h>
@@ -413,6 +415,44 @@
return (0);
}
+/* Report effective MHz from APERF/MPERF. */
+DPCPU_DEFINE_STATIC(uint64_t, iaf_freq_last_aperf);
+DPCPU_DEFINE_STATIC(uint64_t, iaf_freq_last_mperf);
+
+static void
+iaf_freq_baseline_reset(int cpu)
+{
+
+ DPCPU_ID_SET(cpu, iaf_freq_last_aperf, rdmsr(MSR_APERF));
+ DPCPU_ID_SET(cpu, iaf_freq_last_mperf, rdmsr(MSR_MPERF));
+}
+
+static void
+iaf_freq_report(int cpu)
+{
+ uint64_t aperf, mperf, daperf, dmperf, eff_mhz;
+
+ if (!tsc_perf_stat) {
+ pmc_sample_set_freq(cpu, 0, false);
+ return;
+ }
+
+ aperf = rdmsr(MSR_APERF);
+ mperf = rdmsr(MSR_MPERF);
+ daperf = aperf - DPCPU_ID_GET(cpu, iaf_freq_last_aperf);
+ dmperf = mperf - DPCPU_ID_GET(cpu, iaf_freq_last_mperf);
+ DPCPU_ID_SET(cpu, iaf_freq_last_aperf, aperf);
+ DPCPU_ID_SET(cpu, iaf_freq_last_mperf, mperf);
+
+ if (dmperf == 0 || daperf == 0) {
+ pmc_sample_set_freq(cpu, 0, false);
+ return;
+ }
+
+ eff_mhz = (tsc_freq * daperf) / dmperf / 1000000;
+ pmc_sample_set_freq(cpu, (uint16_t)MIN(eff_mhz, UINT16_MAX), true);
+}
+
static int
iaf_start_pmc(int cpu, int ri, struct pmc *pm)
{
@@ -425,6 +465,9 @@
PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
+ if (PMC_IS_KERNEL_SAMPLER(pm) && tsc_perf_stat)
+ iaf_freq_baseline_reset(cpu);
+
cc = core_pcpu[cpu];
cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
wrmsr(IAF_CTRL, cc->pc_iafctrl);
@@ -1104,6 +1147,9 @@
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
+ if (PMC_IS_KERNEL_SAMPLER(pm))
+ iaf_freq_report(cpu);
+
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (__predict_false(error))
intrdisable |= flag;
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -47,6 +47,7 @@
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/pmclog.h>
+#include <sys/pmc_sample.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/queue.h>
@@ -77,6 +78,12 @@
#include <vm/vm_object.h>
#include "hwpmc_soft.h"
+#if defined(__amd64__) || defined(__i386__)
+#include <x86/x86_var.h>
+#include <x86/cputypes.h>
+#include "hwpmc_core.h"
+#include "hwpmc_amd.h"
+#endif
#define PMC_EPOCH_ENTER() \
struct epoch_tracker pmc_et; \
@@ -241,6 +248,8 @@
static void pmc_link_target_process(struct pmc *pm,
struct pmc_process *pp);
static void pmc_log_all_process_mappings(struct pmc_owner *po);
+
+static void pmc_sampler_deliver(struct pmc *pm, struct pmc_sample *ps);
static void pmc_log_kernel_mappings(struct pmc *pm);
static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
static void pmc_maybe_remove_owner(struct pmc_owner *po);
@@ -4777,6 +4786,20 @@
pmc_multipart_add(ps, mp->pl_type, scale * mp->pl_length);
}
+/*
+ * Effective MHz stashed by the PMI handler for pmc_add_sample().
+ */
+DPCPU_DEFINE_STATIC(uint16_t, pmc_sample_freq_mhz);
+DPCPU_DEFINE_STATIC(bool, pmc_sample_freq_valid);
+
+void
+pmc_sample_set_freq(int cpu, uint16_t freq_mhz, bool valid)
+{
+
+ DPCPU_ID_SET(cpu, pmc_sample_freq_mhz, freq_mhz);
+ DPCPU_ID_SET(cpu, pmc_sample_freq_valid, valid);
+}
+
/*
* Find a free slot in the per-cpu array of samples and capture the
* current callchain there. If a sample was successfully added, a bit
@@ -4832,6 +4855,13 @@
ps->ps_ticks = ticks;
ps->ps_cpu = cpu;
ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
+ if (PMC_IS_KERNEL_SAMPLER(pm)) {
+ ps->ps_freq_mhz = DPCPU_GET(pmc_sample_freq_mhz);
+ ps->ps_freq_valid = DPCPU_GET(pmc_sample_freq_valid);
+ } else {
+ ps->ps_freq_mhz = 0;
+ ps->ps_freq_valid = 0;
+ }
ps->ps_nsamples_actual = 0;
callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
@@ -5111,6 +5141,8 @@
td = FIRST_THREAD_IN_PROC(po->po_owner);
addupc_intr(td, ps->ps_pc[0], 1);
}
+ } else if ((po->po_flags & PMC_PO_KERNEL_SAMPLER) != 0) {
+ pmc_sampler_deliver(pm, ps);
} else
pmclog_process_callchain(pm, ps);
@@ -5599,6 +5631,417 @@
return (0);
}
+/*
+ * In-kernel PMU sample consumer support (see sys/pmc_sample.h). Reuses
+ * the normal hwpmc sample path with a synthetic PMC_PO_KERNEL_SAMPLER
+ * owner whose samples are diverted to pmc_sampler_deliver(). x86-only.
+ */
+
+/* Per-session state. Guarded by pmc_sx for start/stop, none for delivery. */
+static struct pmc_owner *pmc_sampler_po; /* synthetic kernel owner */
+static struct pmc **pmc_sampler_pmcs; /* indexed by cpu id */
+static int pmc_sampler_ncpus; /* size of pmc_sampler_pmcs */
+static pmc_sample_cb_t pmc_sampler_cb;
+static void *pmc_sampler_cb_arg;
+
+static int pmc_sampler_alloc_pmc_on_cpu(int cpu, enum pmc_sample_event ev,
+ uint64_t reload);
+static void pmc_sampler_release_pmc(int cpu);
+
+static int
+pmc_sampler_resolve_event(enum pmc_sample_event ev, enum pmc_class *class,
+ uint64_t *config)
+{
+
+ switch (ev) {
+ case PMC_SAMPLE_EV_CPU_CYCLES:
+#if defined(__amd64__) || defined(__i386__)
+ switch (cpu_vendor_id) {
+ case CPU_VENDOR_INTEL:
+ /* IAF CPU_CLK_UNHALTED.REF. */
+ *class = PMC_CLASS_IAF;
+ *config = IAP_EVSEL(0x3C) | IAP_UMASK(0x01) |
+ IAP_OS | IAP_USR | IAP_INT;
+ return (0);
+ case CPU_VENDOR_AMD:
+ case CPU_VENDOR_HYGON:
+ /* K8 BU_CPU_CLK_UNHALTED. */
+ *class = PMC_CLASS_K8;
+ *config = AMD_PMC_TO_EVENTMASK(0x76) |
+ AMD_PMC_USR | AMD_PMC_OS | AMD_PMC_INT;
+ return (0);
+ default:
+ return (ENOTSUP);
+ }
+#else
+ return (ENOTSUP);
+#endif
+ default:
+ return (ENOTSUP);
+ }
+}
+
+/*
+ * Synthesise the pmc_op_pmcallocate that pcd_allocate_pmc() expects,
+ * mirroring libpmc(3) but with a fixed event set.
+ */
+static void
+pmc_sampler_build_pa(struct pmc_op_pmcallocate *pa, enum pmc_class class,
+ uint64_t config, uint64_t reload, int cpu)
+{
+
+ bzero(pa, sizeof(*pa));
+ pa->pm_class = class;
+ pa->pm_mode = PMC_MODE_SS;
+ pa->pm_cpu = cpu;
+ pa->pm_count = reload;
+ pa->pm_caps = PMC_CAP_INTERRUPT | PMC_CAP_SYSTEM | PMC_CAP_USER;
+ pa->pm_flags = PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN | PMC_F_EV_PMU;
+#if defined(__amd64__) || defined(__i386__)
+ switch (class) {
+ case PMC_CLASS_IAF:
+ case PMC_CLASS_IAP:
+ pa->pm_md.pm_iap.pm_iap_config = config;
+ pa->pm_md.pm_iap.pm_iap_rsp = 0;
+ break;
+ case PMC_CLASS_K8:
+ pa->pm_md.pm_amd.pm_amd_config = config;
+ pa->pm_md.pm_amd.pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE;
+ break;
+ default:
+ (void)config;
+ break;
+ }
+#else
+ (void)config;
+#endif
+}
+
+/*
+ * Allocate, configure, and start one system-mode sampling PMC on the
+ * given CPU. pmc_sx must be held exclusive.
+ */
+static int
+pmc_sampler_alloc_pmc_on_cpu(int cpu, enum pmc_sample_event ev,
+ uint64_t reload)
+{
+ struct pmc *pmc;
+ struct pmc_op_pmcallocate pa;
+ struct pmc_classdep *pcd;
+ struct pmc_binding pb;
+ struct pmc_hw *phw;
+ enum pmc_class class;
+ uint64_t config;
+ int adjri, error, n;
+
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ error = pmc_sampler_resolve_event(ev, &class, &config);
+ if (error != 0)
+ return (error);
+
+ if (!pmc_cpu_is_active(cpu))
+ return (ENXIO);
+
+ pcd = pmc_class_to_classdep(class);
+ if (pcd == NULL)
+ return (ENXIO);
+
+ pmc = pmc_allocate_pmc_descriptor();
+ pmc->pm_id = PMC_ID_MAKE_ID(cpu, PMC_MODE_SS, class, PMC_ID_INVALID);
+ pmc->pm_event = 0; /* filled in by the allocator for IAF/IAP */
+ pmc->pm_state = PMC_STATE_FREE;
+ pmc->pm_caps = PMC_CAP_INTERRUPT | PMC_CAP_SYSTEM | PMC_CAP_USER;
+ pmc->pm_flags = PMC_F_CALLCHAIN | PMC_F_USERCALLCHAIN | PMC_F_EV_PMU;
+ pmc->pm_sc.pm_reloadcount = MAX(MAX(1, pmc_mincount), reload);
+
+ pmc_sampler_build_pa(&pa, class, config, reload, cpu);
+
+ /* Find a free row within the requested class. */
+ pmc_save_cpu_binding(&pb);
+ pmc_select_cpu(cpu);
+ for (n = pcd->pcd_ri; n < md->pmd_npmc; n++) {
+ struct pmc_classdep *pcd_n;
+
+ pcd_n = pmc_ri_to_classdep(md, n, &adjri);
+ if (pcd_n != pcd)
+ continue;
+ if (!pmc_can_allocate_row(n, PMC_MODE_SS))
+ continue;
+ if (pmc_pcpu[cpu]->pc_hwpmcs[n]->phw_pmc != NULL)
+ continue; /* already in use */
+
+ if (pcd_n->pcd_allocate_pmc(cpu, adjri, pmc, &pa) == 0)
+ break;
+ }
+ if (n == md->pmd_npmc) {
+ pmc_restore_cpu_binding(&pb);
+ pmc_destroy_pmc_descriptor(pmc);
+ return (ENOMEM);
+ }
+
+ pmc->pm_id = PMC_ID_MAKE_ID(cpu, PMC_MODE_SS, class, n);
+
+ /* Configure the PMC into the hardware row. */
+ phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
+ if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
+ error = ENXIO;
+ goto release;
+ }
+ pcd = pmc_ri_to_classdep(md, n, &adjri);
+ error = pcd->pcd_config_pmc(cpu, adjri, pmc);
+ if (error != 0)
+ goto release;
+
+ pmc->pm_state = PMC_STATE_ALLOCATED;
+ pmc->pm_class = class;
+ PMC_MARK_ROW_STANDALONE(n);
+
+ /* Link into synthetic owner. */
+ KASSERT(pmc_sampler_po != NULL,
+ ("[pmc,%d] kernel sampler owner not set", __LINE__));
+ pmc->pm_owner = pmc_sampler_po;
+ LIST_INSERT_HEAD(&pmc_sampler_po->po_pmcs, pmc, pm_next);
+
+ /* Write reload count, arm interrupt, start counting. */
+ pmc->pm_state = PMC_STATE_RUNNING;
+ pmc_sampler_po->po_sscount++;
+ if (pmc_sampler_po->po_sscount == 1) {
+ atomic_add_rel_int(&pmc_ss_count, 1);
+ CK_LIST_INSERT_HEAD(&pmc_ss_owners, pmc_sampler_po, po_ssnext);
+ }
+ critical_enter();
+ error = pcd->pcd_write_pmc(cpu, adjri, pmc, pmc->pm_sc.pm_reloadcount);
+ if (error == 0) {
+ pmc->pm_pcpu_state[cpu].pps_stalled = 0;
+ pmc->pm_pcpu_state[cpu].pps_cpustate = 1;
+ error = pcd->pcd_start_pmc(cpu, adjri, pmc);
+ }
+ critical_exit();
+ if (error != 0) {
+ /* Roll back start-side bookkeeping; release: handles the rest. */
+ pmc->pm_pcpu_state[cpu].pps_cpustate = 0;
+ pmc_sampler_po->po_sscount--;
+ if (pmc_sampler_po->po_sscount == 0) {
+ atomic_subtract_rel_int(&pmc_ss_count, 1);
+ CK_LIST_REMOVE(pmc_sampler_po, po_ssnext);
+ }
+ pmc->pm_state = PMC_STATE_STOPPED;
+ goto release;
+ }
+
+ pmc_restore_cpu_binding(&pb);
+ pmc_sampler_pmcs[cpu] = pmc;
+ return (0);
+
+release:
+ pmc_restore_cpu_binding(&pb);
+ if (pmc->pm_owner != NULL) {
+ LIST_REMOVE(pmc, pm_next);
+ pmc->pm_owner = NULL;
+ }
+ (void)pcd->pcd_release_pmc(cpu, adjri, pmc);
+ if (pmc->pm_state == PMC_STATE_ALLOCATED ||
+ pmc->pm_state == PMC_STATE_STOPPED)
+ PMC_UNMARK_ROW_STANDALONE(n);
+ pmc->pm_state = PMC_STATE_DELETED;
+ pmc_destroy_pmc_descriptor(pmc);
+ return (error);
+}
+
+/*
+ * Stop and release the per-CPU PMC previously allocated by
+ * pmc_sampler_alloc_pmc_on_cpu(). pmc_sx must be held exclusive.
+ */
+static void
+pmc_sampler_release_pmc(int cpu)
+{
+ struct pmc *pmc;
+ struct pmc_classdep *pcd;
+ struct pmc_binding pb;
+ int adjri, ri;
+
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ pmc = pmc_sampler_pmcs[cpu];
+ if (pmc == NULL)
+ return;
+ pmc_sampler_pmcs[cpu] = NULL;
+
+ ri = PMC_TO_ROWINDEX(pmc);
+ pcd = pmc_ri_to_classdep(md, ri, &adjri);
+
+ pmc_save_cpu_binding(&pb);
+ pmc_select_cpu(cpu);
+
+ /* Stop hardware. */
+ pmc->pm_pcpu_state[cpu].pps_cpustate = 0;
+ if (pmc->pm_state == PMC_STATE_RUNNING &&
+ pmc->pm_pcpu_state[cpu].pps_stalled == 0) {
+ critical_enter();
+ (void)pcd->pcd_stop_pmc(cpu, adjri, pmc);
+ critical_exit();
+ }
+
+ /* Deconfigure the row. */
+ critical_enter();
+ (void)pcd->pcd_config_pmc(cpu, adjri, NULL);
+ critical_exit();
+
+ /* Drop the SS refcount. */
+ if (pmc->pm_state == PMC_STATE_RUNNING) {
+ pmc_sampler_po->po_sscount--;
+ if (pmc_sampler_po->po_sscount == 0) {
+ atomic_subtract_rel_int(&pmc_ss_count, 1);
+ CK_LIST_REMOVE(pmc_sampler_po, po_ssnext);
+ epoch_wait_preempt(global_epoch_preempt);
+ }
+ }
+ pmc->pm_state = PMC_STATE_DELETED;
+
+ pmc_restore_cpu_binding(&pb);
+
+ /* Drain any in-flight samples that still reference this PMC. */
+ pmc_wait_for_pmc_idle(pmc);
+
+ /* Release hardware and free descriptor. */
+ (void)pcd->pcd_release_pmc(cpu, adjri, pmc);
+ PMC_UNMARK_ROW_STANDALONE(ri);
+ if (pmc->pm_owner != NULL) {
+ LIST_REMOVE(pmc, pm_next);
+ pmc->pm_owner = NULL;
+ }
+ pmc_destroy_pmc_descriptor(pmc);
+}
+
+/*
+ * Deliver one sample to the registered callback. Called from
+ * pmc_process_samples() in hardclock context.
+ */
+static void
+pmc_sampler_deliver(struct pmc *pm, struct pmc_sample *ps)
+{
+ struct pmc_sample_info s;
+
+ if (__predict_false(pmc_sampler_cb == NULL))
+ return; /* raced with stop */
+
+ bzero(&s, sizeof(s));
+ s.hs_pid = ps->ps_pid;
+ s.hs_tid = ps->ps_tid;
+ s.hs_cpu = (uint16_t)ps->ps_cpu;
+ s.hs_nframes = (uint16_t)MIN(ps->ps_nsamples, UINT16_MAX);
+ if ((ps->ps_flags & PMC_CC_F_USERSPACE) != 0)
+ s.hs_flags |= PMC_SAMPLE_F_USERMODE;
+ s.hs_tsc = ps->ps_tsc;
+ s.hs_pmc_id = pm->pm_id;
+ s.hs_freq_mhz = ps->ps_freq_mhz;
+ if (ps->ps_freq_valid)
+ s.hs_flags |= PMC_SAMPLE_F_FREQ_VALID;
+ s.hs_pc = (const uintptr_t *)ps->ps_pc;
+ pmc_sampler_cb(&s, pmc_sampler_cb_arg);
+}
+
+/* Start a system-wide kernel PMU sampling session. */
+int
+pmc_sample_start(enum pmc_sample_event ev, uint64_t reload,
+ pmc_sample_cb_t cb, void *arg)
+{
+ struct pmc_owner *po;
+ int cpu, error;
+
+ if (cb == NULL || reload == 0)
+ return (EINVAL);
+
+ sx_xlock(&pmc_sx);
+ if (pmc_sampler_po != NULL) {
+ sx_xunlock(&pmc_sx);
+ return (EBUSY);
+ }
+
+ /* Synthesise a kernel owner. */
+ po = malloc(sizeof(*po), M_PMC, M_WAITOK | M_ZERO);
+ po->po_owner = &proc0;
+ po->po_flags = PMC_PO_KERNEL_SAMPLER;
+ LIST_INIT(&po->po_pmcs);
+ TAILQ_INIT(&po->po_logbuffers);
+ mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-sampler", MTX_SPIN);
+ pmc_sampler_po = po;
+
+ pmc_sampler_ncpus = mp_maxid + 1;
+ pmc_sampler_pmcs = malloc(sizeof(*pmc_sampler_pmcs) *
+ pmc_sampler_ncpus, M_PMC, M_WAITOK | M_ZERO);
+
+ /* Publish the callback before any PMC starts running. */
+ pmc_sampler_cb = cb;
+ pmc_sampler_cb_arg = arg;
+
+ /* Allocate and start a PMC on every online CPU. */
+ error = 0;
+ CPU_FOREACH(cpu) {
+ if (cpu >= pmc_sampler_ncpus)
+ continue;
+ error = pmc_sampler_alloc_pmc_on_cpu(cpu, ev, reload);
+ if (error != 0)
+ break;
+ }
+
+ if (error != 0) {
+ /* Roll back: release everything we started. */
+ CPU_FOREACH(cpu) {
+ if (cpu < pmc_sampler_ncpus)
+ pmc_sampler_release_pmc(cpu);
+ }
+ pmc_sampler_cb = NULL;
+ pmc_sampler_cb_arg = NULL;
+ free(pmc_sampler_pmcs, M_PMC);
+ pmc_sampler_pmcs = NULL;
+ pmc_sampler_ncpus = 0;
+ mtx_destroy(&pmc_sampler_po->po_mtx);
+ free(pmc_sampler_po, M_PMC);
+ pmc_sampler_po = NULL;
+ sx_xunlock(&pmc_sx);
+ return (error);
+ }
+
+ sx_xunlock(&pmc_sx);
+ return (0);
+}
+
+/*
+ * Stop the active session, drain in-flight samples, release per-CPU PMCs.
+ */
+void
+pmc_sample_stop(void)
+{
+ int cpu;
+
+ sx_xlock(&pmc_sx);
+ if (pmc_sampler_po == NULL) {
+ sx_xunlock(&pmc_sx);
+ return;
+ }
+
+ /* Stop and release each per-CPU PMC. */
+ CPU_FOREACH(cpu) {
+ if (cpu < pmc_sampler_ncpus)
+ pmc_sampler_release_pmc(cpu);
+ }
+
+ /* Clear cb only after PMCs are stopped and samples drained. */
+ pmc_sampler_cb = NULL;
+ pmc_sampler_cb_arg = NULL;
+
+ free(pmc_sampler_pmcs, M_PMC);
+ pmc_sampler_pmcs = NULL;
+ pmc_sampler_ncpus = 0;
+ mtx_destroy(&pmc_sampler_po->po_mtx);
+ free(pmc_sampler_po, M_PMC);
+ pmc_sampler_po = NULL;
+
+ sx_xunlock(&pmc_sx);
+}
+
static struct pmc_mdep *
pmc_generic_cpu_initialize(void)
{
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -894,6 +894,10 @@
#define PMC_PO_OWNS_LOGFILE 0x00000001 /* has a log file */
#define PMC_PO_SHUTDOWN 0x00000010 /* in the process of shutdown */
#define PMC_PO_INITIAL_MAPPINGS_DONE 0x00000020
+#define PMC_PO_KERNEL_SAMPLER 0x00000040 /* in-kernel sampler owner */
+#define PMC_IS_KERNEL_SAMPLER(pm) \
+ ((pm)->pm_owner != NULL && \
+ ((pm)->pm_owner->po_flags & PMC_PO_KERNEL_SAMPLER) != 0)
/*
* struct pmc_hw -- describe the state of the PMC hardware
@@ -951,7 +955,8 @@
lwpid_t ps_tid; /* thread id */
pid_t ps_pid; /* process PID or -1 */
int ps_ticks; /* ticks at sample time */
- /* pad */
+ uint16_t ps_freq_mhz; /* effective CPU freq at sample time */
+ uint16_t ps_freq_valid; /* nonzero if ps_freq_mhz is meaningful */
struct thread *ps_td; /* which thread */
struct pmc *ps_pmc; /* interrupting PMC */
uintptr_t *ps_pc; /* (const) callchain start */
@@ -1260,6 +1265,7 @@
struct trapframe *_tf, struct pmc_multipart *mp);
int pmc_process_interrupt(int _ring, struct pmc *_pm,
struct trapframe *_tf);
+void pmc_sample_set_freq(int _cpu, uint16_t _freq_mhz, bool _valid);
int pmc_save_kernel_callchain(uintptr_t *_cc, int _maxsamples,
struct trapframe *_tf);
int pmc_save_user_callchain(uintptr_t *_cc, int _maxsamples,
diff --git a/sys/sys/pmc_sample.h b/sys/sys/pmc_sample.h
new file mode 100644
--- /dev/null
+++ b/sys/sys/pmc_sample.h
@@ -0,0 +1,60 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Netflix, Inc.
+ */
+
+#ifndef _SYS_PMC_SAMPLE_H_
+#define _SYS_PMC_SAMPLE_H_
+
+/*
+ * In-kernel PMU sample consumer KPI for hwpmc(4): arm a system-wide
+ * sampling PMC and receive callchains via callback, bypassing
+ * pmcstat(8)/pmclog. One session at a time. The callback runs from
+ * hardclock (PMC_FN_DO_SAMPLES) and must not sleep or re-enter hwpmc.
+ */
+
+#ifdef _KERNEL
+
+#include <sys/types.h>
+#include <sys/cdefs.h>
+
+/* Logical sampling events; hwpmc maps each to the per-arch counter. */
+enum pmc_sample_event {
+ PMC_SAMPLE_EV_CPU_CYCLES, /* unhalted CPU cycles */
+};
+
+struct pmc_sample_info {
+ uint32_t hs_pid; /* process id at sample time */
+ uint32_t hs_tid; /* thread id at sample time */
+ uint16_t hs_cpu; /* CPU the sample fired on */
+ uint16_t hs_nframes; /* valid entries in hs_pc[] */
+ uint32_t hs_flags; /* PMC_SAMPLE_F_* below */
+ uint64_t hs_tsc; /* TSC at overflow */
+ uint32_t hs_pmc_id; /* hwpmc PMC ID (pm->pm_id) */
+ uint16_t hs_freq_mhz; /* eff. freq; valid iff F_FREQ_VALID */
+ uint16_t hs_reserved; /* must be zero */
+ const uintptr_t *hs_pc; /* valid during callback only */
+};
+
+/* hs_flags */
+#define PMC_SAMPLE_F_USERMODE 0x00000001 /* interrupted user mode */
+#define PMC_SAMPLE_F_FREQ_VALID 0x00000002 /* hs_freq_mhz is meaningful */
+
+typedef void (*pmc_sample_cb_t)(
+ const struct pmc_sample_info *sample, void *arg);
+
+/*
+ * Start a session: reload is the counter overflow value (reload = F/Hz
+ * for a cycles counter at frequency F). Returns EBUSY if a session is
+ * running, ENOTSUP if the event/arch is unsupported, or other errno.
+ */
+int pmc_sample_start(enum pmc_sample_event ev, uint64_t reload,
+ pmc_sample_cb_t cb, void *arg);
+
+/* Stop the session, drain in-flight samples, release PMCs. No-op if idle. */
+void pmc_sample_stop(void);
+
+#endif /* _KERNEL */
+
+#endif /* !_SYS_PMC_SAMPLE_H_ */

File Metadata

Mime Type
text/plain
Expires
Fri, Aug 28, 9:45 PM (9 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
37484839
Default Alt Text
D59230.diff (20 KB)

Event Timeline