D58028.diff
No OneTemporary
Actions

Size

22 KB

Referenced Files

None

Subscribers

None

D58028.diff
View Options

	diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h
	--- a/sys/amd64/include/pmc_mdep.h
	+++ b/sys/amd64/include/pmc_mdep.h
	@@ -42,6 +42,7 @@
	#include <dev/hwpmc/hwpmc_amd.h>
	#include <dev/hwpmc/hwpmc_core.h>
	#include <dev/hwpmc/hwpmc_ibs.h>
	+#include <dev/hwpmc/hwpmc_rapl.h>
	#include <dev/hwpmc/hwpmc_tsc.h>
	#include <dev/hwpmc/hwpmc_uncore.h>

	diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
	--- a/sys/conf/files.x86
	+++ b/sys/conf/files.x86
	@@ -118,6 +118,7 @@
	dev/hwpmc/hwpmc_intel.c optional hwpmc
	dev/hwpmc/hwpmc_core.c optional hwpmc
	dev/hwpmc/hwpmc_uncore.c optional hwpmc
	+dev/hwpmc/hwpmc_rapl.c optional hwpmc
	dev/hwpmc/hwpmc_tsc.c optional hwpmc
	dev/hwpmc/hwpmc_x86.c optional hwpmc
	dev/hyperv/hvsock/hv_sock.c optional hyperv
	diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
	--- a/sys/dev/hwpmc/hwpmc_amd.c
	+++ b/sys/dev/hwpmc/hwpmc_amd.c
	@@ -1073,12 +1073,13 @@

	/*
	* These processors have two or three classes of PMCs: the TSC,
	- * programmable PMCs, and AMD IBS.
	+ * programmable PMCs, and AMD IBS. One extra class slot is reserved
	+ * for the optional RAPL energy counters.
	*/
	if ((amd_feature2 & AMDID2_IBS) != 0) {
	- nclasses = 3;
	+ nclasses = 4;
	} else {
	- nclasses = 2;
	+ nclasses = 3;
	}

	pmc_mdep = pmc_mdep_alloc(nclasses);
	@@ -1124,12 +1125,17 @@

	PMCDBG0(MDP, INI, 0, "amd-initialize");

	- if (nclasses >= 3) {
	+ if ((amd_feature2 & AMDID2_IBS) != 0) {
	error = pmc_ibs_initialize(pmc_mdep, ncpus);
	if (error != 0)
	goto error;
	}

	+ /* RAPL takes the reserved last slot; drop it if the probe fails. */
	+ error = pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1);
	+ if (error != 0)
	+ pmc_mdep->pmd_nclass--;
	+
	return (pmc_mdep);

	error:
	@@ -1145,6 +1151,9 @@
	{
	PMCDBG0(MDP, INI, 1, "amd-finalize");

	+ /* Safe even if the RAPL class was skipped at initialize time. */
	+ pmc_rapl_finalize(md);
	+
	pmc_tsc_finalize(md);

	for (int i = 0; i < pmc_cpu_max(); i++)
	diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c
	--- a/sys/dev/hwpmc/hwpmc_intel.c
	+++ b/sys/dev/hwpmc/hwpmc_intel.c
	@@ -280,6 +280,9 @@
	return (NULL);
	}

	+ /* Reserve one extra class slot for the optional RAPL counters. */
	+ nclasses++;
	+
	/* Allocate base class and initialize machine dependent struct */
	pmc_mdep = pmc_mdep_alloc(nclasses);

	@@ -334,6 +337,10 @@
	default:
	break;
	}
	+
	+ if (error == 0 &&
	+ pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1) != 0)
	+ pmc_mdep->pmd_nclass--;
	error:
	if (error) {
	pmc_mdep_free(pmc_mdep);
	@@ -346,6 +353,8 @@
	void
	pmc_intel_finalize(struct pmc_mdep *md)
	{
	+ pmc_rapl_finalize(md);
	+
	pmc_tsc_finalize(md);

	pmc_core_finalize(md);
	diff --git a/sys/dev/hwpmc/hwpmc_rapl.h b/sys/dev/hwpmc/hwpmc_rapl.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/hwpmc/hwpmc_rapl.h
	@@ -0,0 +1,26 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause
	+ *
	+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
	+ *
	+ * AMD/Intel RAPL energy counters exposed as an hwpmc(4) PMC class.
	+ */
	+
	+#ifndef _DEV_HWPMC_RAPL_H_
	+#define _DEV_HWPMC_RAPL_H_ 1
	+
	+#ifdef _KERNEL
	+
	+struct pmc_mdep;
	+
	+/*
	+ * Max RAPL rows per CPU: package, "cores" and (Intel only) DRAM. The actual
	+ * per-vendor count is determined at initialize time (pcd_num).
	+ */
	+#define RAPL_MAX_NPMCS 3
	+
	+int pmc_rapl_initialize(struct pmc_mdep *_md, int _maxcpu, int _classindex);
	+void pmc_rapl_finalize(struct pmc_mdep *_md);
	+
	+#endif /* _KERNEL */
	+#endif /* _DEV_HWPMC_RAPL_H_ */
	diff --git a/sys/dev/hwpmc/hwpmc_rapl.c b/sys/dev/hwpmc/hwpmc_rapl.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/hwpmc/hwpmc_rapl.c
	@@ -0,0 +1,607 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause
	+ *
	+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
	+ *
	+ * AMD/Intel RAPL energy counters exposed as an hwpmc(4) PMC class.
	+ *
	+ * Read-only, system-scope (PMC_MODE_SC), 64-bit counters reporting
	+ * microjoules.
	+ */
	+
	+#include <sys/param.h>
	+#include <sys/bus.h>
	+#include <sys/callout.h>
	+#include <sys/malloc.h>
	+#include <sys/mutex.h>
	+#include <sys/pmc.h>
	+#include <sys/pmckern.h>
	+#include <sys/priv.h>
	+#include <sys/proc.h>
	+#include <sys/smp.h>
	+#include <sys/systm.h>
	+
	+#include <machine/cpu.h>
	+#include <machine/cpufunc.h>
	+#include <machine/cputypes.h>
	+#include <machine/specialreg.h>
	+
	+#include <x86/x86_var.h>
	+
	+#include <dev/hwpmc/hwpmc_rapl.h>
	+
	+/* Energy counters are per-package/core domains, not per CPU: DOMWIDE. */
	+#define RAPL_CAPS (PMC_CAP_READ \| PMC_CAP_DOMWIDE)
	+
	+/* Worst-case package watts for sizing the guard timer. */
	+#define RAPL_GUARD_WATT 1000
	+
	+/* Guard interval clamp band (ms). */
	+#define RAPL_GUARD_MIN_MS 10
	+#define RAPL_GUARD_MAX_MS 60000
	+
	+struct rapl_event {
	+ enum pmc_event re_ev;
	+ uint32_t re_msr;
	+ uint32_t re_unit; /* unit shift: 1 tick = 1/2^unit J */
	+};
	+
	+struct rapl_value {
	+ uint64_t rv_prev; /* last raw 32-bit MSR value */
	+ uint64_t rv_accum;
	+ sbintime_t rv_prev_time;
	+ bool rv_primed;
	+};
	+
	+struct rapl_cpu {
	+ struct pmc_hw rc_hw[RAPL_MAX_NPMCS];
	+ struct rapl_value rc_value[RAPL_MAX_NPMCS];
	+ struct mtx rc_mtx;
	+ int rc_nalloc; /* allocated RAPL PMCs on this CPU */
	+};
	+
	+static struct rapl_cpu **rapl_pcpu;
	+
	+static struct rapl_event rapl_events[RAPL_MAX_NPMCS];
	+static struct pmc_descr rapl_pmcdesc[RAPL_MAX_NPMCS];
	+static int rapl_npmcs;
	+static int rapl_ri;
	+
	+static struct callout rapl_guard_callout;
	+static sbintime_t rapl_guard_sbt;
	+static int rapl_nalloc;
	+static cpuset_t rapl_cpus; /* CPUs with an allocated RAPL PMC */
	+
	+static struct mtx rapl_alloc_mtx;
	+
	+/* Convert energy ticks to microjoules without overflowing uint64_t. */
	+static uint64_t
	+rapl_raw_to_uj(uint64_t raw, uint32_t shift)
	+{
	+ uint64_t unit, whole, frac;
	+
	+ unit = 1ULL << shift;
	+ whole = raw / unit;
	+ frac = raw % unit;
	+ return (whole * 1000000ULL + (frac * 1000000ULL) / unit);
	+}
	+
	+/* Fold a 32-bit MSR reading into the 64-bit accumulator, can recover one wrap. */
	+static void
	+rapl_update_delta(struct rapl_value *val, uint64_t cur)
	+{
	+ sbintime_t now = sbinuptime();
	+ uint64_t diff;
	+
	+ cur &= UINT32_MAX;
	+ if (!val->rv_primed) {
	+ val->rv_prev = cur;
	+ val->rv_prev_time = now;
	+ val->rv_primed = true;
	+ return;
	+ }
	+ /* Skip sub-ms re-samples; the next sample folds the full interval. */
	+ if (now - val->rv_prev_time < SBT_1MS)
	+ return;
	+ if (cur >= val->rv_prev)
	+ diff = cur - val->rv_prev;
	+ else
	+ diff = (UINT32_MAX - val->rv_prev) + cur + 1;
	+ val->rv_accum += diff;
	+ val->rv_prev = cur;
	+ val->rv_prev_time = now;
	+}
	+
	+/* Sample one row's MSR on the current CPU and return the folded accumulator. */
	+static uint64_t
	+rapl_sample_row(int cpu, int ri)
	+{
	+ struct rapl_cpu *rc;
	+ uint64_t accum, cur;
	+
	+ rc = rapl_pcpu[cpu];
	+ KASSERT(rc != NULL, ("[rapl,%d] null pcpu state cpu %d", __LINE__,
	+ cpu));
	+ mtx_lock_spin(&rc->rc_mtx);
	+ if (rdmsr_safe(rapl_events[ri].re_msr, &cur) == 0)
	+ rapl_update_delta(&rc->rc_value[ri], cur);
	+ accum = rc->rc_value[ri].rv_accum;
	+ mtx_unlock_spin(&rc->rc_mtx);
	+ return (accum);
	+}
	+
	+/* Guard rendezvous handler: sample every row on this CPU. */
	+static void
	+rapl_guard_handler(void *arg __unused)
	+{
	+ int cpu = curcpu;
	+ int ri;
	+
	+ for (ri = 0; ri < rapl_npmcs; ri++)
	+ (void)rapl_sample_row(cpu, ri);
	+}
	+
	+static void rapl_guard_tick(void *arg);
	+
	+/* (Re)arm the guard callout. Caller holds rapl_alloc_mtx. */
	+static void
	+rapl_guard_schedule(void)
	+{
	+ callout_reset_sbt(&rapl_guard_callout, rapl_guard_sbt,
	+ rapl_guard_sbt / 10, rapl_guard_tick, NULL, 0);
	+}
	+
	+/* Periodic overflow guard. */
	+static void
	+rapl_guard_tick(void *arg __unused)
	+{
	+ cpuset_t cpus;
	+
	+ mtx_lock(&rapl_alloc_mtx);
	+ cpus = rapl_cpus;
	+ mtx_unlock(&rapl_alloc_mtx);
	+
	+ if (!CPU_EMPTY(&cpus))
	+ smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier,
	+ rapl_guard_handler, smp_no_rendezvous_barrier, NULL);
	+
	+ /* Keep firing while any RAPL PMC remains allocated. */
	+ mtx_lock(&rapl_alloc_mtx);
	+ if (rapl_nalloc > 0)
	+ rapl_guard_schedule();
	+ mtx_unlock(&rapl_alloc_mtx);
	+}
	+
	+static int
	+rapl_allocate_pmc(int cpu, int ri, struct pmc *pm __unused,
	+ const struct pmc_op_pmcallocate *a)
	+{
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row index %d", __LINE__, ri));
	+
	+ if (a->pm_class != PMC_CLASS_RAPL)
	+ return (EINVAL);
	+
	+ if (a->pm_mode != PMC_MODE_SC)
	+ return (EINVAL);
	+
	+ /* Power side channel (PLATYPUS): require privilege even if syspmcs bypass is set. */
	+ if (priv_check(curthread, PRIV_PMC_SYSTEM) != 0)
	+ return (EPERM);
	+
	+ /* Reject events this vendor does not expose (e.g. DRAM on AMD). */
	+ if (a->pm_ev != rapl_events[ri].re_ev)
	+ return (EINVAL);
	+
	+ /* Arm the guard on the first allocation (per-CPU and global). */
	+ mtx_lock(&rapl_alloc_mtx);
	+ if (rapl_pcpu[cpu]->rc_nalloc++ == 0)
	+ CPU_SET(cpu, &rapl_cpus);
	+ if (rapl_nalloc++ == 0)
	+ rapl_guard_schedule();
	+ mtx_unlock(&rapl_alloc_mtx);
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_config_pmc(int cpu, int ri, struct pmc *pm)
	+{
	+ struct pmc_hw *phw;
	+
	+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ phw = &rapl_pcpu[cpu]->rc_hw[ri];
	+
	+ KASSERT(pm == NULL \|\| phw->phw_pmc == NULL,
	+ ("[rapl,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
	+ pm, phw->phw_pmc));
	+
	+ phw->phw_pmc = pm;
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_describe(int cpu, int ri, struct pmc_info pi, struct pmc *ppmc)
	+{
	+ const struct pmc_descr *pd;
	+ struct pmc_hw *phw;
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ phw = &rapl_pcpu[cpu]->rc_hw[ri];
	+ pd = &rapl_pmcdesc[ri];
	+
	+ strlcpy(pi->pm_name, pd->pd_name, sizeof(pi->pm_name));
	+ pi->pm_class = pd->pd_class;
	+
	+ if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
	+ pi->pm_enabled = TRUE;
	+ *ppmc = phw->phw_pmc;
	+ } else {
	+ pi->pm_enabled = FALSE;
	+ *ppmc = NULL;
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_get_config(int cpu, int ri, struct pmc **ppm)
	+{
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ *ppm = rapl_pcpu[cpu]->rc_hw[ri].phw_pmc;
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_pcpu_init(struct pmc_mdep *md __unused, int cpu)
	+{
	+ struct pmc_cpu *pc;
	+ struct rapl_cpu *rapl_pc;
	+ int ri, n;
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal cpu %d", __LINE__, cpu));
	+ KASSERT(rapl_pcpu, ("[rapl,%d] null pcpu", __LINE__));
	+ KASSERT(rapl_pcpu[cpu] == NULL, ("[rapl,%d] non-null per-cpu",
	+ __LINE__));
	+
	+ rapl_pc = malloc(sizeof(struct rapl_cpu), M_PMC, M_WAITOK \| M_ZERO);
	+ mtx_init(&rapl_pc->rc_mtx, "rapl-cpu", NULL, MTX_SPIN);
	+
	+ for (n = 0; n < rapl_npmcs; n++)
	+ rapl_pc->rc_hw[n].phw_state = PMC_PHW_FLAG_IS_ENABLED \|
	+ PMC_PHW_CPU_TO_STATE(cpu) \| PMC_PHW_INDEX_TO_STATE(n) \|
	+ PMC_PHW_FLAG_IS_SHAREABLE;
	+
	+ rapl_pcpu[cpu] = rapl_pc;
	+
	+ KASSERT(pmc_pcpu, ("[rapl,%d] null generic pcpu", __LINE__));
	+
	+ pc = pmc_pcpu[cpu];
	+
	+ KASSERT(pc, ("[rapl,%d] null generic per-cpu", __LINE__));
	+
	+ for (n = 0; n < rapl_npmcs; n++) {
	+ ri = rapl_ri + n;
	+ pc->pc_hwpmcs[ri] = &rapl_pc->rc_hw[n];
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_pcpu_fini(struct pmc_mdep *md __unused, int cpu)
	+{
	+ struct pmc_cpu *pc;
	+ int ri, n;
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal cpu %d", __LINE__, cpu));
	+ KASSERT(rapl_pcpu[cpu] != NULL, ("[rapl,%d] null pcpu", __LINE__));
	+ KASSERT(rapl_pcpu[cpu]->rc_nalloc == 0,
	+ ("[rapl,%d] %d PMCs still allocated on cpu %d", __LINE__,
	+ rapl_pcpu[cpu]->rc_nalloc, cpu));
	+
	+ /* Last release already drained the guard, so no handler can race here. */
	+ mtx_destroy(&rapl_pcpu[cpu]->rc_mtx);
	+ free(rapl_pcpu[cpu], M_PMC);
	+ rapl_pcpu[cpu] = NULL;
	+
	+ pc = pmc_pcpu[cpu];
	+ for (n = 0; n < rapl_npmcs; n++) {
	+ ri = rapl_ri + n;
	+ pc->pc_hwpmcs[ri] = NULL;
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_read_pmc(int cpu, int ri, struct pmc pm, pmc_value_t v)
	+{
	+ enum pmc_mode mode __diagused;
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal ri %d", __LINE__, ri));
	+
	+ mode = PMC_TO_MODE(pm);
	+
	+ KASSERT(mode == PMC_MODE_SC,
	+ ("[rapl,%d] illegal pmc mode %d", __LINE__, mode));
	+
	+ PMCDBG1(MDP,REA,1, "rapl-read id=%d", ri);
	+
	+ /* Bound to cpu by hwpmc, so rdmsr reads this CPU's domain (see DOMWIDE). */
	+ *v = rapl_raw_to_uj(rapl_sample_row(cpu, ri), rapl_events[ri].re_unit);
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
	+{
	+ struct pmc_hw *phw __diagused;
	+ bool last;
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ phw = &rapl_pcpu[cpu]->rc_hw[ri];
	+
	+ KASSERT(phw->phw_pmc == NULL,
	+ ("[rapl,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
	+
	+ mtx_lock(&rapl_alloc_mtx);
	+ KASSERT(rapl_pcpu[cpu]->rc_nalloc > 0 && rapl_nalloc > 0,
	+ ("[rapl,%d] release underflow", __LINE__));
	+ if (--rapl_pcpu[cpu]->rc_nalloc == 0)
	+ CPU_CLR(cpu, &rapl_cpus);
	+ last = (--rapl_nalloc == 0);
	+ mtx_unlock(&rapl_alloc_mtx);
	+
	+ /* Last release: drain the guard (sleepable here, mutex already dropped). */
	+ if (last)
	+ callout_drain(&rapl_guard_callout);
	+
	+ return (0);
	+}
	+
	+static int
	+rapl_start_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused)
	+{
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ return (0); /* RAPL counters are always running. */
	+}
	+
	+static int
	+rapl_stop_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused)
	+{
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ return (0); /* RAPL counters cannot be stopped. */
	+}
	+
	+static int
	+rapl_write_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused,
	+ pmc_value_t v __unused)
	+{
	+
	+ KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	+ ("[rapl,%d] illegal CPU value %d", __LINE__, cpu));
	+ KASSERT(ri >= 0 && ri < rapl_npmcs,
	+ ("[rapl,%d] illegal row-index %d", __LINE__, ri));
	+
	+ /* Energy counters are not writable; refuse silently like TSC. */
	+ return (0);
	+}
	+
	+/* Fault-safe RAPL MSR presence probe on the current CPU. */
	+static bool
	+rapl_msr_present(uint32_t msr)
	+{
	+ uint64_t v;
	+
	+ return (rdmsr_safe(msr, &v) == 0);
	+}
	+
	+/* Append an event row to the table if its MSR responds on this hardware. */
	+static void
	+rapl_add_event(enum pmc_event ev, uint32_t msr, uint32_t unit,
	+ const char *name)
	+{
	+
	+ if (!rapl_msr_present(msr))
	+ return;
	+ rapl_events[rapl_npmcs].re_ev = ev;
	+ rapl_events[rapl_npmcs].re_msr = msr;
	+ rapl_events[rapl_npmcs].re_unit = unit;
	+ rapl_pmcdesc[rapl_npmcs].pd_class = PMC_CLASS_RAPL;
	+ rapl_pmcdesc[rapl_npmcs].pd_caps = RAPL_CAPS;
	+ rapl_pmcdesc[rapl_npmcs].pd_width = 64;
	+ strlcpy(rapl_pmcdesc[rapl_npmcs].pd_name, name,
	+ sizeof(rapl_pmcdesc[rapl_npmcs].pd_name));
	+ rapl_npmcs++;
	+}
	+
	+/* Guard interval: half the worst-case wrap period at RAPL_GUARD_WATT. */
	+static sbintime_t
	+rapl_compute_guard_sbt(uint32_t shift)
	+{
	+ uint64_t max_energy_uj, guard_ms;
	+
	+ max_energy_uj = rapl_raw_to_uj(UINT32_MAX, shift);
	+ guard_ms = max_energy_uj / (2000ULL * RAPL_GUARD_WATT);
	+ if (guard_ms < RAPL_GUARD_MIN_MS)
	+ guard_ms = RAPL_GUARD_MIN_MS;
	+ else if (guard_ms > RAPL_GUARD_MAX_MS)
	+ guard_ms = RAPL_GUARD_MAX_MS;
	+ return (guard_ms * SBT_1MS);
	+}
	+
	+/* Fixed 2^-16 J DRAM unit, not the ESU (HSX/KNL only -- not SPR/EMR/GNR). */
	+static bool
	+rapl_intel_fixed_dram_unit(void)
	+{
	+
	+ if (CPUID_TO_FAMILY(cpu_id) != 0x6)
	+ return (false);
	+ switch (CPUID_TO_MODEL(cpu_id)) {
	+ case 0x3f: /* Haswell-EP */
	+ case 0x4f: /* Broadwell-EP */
	+ case 0x55: /* Skylake/Cascade Lake/Cooper Lake-SP */
	+ case 0x56: /* Broadwell-DE */
	+ case 0x57: /* Xeon Phi KNL */
	+ case 0x6a: /* Ice Lake-SP */
	+ case 0x6c: /* Ice Lake-D */
	+ case 0x85: /* Xeon Phi KNM */
	+ return (true);
	+ default:
	+ return (false);
	+ }
	+}
	+
	+int
	+pmc_rapl_initialize(struct pmc_mdep *md, int maxcpu, int classindex)
	+{
	+ struct pmc_classdep *pcd;
	+ uint32_t unit_msr, pkg_msr, cores_msr, dram_msr;
	+ uint32_t esu, dram_unit, max_unit;
	+ uint64_t unit_val;
	+ int i;
	+
	+ KASSERT(md != NULL, ("[rapl,%d] md is NULL", __LINE__));
	+ KASSERT(md->pmd_nclass >= 1, ("[rapl,%d] dubious md->nclass %d",
	+ __LINE__, md->pmd_nclass));
	+
	+ /* Select the per-vendor MSR set. */
	+ switch (cpu_vendor_id) {
	+ case CPU_VENDOR_AMD:
	+ case CPU_VENDOR_HYGON:
	+ unit_msr = MSR_AMD_RAPL_POWER_UNIT;
	+ pkg_msr = MSR_AMD_PKG_ENERGY_STATUS;
	+ cores_msr = MSR_AMD_CORE_ENERGY_STATUS;
	+ dram_msr = 0; /* AMD has no DRAM domain */
	+ break;
	+ case CPU_VENDOR_INTEL:
	+ unit_msr = MSR_RAPL_POWER_UNIT;
	+ pkg_msr = MSR_PKG_ENERGY_STATUS;
	+ cores_msr = MSR_PP0_ENERGY_STATUS;
	+ dram_msr = MSR_DRAM_ENERGY_STATUS;
	+ break;
	+ default:
	+ return (ENXIO);
	+ }
	+
	+ /* Decode the energy unit. */
	+ if (rdmsr_safe(unit_msr, &unit_val) != 0)
	+ return (ENXIO);
	+ esu = (unit_val >> 8) & 0x1f;
	+ dram_unit = rapl_intel_fixed_dram_unit() ? 16 : esu;
	+
	+ /* Build the event table from the MSRs that actually respond. */
	+ rapl_npmcs = 0;
	+ rapl_add_event(PMC_EV_RAPL_ENERGY_PKG, pkg_msr, esu,
	+ "RAPL_ENERGY_PKG");
	+ rapl_add_event(PMC_EV_RAPL_ENERGY_CORES, cores_msr, esu,
	+ "RAPL_ENERGY_CORES");
	+ if (dram_msr != 0)
	+ rapl_add_event(PMC_EV_RAPL_ENERGY_DRAM, dram_msr, dram_unit,
	+ "RAPL_ENERGY_DRAM");
	+
	+ /* No RAPL energy MSR responded. */
	+ if (rapl_npmcs == 0)
	+ return (ENXIO);
	+
	+ /* Size the guard for the fastest-wrapping row (largest unit shift). */
	+ max_unit = 0;
	+ for (i = 0; i < rapl_npmcs; i++)
	+ max_unit = MAX(max_unit, rapl_events[i].re_unit);
	+ rapl_guard_sbt = rapl_compute_guard_sbt(max_unit);
	+ rapl_nalloc = 0;
	+ CPU_ZERO(&rapl_cpus);
	+
	+ mtx_init(&rapl_alloc_mtx, "rapl-alloc", NULL, MTX_DEF);
	+ /* It does not need associated mutex, the handler locks itself. */
	+ callout_init(&rapl_guard_callout, 1);
	+
	+ rapl_pcpu = malloc(sizeof(struct rapl_cpu ) maxcpu, M_PMC,
	+ M_ZERO \| M_WAITOK);
	+
	+ pcd = &md->pmd_classdep[classindex];
	+
	+ pcd->pcd_caps = RAPL_CAPS;
	+ pcd->pcd_class = PMC_CLASS_RAPL;
	+ pcd->pcd_num = rapl_npmcs;
	+ pcd->pcd_ri = md->pmd_npmc;
	+ pcd->pcd_width = 64;
	+
	+ pcd->pcd_allocate_pmc = rapl_allocate_pmc;
	+ pcd->pcd_config_pmc = rapl_config_pmc;
	+ pcd->pcd_describe = rapl_describe;
	+ pcd->pcd_get_config = rapl_get_config;
	+ pcd->pcd_pcpu_init = rapl_pcpu_init;
	+ pcd->pcd_pcpu_fini = rapl_pcpu_fini;
	+ pcd->pcd_read_pmc = rapl_read_pmc;
	+ pcd->pcd_release_pmc = rapl_release_pmc;
	+ pcd->pcd_start_pmc = rapl_start_pmc;
	+ pcd->pcd_stop_pmc = rapl_stop_pmc;
	+ pcd->pcd_write_pmc = rapl_write_pmc;
	+
	+ rapl_ri = md->pmd_npmc;
	+ md->pmd_npmc += rapl_npmcs;
	+
	+ return (0);
	+}
	+
	+void
	+pmc_rapl_finalize(struct pmc_mdep *md __unused)
	+{
	+ PMCDBG0(MDP, INI, 1, "rapl-finalize");
	+
	+ if (rapl_pcpu == NULL)
	+ return;
	+
	+ KASSERT(rapl_nalloc == 0, ("[rapl,%d] %d PMCs still allocated",
	+ __LINE__, rapl_nalloc));
	+ for (int i = 0; i < pmc_cpu_max(); i++)
	+ KASSERT(rapl_pcpu[i] == NULL, ("[rapl,%d] non-null pcpu cpu %d",
	+ __LINE__, i));
	+
	+ mtx_destroy(&rapl_alloc_mtx);
	+
	+ free(rapl_pcpu, M_PMC);
	+ rapl_pcpu = NULL;
	+}
	diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h
	--- a/sys/dev/hwpmc/pmc_events.h
	+++ b/sys/dev/hwpmc/pmc_events.h
	@@ -53,6 +53,15 @@
	#define __PMC_EV_ALIAS_TSC() \
	__PMC_EV_ALIAS("cycles", TSC_TSC)

	+/* RAPL energy counters. */
	+#define __PMC_EV_RAPL() \
	+ __PMC_EV(RAPL, ENERGY_PKG) \
	+ __PMC_EV(RAPL, ENERGY_CORES) \
	+ __PMC_EV(RAPL, ENERGY_DRAM)
	+
	+#define PMC_EV_RAPL_FIRST PMC_EV_RAPL_ENERGY_PKG
	+#define PMC_EV_RAPL_LAST PMC_EV_RAPL_ENERGY_DRAM
	+
	/*
	* Software events are dynamically defined.
	*/
	@@ -2428,6 +2437,7 @@
	* 0x14500 0x0020 ARM DMC-620 clkdiv2 events
	* 0x14520 0x0080 ARM DMC-620 clk events
	* 0x14600 0x0100 ARM CMN-600 events
	+ * 0x14700 0x0100 AMD/Intel RAPL energy events
	* 0x20000 0x1000 Software events
	*/
	#define __PMC_EVENTS() \
	@@ -2454,7 +2464,9 @@
	__PMC_EV_BLOCK(DMC620_PMU_C, 0x14520) \
	__PMC_EV_DMC620_PMU_C() \
	__PMC_EV_BLOCK(CMN600_PMU, 0x14600) \
	- __PMC_EV_CMN600_PMU()
	+ __PMC_EV_CMN600_PMU() \
	+ __PMC_EV_BLOCK(RAPL, 0x14700) \
	+ __PMC_EV_RAPL()

	#define PMC_EVENT_FIRST PMC_EV_TSC_TSC
	#define PMC_EVENT_LAST PMC_EV_SOFT_LAST
	diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h
	--- a/sys/i386/include/pmc_mdep.h
	+++ b/sys/i386/include/pmc_mdep.h
	@@ -52,6 +52,7 @@
	#include <dev/hwpmc/hwpmc_amd.h>
	#include <dev/hwpmc/hwpmc_ibs.h>
	#include <dev/hwpmc/hwpmc_core.h>
	+#include <dev/hwpmc/hwpmc_rapl.h>
	#include <dev/hwpmc/hwpmc_tsc.h>
	#include <dev/hwpmc/hwpmc_uncore.h>

	diff --git a/sys/modules/hwpmc/Makefile b/sys/modules/hwpmc/Makefile
	--- a/sys/modules/hwpmc/Makefile
	+++ b/sys/modules/hwpmc/Makefile
	@@ -16,8 +16,8 @@
	.endif

	.if ${MACHINE_CPUARCH} == "amd64"
	-SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_ibs.c hwpmc_intel.c hwpmc_tsc.c
	-SRCS+= hwpmc_x86.c hwpmc_uncore.c
	+SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_ibs.c hwpmc_intel.c hwpmc_rapl.c
	+SRCS+= hwpmc_tsc.c hwpmc_x86.c hwpmc_uncore.c
	.endif

	.if ${MACHINE_CPUARCH} == "arm"
	@@ -30,7 +30,7 @@

	.if ${MACHINE_CPUARCH} == "i386"
	SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_ibs.c hwpmc_intel.c
	-SRCS+= hwpmc_tsc.c hwpmc_x86.c hwpmc_uncore.c
	+SRCS+= hwpmc_rapl.c hwpmc_tsc.c hwpmc_x86.c hwpmc_uncore.c
	.endif

	.if ${MACHINE_CPUARCH} == "powerpc"
	diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
	--- a/sys/sys/pmc.h
	+++ b/sys/sys/pmc.h
	@@ -156,7 +156,8 @@
	__PMC_CLASS(POWER8, 0x15, "IBM POWER8 class") \
	__PMC_CLASS(DMC620_PMU_CD2, 0x16, "ARM DMC620 Memory Controller PMU CLKDIV2") \
	__PMC_CLASS(DMC620_PMU_C, 0x17, "ARM DMC620 Memory Controller PMU CLK") \
	- __PMC_CLASS(CMN600_PMU, 0x18, "Arm CoreLink CMN600 Coherent Mesh Network PMU")
	+ __PMC_CLASS(CMN600_PMU, 0x18, "Arm CoreLink CMN600 Coherent Mesh Network PMU") \
	+ __PMC_CLASS(RAPL, 0x19, "AMD/Intel RAPL energy counters")

	enum pmc_class {
	#undef __PMC_CLASS
	@@ -165,7 +166,7 @@
	};

	#define PMC_CLASS_FIRST PMC_CLASS_TSC
	-#define PMC_CLASS_LAST PMC_CLASS_CMN600_PMU
	+#define PMC_CLASS_LAST PMC_CLASS_RAPL

	/*
	* A PMC can be in the following states:

File Metadata

Mime Type: text/plain
Expires: Tue, Jul 7, 4:51 AM (14 h, 46 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 34784127
Default Alt Text: D58028.diff (22 KB)

D58028.diffNo OneTemporaryActions

D58028.diffView Options

File Metadata

Event Timeline

D58028.diff
No OneTemporary
Actions

D58028.diff
View Options