Page MenuHomeFreeBSD
Files F176529954

D60569.id189848.diff
No OneTemporary
Actions

D60569.id189848.diff
View Options

diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h
--- a/sys/dev/hwpmc/hwpmc_amd.h
+++ b/sys/dev/hwpmc/hwpmc_amd.h
@@ -37,6 +37,9 @@
#define EXTPERFMON_DF_PMCS(x) (((x) >> 10) & 0x3F)
#define EXTPERFMON_UMC_PMCS(x) (((x) >> 16) & 0xFF)
#define EXTPERFMON_PERFMONV2(x) ((x) & 0x1)
+#define EXTPERFMON_LBR_V2(x) (((x) >> 1) & 0x1) /* EAX bit 1: LbrExtV2 */
+#define EXTPERFMON_LBR_FREEZE(x) (((x) >> 2) & 0x1) /* EAX bit 2 */
+#define EXTPERFMON_LBR_DEPTH(x) (((x) >> 4) & 0x3F) /* EBX bits 9:4 */
/* AMD K8 PMCs */
#define AMD_PMC_EVSEL_0 0xC0010000
@@ -72,7 +75,31 @@
#define AMD_PMC_GLOBAL_STATUS 0xC0000300 /* RO */
#define AMD_PMC_GLOBAL_CTL 0xC0000301 /* RW */
#define AMD_PMC_GLOBAL_STATUS_CLR 0xC0000302 /* WO */
-/* A future LBR v2 change adds GLOBAL_STATUS.LBRS_FROZEN (bit 58) here. */
+#define AMD_PMC_GLOBAL_STATUS_LBRS_FROZEN (1ULL << 58)
+
+/*
+ * LBR v2 (LbrExtV2): a ring of From/To branch-record MSR pairs, frozen at
+ * core-PMC overflow and re-armed by clearing LBRS_FROZEN in GLOBAL_STATUS.
+ */
+#define AMD_MSR_SAMP_BR_FROM 0xC0010300 /* From at +2i, To at +2i+1 */
+#define AMD_MSR_LBR_SELECT 0xC000010E
+#define AMD_MSR_DBG_EXTN_CFG 0xC000010F
+#define AMD_DBG_EXTN_CFG_LBRV2EN (1ULL << 6)
+#define AMD_DEBUGCTL_FREEZE_LBRS_ON_PMI (1ULL << 11) /* MSR_DEBUGCTLMSR */
+/* LBR_SELECT suppress-mode filter: a set bit stops recording that type. */
+#define AMD_LBR_SELECT_CPL0 (1ULL << 0) /* branches ending in CPL 0 */
+#define AMD_LBR_SELECT_CPLGT0 (1ULL << 1) /* branches ending in CPL > 0 */
+#define AMD_LBR_MAX_DEPTH 16
+/* To-register flags used for the record validity test. */
+#define AMD_LBR_TO_VALID (1ULL << 63)
+#define AMD_LBR_TO_SPEC (1ULL << 62)
+#define AMD_LBR_TO_RESERVED (1ULL << 61) /* erratum 1452: skip if set */
+#define AMD_LBR_FROM_MISPREDICT (1ULL << 63)
+/*
+ * IP is bits [57:0]. hwpmc canonicalizes the field from the CPU's linear
+ * address width before logging, so consumers sign-extend from bit 57.
+ */
+#define AMD_LBR_IP(x) ((uint64_t)((int64_t)(((uint64_t)(x)) << 6) >> 6))
#define AMD_PMC_COUNTERMASK 0xFF000000
#define AMD_PMC_PRECISERETIRE (1ULL << 43) /* Only valid for PERF_CTL2 */
@@ -235,6 +262,7 @@
/* MD extension for 'struct pmc' */
struct pmc_md_amd_pmc {
uint64_t pm_amd_evsel;
+ uint64_t pm_amd_lbr_select; /* LBR_SELECT filter image */
};
#endif /* _KERNEL */
diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -52,6 +52,8 @@
#include <machine/md_var.h>
#include <machine/specialreg.h>
+#include <dev/hwpmc/hwpmc_amd_lbr.h>
+
#define OVERFLOW_WAIT_COUNT 50
DPCPU_DEFINE_STATIC(uint32_t, nmi_counter);
@@ -68,6 +70,8 @@
static int amd_npmcs;
static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs, amd_umc_npmcs;
+static int amd_lbr_depth; /* LBR v2 stack depth; 0 = none */
+static bool amd_lbr_freeze; /* LbrAndPmcFreeze supported */
struct amd_event_code_map {
enum pmc_event pe_ev; /* enum value */
@@ -192,9 +196,20 @@
volatile u_int pc_global_mask;
volatile u_int pc_virtual_mask;
volatile u_int pc_gate_depth;
+ uint32_t pc_lbr_mask; /* core rows with LBR active */
+ uint64_t pc_lbr_select; /* wanted LBR_SELECT image */
+ uint64_t pc_lbr_hw_select; /* LBR_SELECT as programmed */
+ uint64_t pc_lbr_extn_cfg; /* DBG_EXTN_CFG, LBRV2EN clear */
+ bool pc_lbr_suspended; /* between switch-out and -in */
+ bool pc_lbr_hw_on; /* LBRV2EN set in hardware */
+ bool pc_lbr_dirty; /* ring needs a reset first */
+ volatile u_int pc_lbr_busy; /* LBR state in transition */
};
static struct amd_cpu **amd_pcpu;
+static struct amd_lbr_filter amd_lbr_ts;
+static struct amd_lbr_filter *amd_lbr_ss;
+
/* Populated by amd_init_policy(); PRECISERETIRE is OR-ed in per-allocation. */
static uint64_t amd_core_allowed_mask;
static uint64_t amd_l3_allowed_mask;
@@ -232,6 +247,10 @@
&amd_perfmon_v2, 0,
"AMD PerfMonV2 global-control path selected (read-only)");
+SYSCTL_INT(_kern_hwpmc, OID_AUTO, amd_lbr_depth, CTLFLAG_RD,
+ &amd_lbr_depth, 0,
+ "AMD LBR v2 branch-record stack depth (0 = not present)");
+
static void
amd_init_policy(void)
{
@@ -270,6 +289,203 @@
}
}
+static uint64_t
+amd_lbr_filter_for_caps(uint32_t caps)
+{
+
+ if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == PMC_CAP_USER)
+ return (AMD_LBR_SELECT_CPL0);
+ if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == PMC_CAP_SYSTEM)
+ return (AMD_LBR_SELECT_CPLGT0);
+ return (0);
+}
+
+static int
+amd_lbr_reserve_filter(int cpu, enum pmc_mode mode, uint64_t filter)
+{
+ int error;
+
+ sx_assert(&pmc_sx, SX_XLOCKED);
+ KASSERT(PMC_IS_VIRTUAL_MODE(mode) ||
+ (cpu >= 0 && cpu < pmc_cpu_max()),
+ ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
+ error = amd_lbr_filter_reserve(&amd_lbr_ts, amd_lbr_ss, pmc_cpu_max(),
+ PMC_IS_VIRTUAL_MODE(mode), cpu, filter);
+ if (error != 0)
+ return (EXTERROR(error,
+ "AMD LBR filter conflicts with an allocated LBR PMC"));
+ return (0);
+}
+
+static void
+amd_lbr_release_filter(int cpu, enum pmc_mode mode, uint64_t filter)
+{
+ struct amd_lbr_filter *slot;
+
+ sx_assert(&pmc_sx, SX_XLOCKED);
+ KASSERT(PMC_IS_VIRTUAL_MODE(mode) ||
+ (cpu >= 0 && cpu < pmc_cpu_max()),
+ ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
+ slot = PMC_IS_VIRTUAL_MODE(mode) ? &amd_lbr_ts : &amd_lbr_ss[cpu];
+ KASSERT(slot->refs > 0 && slot->select == filter,
+ ("[amd,%d] invalid LBR filter release", __LINE__));
+ amd_lbr_filter_release(slot);
+}
+
+/* Clear every record and re-arm a ring frozen by an earlier PMI. */
+static void
+amd_lbr_reset(void)
+{
+ int i;
+
+ for (i = 0; i < amd_lbr_depth; i++) {
+ wrmsr(AMD_MSR_SAMP_BR_FROM + 2 * i, 0);
+ wrmsr(AMD_MSR_SAMP_BR_FROM + 2 * i + 1, 0);
+ }
+ wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, AMD_PMC_GLOBAL_STATUS_LBRS_FROZEN);
+}
+
+/*
+ * Bring the LBR hardware of the current CPU in line with the software state.
+ *
+ * Recording is wanted while at least one started core row requests LBR and
+ * the CPU is not between a switch-out and a switch-in. pc_lbr_hw_on and
+ * pc_lbr_hw_select cache the hardware state so that only real transitions
+ * touch MSRs. pc_lbr_dirty means that the ring may hold records from
+ * another address space, filter or owner, and must be cleared before
+ * recording resumes. The dirty flag is set whenever recording stops, so a
+ * ring that was never enabled since its last reset is not cleared again.
+ *
+ * Invariant: pc_lbr_hw_on implies pc_lbr_mask != 0 && !pc_lbr_suspended.
+ * When pc_lbr_mask is zero, pc_lbr_suspended may be stale; it is only a
+ * hint and is corrected by the next switch-in or system-mode activation.
+ *
+ * Called with interrupts disabled and pc_lbr_busy set, on the owning CPU.
+ */
+static void
+amd_lbr_sync(struct amd_cpu *pac)
+{
+ bool want;
+
+ want = pac->pc_lbr_mask != 0 && !pac->pc_lbr_suspended;
+ if (pac->pc_lbr_hw_on) {
+ if (want && !pac->pc_lbr_dirty &&
+ pac->pc_lbr_hw_select == pac->pc_lbr_select)
+ return;
+ wrmsr(AMD_MSR_DBG_EXTN_CFG, pac->pc_lbr_extn_cfg);
+ pac->pc_lbr_hw_on = false;
+ pac->pc_lbr_dirty = true;
+ }
+ if (!want)
+ return;
+ if (pac->pc_lbr_dirty) {
+ amd_lbr_reset();
+ pac->pc_lbr_dirty = false;
+ }
+ if (pac->pc_lbr_hw_select != pac->pc_lbr_select) {
+ wrmsr(AMD_MSR_LBR_SELECT, pac->pc_lbr_select);
+ pac->pc_lbr_hw_select = pac->pc_lbr_select;
+ }
+ wrmsr(AMD_MSR_DBG_EXTN_CFG,
+ pac->pc_lbr_extn_cfg | AMD_DBG_EXTN_CFG_LBRV2EN);
+ pac->pc_lbr_hw_on = true;
+}
+
+/*
+ * A late NMI can arrive at any point. Exclude ordinary interrupts (the
+ * hardclock sample sweep may restart a stalled row) and tell the NMI handler
+ * not to read or change the LBR hardware until the ring, filter and row mask
+ * form a consistent state. Core counters keep running: an overflow during
+ * the window is still sampled, with an empty LBR payload.
+ */
+static register_t
+amd_lbr_update_begin(struct amd_cpu *pac)
+{
+ register_t flags;
+
+ flags = intr_disable();
+ KASSERT(pac == amd_pcpu[curcpu],
+ ("[amd,%d] LBR update for a foreign CPU", __LINE__));
+ KASSERT(atomic_load_int(&pac->pc_lbr_busy) == 0,
+ ("[amd,%d] nested LBR update", __LINE__));
+ /* Only an NMI on this CPU reads the state; a compiler fence is enough. */
+ atomic_store_int(&pac->pc_lbr_busy, 1);
+ atomic_interrupt_fence();
+ return (flags);
+}
+
+static void
+amd_lbr_update_end(struct amd_cpu *pac, register_t flags)
+{
+
+ atomic_interrupt_fence();
+ atomic_store_int(&pac->pc_lbr_busy, 0);
+ intr_restore(flags);
+}
+
+static void
+amd_lbr_activate(int cpu, int ri, struct pmc *pm)
+{
+ struct amd_cpu *pac;
+ register_t flags;
+ u_int mask;
+
+ if ((pm->pm_caps & PMC_CAP_LBR) == 0)
+ return;
+ pac = amd_pcpu[cpu];
+ mask = 1U << ri;
+ /* A stalled row keeps its bit; its restart is then free. */
+ if ((pac->pc_lbr_mask & mask) != 0)
+ return;
+
+ flags = amd_lbr_update_begin(pac);
+ if (pac->pc_lbr_mask == 0) {
+ pac->pc_lbr_select = pm->pm_md.pm_amd.pm_amd_lbr_select;
+ pac->pc_lbr_dirty = true;
+ /*
+ * The suspension flag is only a hint while the mask is empty.
+ * A thread PMC starts during switch-in, before amd_lbr_csw()
+ * lifts the suspension and resets the ring once for all
+ * incoming rows.
+ */
+ pac->pc_lbr_suspended = !PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm));
+ } else {
+ KASSERT(pac->pc_lbr_select ==
+ pm->pm_md.pm_amd.pm_amd_lbr_select,
+ ("[amd,%d] incompatible reserved LBR filter", __LINE__));
+ }
+ /*
+ * A system PMC starts in the running thread, outside a switch pair.
+ * Any thread PMC already active here belongs to that thread.
+ */
+ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
+ pac->pc_lbr_suspended = false;
+ pac->pc_lbr_mask |= mask;
+ amd_lbr_sync(pac);
+ amd_lbr_update_end(pac, flags);
+}
+
+static void
+amd_lbr_deactivate(int cpu, int ri, struct pmc *pm)
+{
+ struct amd_cpu *pac;
+ register_t flags;
+ u_int mask;
+
+ if ((pm->pm_caps & PMC_CAP_LBR) == 0)
+ return;
+ pac = amd_pcpu[cpu];
+ mask = 1U << ri;
+ /* Stop and deconfigure both deactivate; the second call is free. */
+ if ((pac->pc_lbr_mask & mask) == 0)
+ return;
+
+ flags = amd_lbr_update_begin(pac);
+ pac->pc_lbr_mask &= ~mask;
+ amd_lbr_sync(pac);
+ amd_lbr_update_end(pac, flags);
+}
+
static __inline u_int
amd_v2_counter_mask(int ri)
{
@@ -492,8 +708,10 @@
__LINE__, pm, phw->phw_pmc));
if (amd_perfmon_v2 && pm == NULL && phw->phw_pmc != NULL &&
- amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE)
+ amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE) {
+ amd_lbr_deactivate(cpu, ri, phw->phw_pmc);
amd_v2_forget_core(cpu, ri, phw->phw_pmc);
+ }
phw->phw_pmc = pm;
return (0);
@@ -572,6 +790,16 @@
((pd->pd_caps & PMC_CAP_PRECISE) == 0))
return (EINVAL);
+ /* LBR: core sampling PMCs on LBR v2 hardware only. */
+ if ((caps & PMC_CAP_LBR) != 0) {
+ if ((pd->pd_caps & PMC_CAP_LBR) == 0)
+ return (EXTERROR(EOPNOTSUPP,
+ "AMD LBR requires an LBR-capable core counter"));
+ if (!PMC_IS_SAMPLING_MODE(a->pm_mode))
+ return (EXTERROR(EINVAL,
+ "AMD LBR requires sampling mode"));
+ }
+
/* PMC_F_EV_PMU: config comes from pmu-events tables. */
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
@@ -631,6 +859,14 @@
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
pm->pm_md.pm_amd.pm_amd_evsel = config;
+ if ((caps & PMC_CAP_LBR) != 0) {
+ pm->pm_md.pm_amd.pm_amd_lbr_select =
+ amd_lbr_filter_for_caps(caps);
+ error = amd_lbr_reserve_filter(a->pm_cpu, a->pm_mode,
+ pm->pm_md.pm_amd.pm_amd_lbr_select);
+ if (error != 0)
+ return (error);
+ }
PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
ri, (uintmax_t)config);
return (0);
@@ -670,6 +906,14 @@
config |= AMD_PMC_INT;
pm->pm_md.pm_amd.pm_amd_evsel = config;
+ if ((caps & PMC_CAP_LBR) != 0) {
+ pm->pm_md.pm_amd.pm_amd_lbr_select =
+ amd_lbr_filter_for_caps(caps);
+ error = amd_lbr_reserve_filter(a->pm_cpu, a->pm_mode,
+ pm->pm_md.pm_amd.pm_amd_lbr_select);
+ if (error != 0)
+ return (error);
+ }
PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%x", ri, config);
@@ -681,7 +925,7 @@
* no-op on this architecture.
*/
static int
-amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
+amd_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw __diagused;
@@ -695,6 +939,10 @@
KASSERT(phw->phw_pmc == NULL,
("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
+ if ((pmc->pm_caps & PMC_CAP_LBR) != 0)
+ amd_lbr_release_filter(cpu, PMC_TO_MODE(pmc),
+ pmc->pm_md.pm_amd.pm_amd_lbr_select);
+
return (0);
}
@@ -790,6 +1038,9 @@
DPCPU_SET(nmi_counter, 1);
}
+ if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE)
+ amd_lbr_activate(cpu, ri, pm);
+
/* Enable EVSEL while the virtual slot's global bit is off. */
config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
wrmsr(pd->pm_evsel, config);
@@ -885,6 +1136,9 @@
wrmsr(pd->pm_evsel,
pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE);
+ if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE)
+ amd_lbr_deactivate(cpu, ri, pm);
+
/* Wait out an in-flight overflow NMI. The handler clears the status bit. */
if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE &&
PMC_IS_SAMPLING_MODE(mode)) {
@@ -1244,6 +1498,24 @@
__LINE__, cpu));
amd_v2_disable_all();
}
+ if (amd_lbr_depth > 0) {
+ /*
+ * hwpmc owns DBG_EXTN_CFG.LBRV2EN and LBR_SELECT while loaded;
+ * other DBG_EXTN_CFG bits are preserved from this snapshot.
+ * Freezing on PMI is harmless while recording is off, so set
+ * it once here instead of on every enable.
+ */
+ pac->pc_lbr_extn_cfg = rdmsr(AMD_MSR_DBG_EXTN_CFG) &
+ ~AMD_DBG_EXTN_CFG_LBRV2EN;
+ wrmsr(AMD_MSR_DBG_EXTN_CFG, pac->pc_lbr_extn_cfg);
+ if (amd_lbr_freeze)
+ wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) |
+ AMD_DEBUGCTL_FREEZE_LBRS_ON_PMI);
+ amd_lbr_reset();
+ wrmsr(AMD_MSR_LBR_SELECT, 0);
+ pac->pc_lbr_hw_select = 0;
+ pac->pc_lbr_suspended = true;
+ }
/*
* Set the content of the hardware descriptors to a known
@@ -1292,8 +1564,19 @@
("[amd,%d] nonzero desired mask on CPU %d", __LINE__, cpu));
KASSERT(atomic_load_acq_int(&pac->pc_virtual_mask) == 0,
("[amd,%d] nonzero virtual mask on CPU %d", __LINE__, cpu));
+ KASSERT(pac->pc_lbr_mask == 0,
+ ("[amd,%d] nonzero LBR mask on CPU %d", __LINE__, cpu));
amd_v2_disable_all();
}
+ if (amd_lbr_depth > 0) {
+ KASSERT(!pac->pc_lbr_hw_on,
+ ("[amd,%d] LBR still recording on CPU %d", __LINE__, cpu));
+ wrmsr(AMD_MSR_DBG_EXTN_CFG, pac->pc_lbr_extn_cfg);
+ if (amd_lbr_freeze)
+ wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) &
+ ~AMD_DEBUGCTL_FREEZE_LBRS_ON_PMI);
+ wrmsr(AMD_MSR_LBR_SELECT, 0);
+ }
amd_pcpu[cpu] = NULL;
#ifdef HWPMC_DEBUG
@@ -1324,6 +1607,7 @@
struct amd_v2_hwcheck_state {
volatile u_int avh_read_error;
volatile u_int avh_enabled;
+ volatile u_int avh_lbr_unusable; /* LBR MSRs missing or owned */
};
static void
@@ -1343,6 +1627,22 @@
if ((reg & AMD_PMC_ENABLE) != 0)
atomic_set_int(&state->avh_enabled, 1);
}
+ /*
+ * LBR is optional. A hypervisor may advertise LbrExtV2 without
+ * implementing its MSRs, and firmware or another agent may already
+ * be recording. Either way, disable LBR support instead of failing
+ * the whole module. Only read MSRs here; nothing is enabled yet.
+ */
+ if (amd_lbr_depth > 0) {
+ if (rdmsr_safe(AMD_MSR_DBG_EXTN_CFG, &reg) != 0 ||
+ (reg & AMD_DBG_EXTN_CFG_LBRV2EN) != 0 ||
+ rdmsr_safe(AMD_MSR_LBR_SELECT, &reg) != 0 ||
+ rdmsr_safe(AMD_MSR_SAMP_BR_FROM +
+ 2 * (amd_lbr_depth - 1) + 1, &reg) != 0 ||
+ (amd_lbr_freeze &&
+ rdmsr_safe(MSR_DEBUGCTLMSR, &reg) != 0))
+ atomic_set_int(&state->avh_lbr_unusable, 1);
+ }
}
/*
@@ -1361,6 +1661,7 @@
if (amd_perfmon_v2) {
state.avh_read_error = 0;
state.avh_enabled = 0;
+ state.avh_lbr_unusable = 0;
smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier,
amd_v2_hwcheck_cpu, smp_no_rendezvous_barrier, &state);
if (state.avh_read_error != 0) {
@@ -1368,6 +1669,12 @@
"or more CPUs!\n");
return (-1);
}
+ if (state.avh_lbr_unusable != 0) {
+ printf("hwpmc: AMD LBR v2 registers unavailable or in "
+ "use; LBR capture disabled\n");
+ amd_lbr_depth = 0;
+ amd_lbr_freeze = false;
+ }
if (state.avh_enabled != 0) {
printf("hwpmc: PMCs maybe in use by firmware!\n");
printf("hwpmc: Disable the PMC use in the BIOS before "
@@ -1507,6 +1814,18 @@
/* EAX bit 0 holds the PerfMonV2 flag. */
if (EXTPERFMON_PERFMONV2(regs[0]) && family >= 0x19)
amd_perfmon_v2 = true;
+#if defined(__amd64__)
+ /* LBR v2 rides on the PerfMonV2 machinery. */
+ if (amd_perfmon_v2 && EXTPERFMON_LBR_V2(regs[0])) {
+ amd_lbr_depth = EXTPERFMON_LBR_DEPTH(regs[1]);
+ if (amd_lbr_depth > AMD_LBR_MAX_DEPTH) {
+ printf("hwpmc: unsupported LBR depth %d\n",
+ amd_lbr_depth);
+ amd_lbr_depth = 0;
+ }
+ amd_lbr_freeze = EXTPERFMON_LBR_FREEZE(regs[0]) != 0;
+ }
+#endif
}
/*
@@ -1625,6 +1944,10 @@
*/
amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC,
M_WAITOK | M_ZERO);
+ if (amd_lbr_depth > 0) {
+ amd_lbr_ss = malloc(sizeof(*amd_lbr_ss) * pmc_cpu_max(), M_PMC,
+ M_WAITOK | M_ZERO);
+ }
/*
* These processors have two or three classes of PMCs: the TSC,
@@ -1717,9 +2040,11 @@
return (pmc_mdep);
error:
- free(pmc_mdep, M_PMC);
+ free(amd_lbr_ss, M_PMC);
+ amd_lbr_ss = NULL;
free(amd_pcpu, M_PMC);
amd_pcpu = NULL;
+ free(pmc_mdep, M_PMC);
free(amd_pmcdesc, M_PMC);
amd_pmcdesc = NULL;
amd_npmcs = 0;
@@ -1747,6 +2072,8 @@
KASSERT(amd_pcpu[i] == NULL,
("[amd,%d] non-null pcpu cpu %d", __LINE__, i));
+ free(amd_lbr_ss, M_PMC);
+ amd_lbr_ss = NULL;
free(amd_pcpu, M_PMC);
amd_pcpu = NULL;
diff --git a/sys/dev/hwpmc/hwpmc_amd_lbr.h b/sys/dev/hwpmc/hwpmc_amd_lbr.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/hwpmc/hwpmc_amd_lbr.h
@@ -0,0 +1,54 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _DEV_HWPMC_AMD_LBR_H_
+#define _DEV_HWPMC_AMD_LBR_H_
+
+#include <sys/types.h>
+#include <sys/errno.h>
+
+/*
+ * Filter reservations, serialized by pmc_sx. They do not depend on whether
+ * the PMC is running.
+ */
+struct amd_lbr_filter {
+ uint64_t select;
+ u_int refs;
+};
+
+static __inline int
+amd_lbr_filter_reserve(struct amd_lbr_filter *ts, struct amd_lbr_filter *ss,
+ u_int ncpu, int is_thread, u_int cpu, uint64_t select)
+{
+ u_int i;
+
+ if (ts->refs != 0 && ts->select != select)
+ return (EBUSY);
+ if (is_thread) {
+ for (i = 0; i < ncpu; i++) {
+ if (ss[i].refs != 0 && ss[i].select != select)
+ return (EBUSY);
+ }
+ if (ts->refs++ == 0)
+ ts->select = select;
+ } else {
+ if (ss[cpu].refs != 0 && ss[cpu].select != select)
+ return (EBUSY);
+ if (ss[cpu].refs++ == 0)
+ ss[cpu].select = select;
+ }
+ return (0);
+}
+
+/* The caller must hold a reservation in this slot. */
+static __inline void
+amd_lbr_filter_release(struct amd_lbr_filter *slot)
+{
+ if (--slot->refs == 0)
+ slot->select = 0;
+}
+
+#endif /* _DEV_HWPMC_AMD_LBR_H_ */
diff --git a/tests/sys/pmc/Makefile b/tests/sys/pmc/Makefile
--- a/tests/sys/pmc/Makefile
+++ b/tests/sys/pmc/Makefile
@@ -10,10 +10,13 @@
ATF_TESTS_C+= pmc_lifecycle_test
ATF_TESTS_C+= pmc_log_test
ATF_TESTS_C+= pmc_multipart_test
+ATF_TESTS_C+= pmc_lbr_test
ATF_TESTS_C+= pmc_wrap_test
TEST_METADATA.pmc_exterr_test+= required_user="root"
+CFLAGS.pmc_lbr_test+= -I${SRCTOP}/sys/dev/hwpmc
+
LIBADD+= pmc
LIBADD.pmc_detach_test+= pthread
diff --git a/tests/sys/pmc/pmc_lbr_test.c b/tests/sys/pmc/pmc_lbr_test.c
new file mode 100644
--- /dev/null
+++ b/tests/sys/pmc/pmc_lbr_test.c
@@ -0,0 +1,235 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <sys/types.h>
+#include <sys/errno.h>
+
+#include <atf-c.h>
+#include <hwpmc_amd_lbr.h>
+#include <stdint.h>
+
+/*
+ * Exercise the production allocation policy without PMC syscalls or hardware.
+ * Calls are sequential, as they are under pmc_sx in the kernel. The filter
+ * values are opaque here; capability normalization belongs to the MD caller.
+ */
+#define NCPU 3
+#define FILTER_A UINT64_C(1)
+#define FILTER_B UINT64_C(2)
+
+struct reservations {
+ struct amd_lbr_filter ts;
+ struct amd_lbr_filter ss[NCPU];
+};
+
+static void
+check_reservations(const struct reservations *actual,
+ const struct reservations *expected)
+{
+ u_int cpu;
+
+ ATF_CHECK_EQ(expected->ts.select, actual->ts.select);
+ ATF_CHECK_EQ(expected->ts.refs, actual->ts.refs);
+ for (cpu = 0; cpu < NCPU; cpu++) {
+ ATF_CHECK_EQ_MSG(expected->ss[cpu].select, actual->ss[cpu].select,
+ "SS filter changed on CPU %u", cpu);
+ ATF_CHECK_EQ_MSG(expected->ss[cpu].refs, actual->ss[cpu].refs,
+ "SS reference count changed on CPU %u", cpu);
+ }
+}
+
+static void
+reject_filter(struct reservations *r, int is_thread, u_int cpu,
+ uint64_t select)
+{
+ struct reservations before;
+
+ before = *r;
+ ATF_REQUIRE_EQ(EBUSY, amd_lbr_filter_reserve(&r->ts, r->ss, NCPU,
+ is_thread, cpu, select));
+ check_reservations(r, &before);
+}
+
+ATF_TC(lbr_ts_references);
+ATF_TC_HEAD(lbr_ts_references, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "TS reservations share one filter until the last owner releases it");
+}
+ATF_TC_BODY(lbr_ts_references, tc)
+{
+ struct reservations r = {0}, empty = {0};
+ u_int owner;
+
+ for (owner = 0; owner < 3; owner++) {
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, owner, FILTER_A));
+ ATF_CHECK_EQ(owner + 1, r.ts.refs);
+ ATF_CHECK_EQ(FILTER_A, r.ts.select);
+ }
+ reject_filter(&r, 1, 0, FILTER_B);
+
+ /*
+ * There is no start/stop operation in this policy. Allocated, stopped
+ * PMCs retain their references; only release may make a filter reusable.
+ */
+ for (owner = 2; owner > 0; owner--) {
+ amd_lbr_filter_release(&r.ts);
+ ATF_CHECK_EQ(owner, r.ts.refs);
+ ATF_CHECK_EQ(FILTER_A, r.ts.select);
+ reject_filter(&r, 1, 0, FILTER_B);
+ }
+ amd_lbr_filter_release(&r.ts);
+ check_reservations(&r, &empty);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, FILTER_B));
+ ATF_CHECK_EQ(FILTER_B, r.ts.select);
+ ATF_CHECK_EQ(1, r.ts.refs);
+ amd_lbr_filter_release(&r.ts);
+ check_reservations(&r, &empty);
+}
+
+ATF_TC(lbr_ts_before_ss);
+ATF_TC_HEAD(lbr_ts_before_ss, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "a TS reservation constrains SS on every CPU, including idle CPUs");
+}
+ATF_TC_BODY(lbr_ts_before_ss, tc)
+{
+ struct reservations r = {0}, empty = {0};
+ u_int cpu;
+
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, FILTER_A));
+ for (cpu = 0; cpu < NCPU; cpu++) {
+ reject_filter(&r, 0, cpu, FILTER_B);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, cpu, FILTER_A));
+ ATF_CHECK_EQ(1, r.ss[cpu].refs);
+ ATF_CHECK_EQ(FILTER_A, r.ss[cpu].select);
+ }
+ ATF_CHECK_EQ(1, r.ts.refs);
+ ATF_CHECK_EQ(FILTER_A, r.ts.select);
+ amd_lbr_filter_release(&r.ts);
+ /* Releasing TS must not release any SS owner's reservation. */
+ reject_filter(&r, 1, 0, FILTER_B);
+ for (cpu = 0; cpu < NCPU; cpu++)
+ amd_lbr_filter_release(&r.ss[cpu]);
+ check_reservations(&r, &empty);
+}
+
+ATF_TC(lbr_ss_before_ts);
+ATF_TC_HEAD(lbr_ss_before_ts, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "TS allocation checks even the last CPU's SS reservation");
+}
+ATF_TC_BODY(lbr_ss_before_ts, tc)
+{
+ struct reservations r = {0}, empty = {0};
+
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, NCPU - 1, FILTER_B));
+ reject_filter(&r, 1, 0, FILTER_A);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, FILTER_B));
+ ATF_CHECK_EQ(FILTER_B, r.ts.select);
+ ATF_CHECK_EQ(1, r.ts.refs);
+ ATF_CHECK_EQ(FILTER_B, r.ss[NCPU - 1].select);
+ ATF_CHECK_EQ(1, r.ss[NCPU - 1].refs);
+ amd_lbr_filter_release(&r.ss[NCPU - 1]);
+ reject_filter(&r, 0, NCPU - 1, FILTER_A);
+ amd_lbr_filter_release(&r.ts);
+ check_reservations(&r, &empty);
+}
+
+ATF_TC(lbr_disjoint_ss);
+ATF_TC_HEAD(lbr_disjoint_ss, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "SS filters may differ across CPUs but share references on one CPU");
+}
+ATF_TC_BODY(lbr_disjoint_ss, tc)
+{
+ struct reservations r = {0}, empty = {0};
+
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 0, FILTER_A));
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, NCPU - 1, FILTER_B));
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 0, FILTER_A));
+ ATF_CHECK_EQ(2, r.ss[0].refs);
+ ATF_CHECK_EQ(FILTER_A, r.ss[0].select);
+ ATF_CHECK_EQ(1, r.ss[NCPU - 1].refs);
+ ATF_CHECK_EQ(FILTER_B, r.ss[NCPU - 1].select);
+ reject_filter(&r, 0, 0, FILTER_B);
+ reject_filter(&r, 1, 0, FILTER_A);
+ reject_filter(&r, 1, 0, FILTER_B);
+ amd_lbr_filter_release(&r.ss[0]);
+ ATF_CHECK_EQ(1, r.ss[0].refs);
+ ATF_CHECK_EQ(FILTER_A, r.ss[0].select);
+ reject_filter(&r, 0, 0, FILTER_B);
+ amd_lbr_filter_release(&r.ss[0]);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 0, FILTER_B));
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, FILTER_B));
+ amd_lbr_filter_release(&r.ts);
+ amd_lbr_filter_release(&r.ss[0]);
+ amd_lbr_filter_release(&r.ss[NCPU - 1]);
+ check_reservations(&r, &empty);
+}
+
+ATF_TC(lbr_allocation_rollback);
+ATF_TC_HEAD(lbr_allocation_rollback, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "releasing a successful reservation restores existing owners "
+ "after a later allocation failure");
+}
+ATF_TC_BODY(lbr_allocation_rollback, tc)
+{
+ struct reservations r = {0}, before, empty = {0};
+
+ /* Zero is a valid filter, not an indication that the slot is free. */
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, 0));
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 1, 0));
+ reject_filter(&r, 1, 0, FILTER_A);
+ reject_filter(&r, 0, 1, FILTER_A);
+ before = r;
+
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 1, 0, 0));
+ amd_lbr_filter_release(&r.ts);
+ check_reservations(&r, &before);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 1, 0));
+ amd_lbr_filter_release(&r.ss[1]);
+ check_reservations(&r, &before);
+ ATF_REQUIRE_EQ(0, amd_lbr_filter_reserve(&r.ts, r.ss, NCPU,
+ 0, 2, 0));
+ amd_lbr_filter_release(&r.ss[2]);
+ check_reservations(&r, &before);
+
+ amd_lbr_filter_release(&r.ts);
+ amd_lbr_filter_release(&r.ss[1]);
+ check_reservations(&r, &empty);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+ ATF_TP_ADD_TC(tp, lbr_ts_references);
+ ATF_TP_ADD_TC(tp, lbr_ts_before_ss);
+ ATF_TP_ADD_TC(tp, lbr_ss_before_ts);
+ ATF_TP_ADD_TC(tp, lbr_disjoint_ss);
+ ATF_TP_ADD_TC(tp, lbr_allocation_rollback);
+
+ return (atf_no_error());
+}

File Metadata

Mime Type
text/plain
Expires
Tue, Oct 20, 2:34 PM (18 h, 39 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
40774621
Default Alt Text
D60569.id189848.diff (26 KB)

Event Timeline