Page MenuHomeFreeBSD
Files F176845814

D60570.1793487167.diff
No OneTemporary
Actions

Authored By
Unknown
Size
15 KB
Referenced Files
None
Subscribers
None

D60570.1793487167.diff
View Options

diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -33,12 +33,14 @@
/* Support for the AMD K8 and later processors */
#include <sys/param.h>
+#include <sys/counter.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
+#include <sys/pmclog.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
@@ -56,6 +58,10 @@
#define OVERFLOW_WAIT_COUNT 50
+/* The multipart carrier must hold a full LBR ring snapshot. */
+CTASSERT(2 * AMD_LBR_MAX_DEPTH <=
+ nitems(((struct pmc_multipart *)0)->pl_mpdata));
+
DPCPU_DEFINE_STATIC(uint32_t, nmi_counter);
/* AMD K8 PMCs */
@@ -72,6 +78,9 @@
static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs, amd_umc_npmcs;
static int amd_lbr_depth; /* LBR v2 stack depth; 0 = none */
static bool amd_lbr_freeze; /* LbrAndPmcFreeze supported */
+/* Implemented linear-address width, for LBR IP canonicalization. */
+static u_int amd_lbr_va_width = AMD_LBR_VA_BITS_DEFAULT;
+static counter_u64_t amd_lbr_empty; /* LBR samples without records */
struct amd_event_code_map {
enum pmc_event pe_ev; /* enum value */
@@ -486,6 +495,89 @@
amd_lbr_update_end(pac, flags);
}
+/*
+ * The per-CPU LBR state of the CPU running a context-switch or exec hook.
+ * The MI layer calls these hooks in a critical section for the current CPU.
+ * struct pmc_cpu does not record its CPU number (pc_state is never set), so
+ * use curcpu instead of deriving the CPU from 'pc'.
+ */
+static struct amd_cpu *
+amd_lbr_curcpu(struct pmc_cpu *pc __diagused)
+{
+
+ CRITICAL_ASSERT(curthread);
+ KASSERT(pc == NULL || pc == pmc_pcpu[curcpu],
+ ("[amd,%d] LBR hook for a foreign CPU", __LINE__));
+ return (amd_pcpu[curcpu]);
+}
+
+static void
+amd_lbr_csw(struct pmc_cpu *pc, bool switch_in)
+{
+ struct amd_cpu *pac;
+ register_t flags;
+
+ pac = amd_lbr_curcpu(pc);
+ if (pac == NULL)
+ return;
+ /*
+ * An idle CPU has recording off; only track the transition. The NMI
+ * handler ignores these fields while the row mask is empty.
+ */
+ if (pac->pc_lbr_mask == 0) {
+ pac->pc_lbr_suspended = !switch_in;
+ return;
+ }
+
+ flags = amd_lbr_update_begin(pac);
+ pac->pc_lbr_suspended = !switch_in;
+ /*
+ * The address space may have changed since the ring was last reset,
+ * even without a paired switch-out (a thread exit throws directly to
+ * the next thread). Always reset on switch-in.
+ */
+ if (switch_in)
+ pac->pc_lbr_dirty = true;
+ amd_lbr_sync(pac);
+ amd_lbr_update_end(pac, flags);
+}
+
+/*
+ * Called from the process_exec event handler, before the old address space
+ * is torn down. Only kernel branches can be recorded between this reset and
+ * the return to the new image.
+ */
+static void
+amd_lbr_exec(struct pmc_cpu *pc)
+{
+ struct amd_cpu *pac;
+ register_t flags;
+
+ pac = amd_lbr_curcpu(pc);
+ if (pac == NULL || pac->pc_lbr_mask == 0)
+ return;
+
+ /* Clear the ring; recording state and suspension are unchanged. */
+ flags = amd_lbr_update_begin(pac);
+ pac->pc_lbr_dirty = true;
+ amd_lbr_sync(pac);
+ amd_lbr_update_end(pac, flags);
+}
+
+/* This file is also built for Intel CPUs, where the counter never exists. */
+static int
+sysctl_amd_lbr_empty(SYSCTL_HANDLER_ARGS)
+{
+ uint64_t total;
+
+ total = amd_lbr_empty != NULL ? counter_u64_fetch(amd_lbr_empty) : 0;
+ return (sysctl_handle_64(oidp, &total, 0, req));
+}
+SYSCTL_PROC(_kern_hwpmc, OID_AUTO, amd_lbr_empty_samples,
+ CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_amd_lbr_empty,
+ "QU", "AMD LBR samples logged without branch records (ring in "
+ "transition, suspended, or not frozen for this overflow)");
+
static __inline u_int
amd_v2_counter_mask(int ri)
{
@@ -1273,6 +1365,41 @@
return (retval);
}
+/*
+ * Copy the valid records of a stopped or frozen ring, most recent first.
+ * The IP fields are canonicalized from the implemented linear-address
+ * width, so consumers can sign-extend from bit 57. With a user-only filter
+ * the hardware suppresses branches that end in CPL 0, but returns to user
+ * mode still carry a kernel From address; drop any record that names a
+ * kernel address.
+ */
+static void
+amd_lbr_read(struct amd_cpu *pac, struct pmc_multipart *mpd)
+{
+ uint64_t from, to;
+ int i, n;
+ bool usronly;
+
+ usronly = (pac->pc_lbr_select & AMD_LBR_SELECT_CPL0) != 0;
+ for (i = 0, n = 0; i < amd_lbr_depth; i++) {
+ to = rdmsr(AMD_MSR_SAMP_BR_FROM + 2 * i + 1);
+ /* valid == spec == 0: empty; reserved: erratum 1452. */
+ if ((to & (AMD_LBR_TO_VALID | AMD_LBR_TO_SPEC)) == 0 ||
+ (to & AMD_LBR_TO_RESERVED) != 0)
+ continue;
+ from = rdmsr(AMD_MSR_SAMP_BR_FROM + 2 * i);
+ if (usronly &&
+ (amd_lbr_ip_is_kernel(from, amd_lbr_va_width) ||
+ amd_lbr_ip_is_kernel(to, amd_lbr_va_width)))
+ continue;
+ mpd->pl_mpdata[n++] = amd_lbr_canonicalize(from,
+ amd_lbr_va_width);
+ mpd->pl_mpdata[n++] = amd_lbr_canonicalize(to,
+ amd_lbr_va_width);
+ }
+ mpd->pl_length = n;
+}
+
/*
* This is the v2 interrupt handler: freeze counters; read GLOBAL_STATUS once;
* reload; thaw.
@@ -1282,10 +1409,12 @@
{
struct amd_cpu *pac;
struct pmc *pm;
+ struct pmc_multipart mpd;
pmc_value_t v;
uint64_t status, pending;
uint32_t active = 0, count = 0;
- int i, error, retval, cpu;
+ int i, error, len, retval, cpu;
+ bool lbr_frozen, lbr_sample;
cpu = curcpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
@@ -1304,8 +1433,32 @@
/* Read the overflow bitmap once. */
status = rdmsr(AMD_PMC_GLOBAL_STATUS);
+ lbr_frozen = (status & AMD_PMC_GLOBAL_STATUS_LBRS_FROZEN) != 0;
status &= amd_global_cntr_mask;
+ /*
+ * Never expose an old or partially reset ring during a transition.
+ * pc_lbr_hw_on implies a nonempty row mask and a CPU that is not
+ * between a switch-out and a switch-in. A reset may have cleared
+ * LBRS_FROZEN while leaving an overflow pending: its delayed NMI
+ * must log an empty payload, not read the now-running ring.
+ */
+ lbr_sample = amd_lbr_depth > 0 &&
+ atomic_load_int(&pac->pc_lbr_busy) == 0 && pac->pc_lbr_hw_on &&
+ (!amd_lbr_freeze || lbr_frozen);
+ mpd.pl_type = PMC_CC_MULTIPART_LBR;
+ mpd.pl_length = 0;
+ if (lbr_sample && (status & pac->pc_lbr_mask) != 0) {
+ /* Without LbrAndPmcFreeze, stop recording before reading. */
+ if (!amd_lbr_freeze)
+ wrmsr(AMD_MSR_DBG_EXTN_CFG, pac->pc_lbr_extn_cfg);
+ amd_lbr_read(pac, &mpd);
+ if (!amd_lbr_freeze)
+ wrmsr(AMD_MSR_DBG_EXTN_CFG,
+ pac->pc_lbr_extn_cfg | AMD_DBG_EXTN_CFG_LBRV2EN);
+ } else
+ lbr_sample = false;
+
/*
* Count all active sampling PMCs, not just the ones that
* overflowed. Count the in-flight NMI too, its counter has
@@ -1338,16 +1491,40 @@
AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
/*
- * On a log failure, leave the PMC disabled. MI code restarts
- * it via pcd_start_pmc.
- */
- error = pmc_process_interrupt(PMC_HR, pm, tf);
+ * An LBR PMC always logs a multipart sample so that sample
+ * counts do not depend on the ring state; the payload is
+ * empty when the ring could not be read for this overflow.
+ */
+ if ((pm->pm_caps & PMC_CAP_LBR) != 0) {
+ if (!lbr_sample ||
+ (pac->pc_lbr_mask & (1U << i)) == 0) {
+ /* Share the NMI-stack buffer; restore after. */
+ len = mpd.pl_length;
+ mpd.pl_length = 0;
+ counter_u64_add(amd_lbr_empty, 1);
+ error = pmc_process_interrupt_mp(PMC_HR, pm, tf,
+ &mpd);
+ mpd.pl_length = len;
+ } else
+ error = pmc_process_interrupt_mp(PMC_HR, pm, tf,
+ &mpd);
+ } else
+ error = pmc_process_interrupt(PMC_HR, pm, tf);
+
+ /*
+ * On a log failure, leave the PMC disabled. MI code restarts
+ * it via pcd_start_pmc. Its GLOBAL_CTL and LBR row bits stay
+ * set: a disabled EVSEL does not count, and the ring state is
+ * only changed outside of NMI context.
+ */
if (error != 0)
wrmsr(amd_pmcdesc[i].pm_evsel,
pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE);
}
- /* Acknowledge the overflow bits through GLOBAL_STATUS_CLR. */
+ /* Acknowledge overflow bits; clearing LBRS_FROZEN re-arms the ring. */
+ if (amd_lbr_depth > 0)
+ status |= AMD_PMC_GLOBAL_STATUS_LBRS_FROZEN;
wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, status);
/* Thaw the core counters. */
@@ -1827,6 +2004,13 @@
}
#endif
}
+ if (amd_lbr_depth > 0) {
+ u_int vregs[4];
+
+ /* CPUID 0x80000008 EAX[15:8]: linear-address width. */
+ do_cpuid(0x80000008, vregs);
+ amd_lbr_va_width = amd_lbr_va_bits((vregs[0] >> 8) & 0xff);
+ }
/*
* Normalize per-class counts against feature bits so that allocation,
@@ -1938,6 +2122,12 @@
return (NULL);
}
+ /* The hardware check may have withdrawn LBR support. */
+ if (amd_lbr_depth > 0) {
+ for (i = 0; i < amd_core_npmcs; i++)
+ amd_pmcdesc[i].pm_descr.pd_caps |= PMC_CAP_LBR;
+ }
+
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
@@ -1947,6 +2137,7 @@
if (amd_lbr_depth > 0) {
amd_lbr_ss = malloc(sizeof(*amd_lbr_ss) * pmc_cpu_max(), M_PMC,
M_WAITOK | M_ZERO);
+ amd_lbr_empty = counter_u64_alloc(M_WAITOK);
}
/*
@@ -1984,6 +2175,8 @@
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_K8];
pcd->pcd_caps = AMD_PMC_CAPS;
+ if (amd_lbr_depth > 0)
+ pcd->pcd_caps |= PMC_CAP_LBR;
pcd->pcd_class = PMC_CLASS_K8;
pcd->pcd_num = amd_npmcs;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
@@ -2016,6 +2209,10 @@
pcd->pcd_stop_all = amd_stop_pmc_all_v2;
pmc_mdep->pmd_intr = amd_intr_v2;
}
+ if (amd_lbr_depth > 0) {
+ pmc_mdep->pmd_lbr_csw = amd_lbr_csw;
+ pmc_mdep->pmd_lbr_exec = amd_lbr_exec;
+ }
pmc_mdep->pmd_npmc += amd_npmcs;
@@ -2040,6 +2237,10 @@
return (pmc_mdep);
error:
+ if (amd_lbr_empty != NULL) {
+ counter_u64_free(amd_lbr_empty);
+ amd_lbr_empty = NULL;
+ }
free(amd_lbr_ss, M_PMC);
amd_lbr_ss = NULL;
free(amd_pcpu, M_PMC);
@@ -2072,6 +2273,10 @@
KASSERT(amd_pcpu[i] == NULL,
("[amd,%d] non-null pcpu cpu %d", __LINE__, i));
+ if (amd_lbr_empty != NULL) {
+ counter_u64_free(amd_lbr_empty);
+ amd_lbr_empty = NULL;
+ }
free(amd_lbr_ss, M_PMC);
amd_lbr_ss = NULL;
free(amd_pcpu, M_PMC);
diff --git a/sys/dev/hwpmc/hwpmc_amd_lbr.h b/sys/dev/hwpmc/hwpmc_amd_lbr.h
--- a/sys/dev/hwpmc/hwpmc_amd_lbr.h
+++ b/sys/dev/hwpmc/hwpmc_amd_lbr.h
@@ -51,4 +51,51 @@
slot->select = 0;
}
+/*
+ * Branch record IP handling. The hardware IP field is bits [57:0], but only
+ * the bits covered by the implemented linear-address width (CPUID
+ * 0x80000008 EAX[15:8]) are architecturally meaningful. Do not assume the
+ * bits above that width are a copy of the sign bit: sign-extend from the
+ * implemented width, as the Linux driver does.
+ */
+#define AMD_LBR_IP_BITS 58
+#define AMD_LBR_IP_MASK ((1ULL << AMD_LBR_IP_BITS) - 1)
+#define AMD_LBR_VA_BITS_MIN 48
+#define AMD_LBR_VA_BITS_DEFAULT 48
+
+/* Clamp a CPUID linear-address width to the range the IP field can hold. */
+static __inline u_int
+amd_lbr_va_bits(u_int cpuid_bits)
+{
+ if (cpuid_bits < AMD_LBR_VA_BITS_MIN)
+ return (AMD_LBR_VA_BITS_DEFAULT);
+ if (cpuid_bits > AMD_LBR_IP_BITS)
+ return (AMD_LBR_IP_BITS);
+ return (cpuid_bits);
+}
+
+/* True if a raw From/To word holds a supervisor-half (kernel) address. */
+static __inline int
+amd_lbr_ip_is_kernel(uint64_t raw, u_int va_bits)
+{
+ return (((raw & AMD_LBR_IP_MASK) >> (va_bits - 1)) != 0);
+}
+
+/*
+ * Rewrite the IP field of a raw From/To word so that sign-extending it from
+ * bit 57 yields the canonical address. The flag bits [63:58] are preserved,
+ * so consumers can decode the record without knowing the recording CPU's
+ * linear-address width.
+ */
+static __inline uint64_t
+amd_lbr_canonicalize(uint64_t raw, u_int va_bits)
+{
+ uint64_t ip;
+ u_int shift;
+
+ shift = 64 - va_bits;
+ ip = (uint64_t)((int64_t)(raw << shift) >> shift);
+ return ((raw & ~AMD_LBR_IP_MASK) | (ip & AMD_LBR_IP_MASK));
+}
+
#endif /* _DEV_HWPMC_AMD_LBR_H_ */
diff --git a/tests/sys/pmc/pmc_lbr_test.c b/tests/sys/pmc/pmc_lbr_test.c
--- a/tests/sys/pmc/pmc_lbr_test.c
+++ b/tests/sys/pmc/pmc_lbr_test.c
@@ -223,6 +223,111 @@
check_reservations(&r, &empty);
}
+/*
+ * Branch record IP canonicalization. The raw words below use the From/To
+ * flag layout: From[63] mispredict, To[63] valid, To[62] speculative,
+ * To[61] reserved.
+ */
+#define FLAGS_FROM (UINT64_C(1) << 63)
+#define FLAGS_TO (UINT64_C(3) << 62)
+#define IP_MASK ((UINT64_C(1) << 58) - 1)
+
+/* The consumer decode documented for PMC_CC_MULTIPART_LBR. */
+static uint64_t
+decode_ip(uint64_t word)
+{
+ return ((uint64_t)((int64_t)(word << 6) >> 6));
+}
+
+ATF_TC(lbr_va_bits_clamp);
+ATF_TC_HEAD(lbr_va_bits_clamp, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "CPUID linear-address widths are clamped to the IP field");
+}
+ATF_TC_BODY(lbr_va_bits_clamp, tc)
+{
+ ATF_CHECK_EQ(48, amd_lbr_va_bits(0));
+ ATF_CHECK_EQ(48, amd_lbr_va_bits(32));
+ ATF_CHECK_EQ(48, amd_lbr_va_bits(48));
+ ATF_CHECK_EQ(57, amd_lbr_va_bits(57));
+ ATF_CHECK_EQ(58, amd_lbr_va_bits(58));
+ ATF_CHECK_EQ(58, amd_lbr_va_bits(64));
+}
+
+ATF_TC(lbr_canonicalize_48);
+ATF_TC_HEAD(lbr_canonicalize_48, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "on a 48-bit CPU, kernel IPs with zero upper IP bits decode to "
+ "canonical kernel addresses and are classified as kernel");
+}
+ATF_TC_BODY(lbr_canonicalize_48, tc)
+{
+ uint64_t kraw, uraw, k, u;
+
+ /* Hardware may leave IP bits [57:48] zero for a kernel address. */
+ kraw = FLAGS_FROM | UINT64_C(0x0000ffff80001234);
+ uraw = FLAGS_TO | UINT64_C(0x00007fffdeadbeef);
+
+ /* Without canonicalization the kernel address would look like user. */
+ ATF_CHECK((int64_t)decode_ip(kraw) > 0);
+ ATF_CHECK(amd_lbr_ip_is_kernel(kraw, 48));
+ ATF_CHECK(!amd_lbr_ip_is_kernel(uraw, 48));
+
+ k = amd_lbr_canonicalize(kraw, 48);
+ u = amd_lbr_canonicalize(uraw, 48);
+ ATF_CHECK_EQ(UINT64_C(0xffffffff80001234), decode_ip(k));
+ ATF_CHECK_EQ(UINT64_C(0x00007fffdeadbeef), decode_ip(u));
+ /* Flag bits survive. */
+ ATF_CHECK_EQ(FLAGS_FROM, k & ~IP_MASK);
+ ATF_CHECK_EQ(FLAGS_TO, u & ~IP_MASK);
+ /* Canonicalization is idempotent and keeps the classification. */
+ ATF_CHECK_EQ(k, amd_lbr_canonicalize(k, 48));
+ ATF_CHECK(amd_lbr_ip_is_kernel(k, 48));
+ ATF_CHECK(!amd_lbr_ip_is_kernel(u, 48));
+}
+
+ATF_TC(lbr_canonicalize_57);
+ATF_TC_HEAD(lbr_canonicalize_57, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "on an LA57 CPU, the full 57-bit user range stays user and "
+ "kernel addresses sign-extend from bit 56");
+}
+ATF_TC_BODY(lbr_canonicalize_57, tc)
+{
+ uint64_t kraw, uraw;
+
+ uraw = UINT64_C(0x00ff000000001000); /* 5-level user address */
+ kraw = UINT64_C(0x0100000000002000); /* bit 56 set: kernel */
+ ATF_CHECK(!amd_lbr_ip_is_kernel(uraw, 57));
+ ATF_CHECK(amd_lbr_ip_is_kernel(kraw, 57));
+ /* A 48-bit interpretation would call this LA57 user address kernel. */
+ ATF_CHECK(amd_lbr_ip_is_kernel(uraw, 48));
+ ATF_CHECK_EQ(UINT64_C(0x00ff000000001000),
+ decode_ip(amd_lbr_canonicalize(uraw, 57)));
+ ATF_CHECK_EQ(UINT64_C(0xff00000000002000),
+ decode_ip(amd_lbr_canonicalize(kraw, 57)));
+}
+
+ATF_TC(lbr_canonicalize_58);
+ATF_TC_HEAD(lbr_canonicalize_58, tc)
+{
+ atf_tc_set_md_var(tc, "descr",
+ "at the maximum width canonicalization is the identity");
+}
+ATF_TC_BODY(lbr_canonicalize_58, tc)
+{
+ uint64_t raw;
+
+ raw = FLAGS_TO | UINT64_C(0x03ffffff80001234);
+ ATF_CHECK_EQ(raw, amd_lbr_canonicalize(raw, 58));
+ ATF_CHECK(amd_lbr_ip_is_kernel(raw, 58));
+ ATF_CHECK_EQ(UINT64_C(0xffffffff80001234),
+ decode_ip(amd_lbr_canonicalize(raw, 58)));
+}
+
ATF_TP_ADD_TCS(tp)
{
ATF_TP_ADD_TC(tp, lbr_ts_references);
@@ -230,6 +335,10 @@
ATF_TP_ADD_TC(tp, lbr_ss_before_ts);
ATF_TP_ADD_TC(tp, lbr_disjoint_ss);
ATF_TP_ADD_TC(tp, lbr_allocation_rollback);
+ ATF_TP_ADD_TC(tp, lbr_va_bits_clamp);
+ ATF_TP_ADD_TC(tp, lbr_canonicalize_48);
+ ATF_TP_ADD_TC(tp, lbr_canonicalize_57);
+ ATF_TP_ADD_TC(tp, lbr_canonicalize_58);
return (atf_no_error());
}

File Metadata

Mime Type
text/plain
Expires
Sat, Oct 31, 10:52 PM (23 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41270735
Default Alt Text
D60570.1793487167.diff (15 KB)

Event Timeline