Page MenuHomeFreeBSD
Files F163306911

D56914.id177522.diff
No OneTemporary
Actions

D56914.id177522.diff
View Options

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -39,6 +39,7 @@
#include <ctype.h>
#include <errno.h>
#include <err.h>
+#include <stdbool.h>
#include <fcntl.h>
#include <pmc.h>
#include <stdio.h>
@@ -700,13 +701,14 @@
struct pmc_op_pmcallocate *pmc_config)
{
char *e, *p, *q;
- uint64_t ctl, ldlat;
- u_int ibs_features;
- u_int regs[4];
+ uint64_t ctl, ctl2, ldlat, addr63, fetchlat;
+ bool seen_l3miss, seen_randomize, seen_ldlat, seen_fetchlat;
+ bool seen_opcount, seen_addr63, seen_streamstore;
pmc_config->pm_caps |=
(PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_PRECISE);
pmc_config->pm_md.pm_ibs.ibs_ctl = 0;
+ pmc_config->pm_md.pm_ibs.ibs_ctl2 = 0;
/* setup parsing tables */
switch (pe) {
@@ -725,25 +727,79 @@
return (-1);
}
- /* Read the ibs feature flags */
- ibs_features = 0;
- do_cpuid(0x80000000, regs);
- if (regs[0] >= CPUID_IBSID) {
- do_cpuid(CPUID_IBSID, regs);
- ibs_features = regs[0];
- }
+ /*
+ * Capability gating is kernel-side; we only check syntax
+ * and value ranges here.
+ */
- /* parse parameters */
ctl = 0;
+ ctl2 = 0;
+ seen_l3miss = false;
+ seen_randomize = false;
+ seen_ldlat = false;
+ seen_fetchlat = false;
+ seen_opcount = false;
+ seen_addr63 = false;
+ seen_streamstore = false;
if (pe == PMC_EV_IBS_FETCH) {
while ((p = strsep(&ctrspec, ",")) != NULL) {
if (KWMATCH(p, "l3miss")) {
- if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) == 0)
+ if (seen_l3miss)
return (-1);
+ seen_l3miss = true;
ctl |= IBS_FETCH_CTL_L3MISSONLY;
} else if (KWMATCH(p, "randomize")) {
+ if (seen_randomize)
+ return (-1);
+ seen_randomize = true;
ctl |= IBS_FETCH_CTL_RANDOMIZE;
+ } else if (KWPREFIXMATCH(p, "fetchlat=")) {
+ if (seen_fetchlat)
+ return (-1);
+ seen_fetchlat = true;
+
+ q = strchr(p, '=');
+ if (*++q == '\0') /* skip '=' */
+ return (-1);
+
+ fetchlat = strtoull(q, &e, 0);
+ if (e == q || *e != '\0')
+ return (-1);
+
+ if (fetchlat < IBS_FETCH_CTL2_LAT_MIN ||
+ fetchlat > IBS_FETCH_CTL2_LAT_MAX)
+ return (-1);
+ if ((fetchlat % IBS_FETCH_CTL2_LAT_STEP) != 0)
+ return (-1);
+
+ ctl2 |= IBS_FETCH_CTL2_LAT_TO_CTL(fetchlat);
+ /*
+ * Defence-in-depth round-trip check: catch any
+ * future macro/range drift that would silently
+ * truncate fetchlat to the "no filter" encoding.
+ */
+ if ((ctl2 & IBS_FETCH_CTL2_LATFILTERMASK) == 0)
+ return (-1);
+ } else if (KWPREFIXMATCH(p, "addr63=")) {
+ if (seen_addr63)
+ return (-1);
+ seen_addr63 = true;
+
+ q = strchr(p, '=');
+ if (*++q == '\0') /* skip '=' */
+ return (-1);
+
+ addr63 = strtoull(q, &e, 0);
+ if (e == q || *e != '\0')
+ return (-1);
+
+ if (addr63 == 0)
+ ctl2 |= IBS_FETCH_CTL2_EXCLADDR63EQ1;
+ else if (addr63 == 1)
+ ctl2 |= IBS_FETCH_CTL2_EXCLADDR63EQ0;
+ else
+ return (-1);
} else {
return (-1);
}
@@ -757,10 +813,14 @@
} else {
while ((p = strsep(&ctrspec, ",")) != NULL) {
if (KWMATCH(p, "l3miss")) {
+ if (seen_l3miss)
+ return (-1);
+ seen_l3miss = true;
ctl |= IBS_OP_CTL_L3MISSONLY;
} else if (KWPREFIXMATCH(p, "ldlat=")) {
- if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0)
+ if (seen_ldlat)
return (-1);
+ seen_ldlat = true;
q = strchr(p, '=');
if (*++q == '\0') /* skip '=' */
@@ -786,10 +846,35 @@
ctl |= IBS_OP_CTL_LDLAT_TO_CTL(ldlat);
ctl |= IBS_OP_CTL_L3MISSONLY | IBS_OP_CTL_LATFLTEN;
} else if (KWMATCH(p, "opcount")) {
- if ((ibs_features & CPUID_IBSID_OPCNT) == 0)
+ if (seen_opcount)
return (-1);
+ seen_opcount = true;
ctl |= IBS_OP_CTL_COUNTERCONTROL;
+ } else if (KWPREFIXMATCH(p, "addr63=")) {
+ if (seen_addr63)
+ return (-1);
+ seen_addr63 = true;
+
+ q = strchr(p, '=');
+ if (*++q == '\0') /* skip '=' */
+ return (-1);
+
+ addr63 = strtoull(q, &e, 0);
+ if (e == q || *e != '\0')
+ return (-1);
+
+ if (addr63 == 0)
+ ctl2 |= IBS_OP_CTL2_EXCLRIP63EQ1;
+ else if (addr63 == 1)
+ ctl2 |= IBS_OP_CTL2_EXCLRIP63EQ0;
+ else
+ return (-1);
+ } else if (KWMATCH(p, "streamstore")) {
+ if (seen_streamstore)
+ return (-1);
+ seen_streamstore = true;
+ ctl2 |= IBS_OP_CTL2_STRMSTFILTER;
} else {
return (-1);
}
@@ -799,15 +884,12 @@
pmc_config->pm_count > IBS_OP_MAX_RATE)
return (-1);
- if (((ibs_features & CPUID_IBSID_OPCNTEXT) == 0) &&
- (pmc_config->pm_count > IBS_OP_MAX_RATE_PREEXT))
- return (-1);
-
ctl |= IBS_OP_INTERVAL_TO_CTL(pmc_config->pm_count);
}
pmc_config->pm_md.pm_ibs.ibs_ctl |= ctl;
+ pmc_config->pm_md.pm_ibs.ibs_ctl2 |= ctl2;
return (0);
}
diff --git a/lib/libpmc/pmc.ibs.3 b/lib/libpmc/pmc.ibs.3
--- a/lib/libpmc/pmc.ibs.3
+++ b/lib/libpmc/pmc.ibs.3
@@ -98,24 +98,81 @@
Event specifiers for AMD IBS can have the following optional
qualifiers:
.Bl -tag -width "ldlat=value"
+.It Li addr63= Ns Ar bit
+Valid for both
+.Ar ibs-fetch
+and
+.Ar ibs-op
+events.
+Configure the counter to only sample events whose virtual address has bit 63
+equal to
+.Ar bit
+.Pq Li 0 No or Li 1 .
+Requires Zen6 IBS extensions
+.Pq CPUID Fn Fn8000_0001B
+.Va EAX[IbsAddrBit63Filtering] ,
+and is rejected when the CPU does not advertise support.
+.It Li fetchlat= Ns Ar value
+Valid only for
+.Ar ibs-fetch
+events.
+Configure the counter to only sample fetches whose latency is greater than or
+equal to
+.Ar value
+core clock cycles.
+The valid range is 128 to 1920 in steps of 128.
+Requires Zen6 IBS extensions
+.Pq CPUID Fn Fn8000_0001B
+.Va EAX[IbsFetchLatencyFiltering] ,
+and is rejected when the CPU does not advertise support.
.It Li l3miss
+Valid for both
+.Ar ibs-fetch
+and
+.Ar ibs-op
+events.
Configure IBS to only sample if an l3miss occurred.
.It Li ldlat= Ns Ar value
+Valid only for
+.Ar ibs-op
+events.
Configure the counter to only sample events with load latencies above
.Ar ldlat .
IBS only supports filtering latencies that are a multiple of 128 and between
128 and 2048.
-Load latency filtering can only be used with ibs-op events and imply the
-l3miss qualifier.
+On pre-Zen6 hardware this qualifier implies the
+.Li l3miss
+qualifier; on Zen6 and later, latency-only filtering without
+.Li l3miss
+is permitted.
.It Li opcount
+Valid only for
+.Ar ibs-op
+events.
Count ops rather than cycles.
.It Li randomize
+Valid only for
+.Ar ibs-fetch
+events.
Randomize the sampling rate.
+.It Li streamstore
+Valid only for
+.Ar ibs-op
+events.
+Configure the counter to only sample streaming
+.Pq non-temporal
+store operations.
+Requires Zen6 IBS extensions
+.Pq CPUID Fn Fn8000_0001B
+.Va EAX[IbsStrmStAndRmtSocket] ,
+and is rejected when the CPU does not advertise support.
.El
.Ss AMD IBS Events Specifiers
The IBS event class provides only two event specifiers:
.Bl -tag -width indent
.It Li ibs-fetch Xo
+.Op ,addr63= Ns Ar bit
+.Op ,fetchlat= Ns Ar value
.Op ,l3miss
.Op ,randomize
.Xc
@@ -124,9 +181,11 @@
.Ar randomize
qualifier randomly sets the bottom four bits of the sample rate.
.It Li ibs-op Xo
+.Op ,addr63= Ns Ar bit
.Op ,l3miss
.Op ,ldlat= Ns Ar ldlat
.Op ,opcount
+.Op ,streamstore
.Xc
Collect performance samples during instruction execution.
The
diff --git a/sys/dev/hwpmc/hwpmc_ibs.h b/sys/dev/hwpmc/hwpmc_ibs.h
--- a/sys/dev/hwpmc/hwpmc_ibs.h
+++ b/sys/dev/hwpmc/hwpmc_ibs.h
@@ -50,6 +50,12 @@
#define CPUID_IBSID_IBSOPDATA4 0x00000400 /* IBS OP DATA4 */
#define CPUID_IBSID_ZEN4IBSEXTENSIONS 0x00000800 /* IBS Zen 4 Extensions */
#define CPUID_IBSID_IBSLOADLATENCYFILT 0x00001000 /* Load Latency Filtering */
+#define CPUID_IBSID_IBSDIS 0x00002000 /* Alternate IBS Disable */
+#define CPUID_IBSID_FETCHLATFILTERING 0x00004000 /* Fetch Latency Filter */
+#define CPUID_IBSID_ADDRBIT63FILTERING 0x00008000 /* Addr Bit 63 Filter */
+#define CPUID_IBSID_STRMSTANDRMTSOCKET 0x00010000 /* StrmSt + RmtSocket */
+#define CPUID_IBSID_BUFFERV1 0x00020000 /* IBS Buffering V1 */
+#define CPUID_IBSID_MEMPROFILERV1 0x00040000 /* IBS Memory Profiler V1 */
#define CPUID_IBSID_IBSUPDTDDTLBSTATS 0x00080000 /* Simplified DTLB Stats */
/*
@@ -107,11 +113,27 @@
#define IBS_FETCH_PHYSADDR 0xC0011032 /* Fetch Physical Address */
#define IBS_FETCH_EXTCTL 0xC001103C /* Fetch Control Extended */
+/* IBS Fetch Control 2 (Zen6) */
+#define IBS_FETCH_CTL2 0xC001103F /* IBS Fetch Control 2 */
+#define IBS_FETCH_CTL2_DISABLE (1ULL << 0) /* IBS Fetch Disable */
+#define IBS_FETCH_CTL2_LATFILTERMASK (0xFULL << 1) /* Fetch Latency Filter */
+#define IBS_FETCH_CTL2_EXCLADDR63EQ1 (1ULL << 5) /* Exclude addr bit63=1 */
+#define IBS_FETCH_CTL2_EXCLADDR63EQ0 (1ULL << 6) /* Exclude addr bit63=0 */
+#define IBS_FETCH_CTL2_ADDR63MASK (IBS_FETCH_CTL2_EXCLADDR63EQ0 | \
+ IBS_FETCH_CTL2_EXCLADDR63EQ1)
+
+#define IBS_FETCH_CTL2_LAT_MIN 128
+#define IBS_FETCH_CTL2_LAT_MAX 1920
+#define IBS_FETCH_CTL2_LAT_STEP 128
+#define IBS_FETCH_CTL2_LAT_TO_CTL(_l) ((((_l) >> 7) & 0xFULL) << 1)
+#define IBS_FETCH_CTL2_CTL_TO_LAT(_c) ((((_c) >> 1) & 0xFULL) << 7)
+
#define PMC_MPIDX_FETCH_CTL 0
#define PMC_MPIDX_FETCH_EXTCTL 1
#define PMC_MPIDX_FETCH_LINADDR 2
#define PMC_MPIDX_FETCH_PHYSADDR 3
#define PMC_MPIDX_FETCH_MAX (PMC_MPIDX_FETCH_PHYSADDR + 1)
+#define PMC_MPIDX_FETCH_CTL2 4
/* IBS Execution Control */
#define IBS_OP_CTL 0xC0011033 /* IBS Execution Control */
@@ -148,6 +170,8 @@
#define IBS_OP_DATA_RETURN (1ULL << 34) /* Return */
#define IBS_OP_DATA2 0xC0011036 /* IBS Op Data 2 */
+#define IBS_OP_DATA2_RMTSOCKET (1ULL << 9) /* Remote Socket */
+#define IBS_OP_DATA2_STRMST (1ULL << 8) /* Streaming Store */
#define IBS_OP_DATA3 0xC0011037 /* IBS Op Data 3 */
#define IBS_OP_DATA3_DCPHYADDRVALID (1ULL << 18) /* DC Physical Address */
#define IBS_OP_DATA3_DCLINADDRVALID (1ULL << 17) /* DC Linear Address */
@@ -169,6 +193,15 @@
#define IBS_OP_DATA4 0xC001103D /* IBS Op Data 4 */
#define IBS_OP_DATA4_LDRESYNC (1ULL << 0) /* Load Resync */
+/* IBS Execution Control 2 (Zen6) */
+#define IBS_OP_CTL2 0xC001103E /* IBS Execution Control 2 */
+#define IBS_OP_CTL2_DISABLE (1ULL << 0) /* IBS Execution Disable */
+#define IBS_OP_CTL2_EXCLRIP63EQ0 (1ULL << 1) /* Exclude RIP bit63=0 */
+#define IBS_OP_CTL2_EXCLRIP63EQ1 (1ULL << 2) /* Exclude RIP bit63=1 */
+#define IBS_OP_CTL2_STRMSTFILTER (1ULL << 3) /* Streaming Store Filter */
+#define IBS_OP_CTL2_RIP63MASK (IBS_OP_CTL2_EXCLRIP63EQ0 | \
+ IBS_OP_CTL2_EXCLRIP63EQ1)
+
#define PMC_MPIDX_OP_CTL 0
#define PMC_MPIDX_OP_RIP 1
#define PMC_MPIDX_OP_DATA 2
@@ -179,6 +212,7 @@
#define PMC_MPIDX_OP_TGT_RIP 7
#define PMC_MPIDX_OP_DATA4 8
#define PMC_MPIDX_OP_MAX (PMC_MPIDX_OP_DATA4 + 1)
+#define PMC_MPIDX_OP_CTL2 9
/*
* IBS data is encoded as using the multipart flag in the existing callchain
diff --git a/sys/dev/hwpmc/hwpmc_ibs.c b/sys/dev/hwpmc/hwpmc_ibs.c
--- a/sys/dev/hwpmc/hwpmc_ibs.c
+++ b/sys/dev/hwpmc/hwpmc_ibs.c
@@ -60,9 +60,15 @@
static uint64_t ibs_features;
static uint64_t ibs_fetch_allowed_mask;
static uint64_t ibs_op_allowed_mask;
+static uint64_t ibs_fetch_ctl2_allowed_mask;
+static uint64_t ibs_op_ctl2_allowed_mask;
+static bool ibs_fetch_ctl2_supported;
+static bool ibs_op_ctl2_supported;
static uint64_t ibs_fetch_extra_mask;
static uint64_t ibs_op_extra_mask;
+static uint64_t ibs_fetch_ctl2_extra_mask;
+static uint64_t ibs_op_ctl2_extra_mask;
SYSCTL_DECL(_kern_hwpmc);
@@ -74,6 +80,14 @@
&ibs_op_extra_mask, 0,
"Extra allowed bits in the IBS op control MSR (override; default 0)");
+SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_fetch_ctl2_extra_mask, CTLFLAG_RDTUN,
+ &ibs_fetch_ctl2_extra_mask, 0,
+ "Extra allowed bits in the IBS fetch control 2 MSR (override; default 0)");
+
+SYSCTL_U64(_kern_hwpmc, OID_AUTO, ibs_op_ctl2_extra_mask, CTLFLAG_RDTUN,
+ &ibs_op_ctl2_extra_mask, 0,
+ "Extra allowed bits in the IBS op control 2 MSR (override; default 0)");
+
/*
* Per-processor information
*/
@@ -92,8 +106,10 @@
{
ibs_fetch_allowed_mask = IBS_FETCH_ALLOWED_MASK_BASE;
+ ibs_fetch_ctl2_allowed_mask = 0;
ibs_op_allowed_mask = IBS_OP_CTL_MAXCNTBASEMASK;
+ ibs_op_ctl2_allowed_mask = 0;
if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
ibs_fetch_allowed_mask |= IBS_FETCH_CTL_L3MISSONLY;
@@ -106,6 +122,29 @@
if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
ibs_op_allowed_mask |= IBS_OP_CTL_L3MISSONLY;
+
+ if ((ibs_features & CPUID_IBSID_FETCHLATFILTERING) != 0)
+ ibs_fetch_ctl2_allowed_mask |= IBS_FETCH_CTL2_LATFILTERMASK;
+
+ if ((ibs_features & CPUID_IBSID_ADDRBIT63FILTERING) != 0) {
+ ibs_fetch_ctl2_allowed_mask |= IBS_FETCH_CTL2_ADDR63MASK;
+ ibs_op_ctl2_allowed_mask |= IBS_OP_CTL2_RIP63MASK;
+ }
+
+ if ((ibs_features & CPUID_IBSID_STRMSTANDRMTSOCKET) != 0)
+ ibs_op_ctl2_allowed_mask |= IBS_OP_CTL2_STRMSTFILTER;
+
+ if ((ibs_features & CPUID_IBSID_IBSDIS) != 0) {
+ ibs_fetch_ctl2_allowed_mask |= IBS_FETCH_CTL2_DISABLE;
+ ibs_op_ctl2_allowed_mask |= IBS_OP_CTL2_DISABLE;
+ }
+
+ /*
+ * ctl2 MSRs only exist on Zen6; writing them on older silicon
+ * would #GP.
+ */
+ ibs_fetch_ctl2_supported = (ibs_fetch_ctl2_allowed_mask != 0);
+ ibs_op_ctl2_supported = (ibs_op_ctl2_allowed_mask != 0);
}
static int
@@ -128,7 +167,12 @@
if ((config & IBS_OP_CTL_LATFLTEN) != 0) {
if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0)
return (EINVAL);
- if ((config & IBS_OP_CTL_L3MISSONLY) == 0)
+ /*
+ * Zen6 decouples L3MISSONLY from load-latency filtering
+ * (AMD pub 69205); enforce the pairing only on older parts.
+ */
+ if ((ibs_features & CPUID_IBSID_IBSDIS) == 0 &&
+ (config & IBS_OP_CTL_L3MISSONLY) == 0)
return (EINVAL);
allowed_mask |= IBS_OP_CTL_LDLATMASK | IBS_OP_CTL_L3MISSONLY;
@@ -143,16 +187,84 @@
}
static int
-ibs_validate_pmc_config(int ri, uint64_t config)
+ibs_validate_fetch_ctl2_config(uint64_t config)
+{
+ uint64_t allowed_mask;
+
+ if (config == 0)
+ return (0);
+
+ if (!ibs_fetch_ctl2_supported)
+ return (EXTERROR(EINVAL,
+ "IBS fetch ctl2 features are not supported on this CPU"));
+
+ allowed_mask = ibs_fetch_ctl2_allowed_mask | ibs_fetch_ctl2_extra_mask;
+
+ if ((config & ~allowed_mask) != 0)
+ return (EXTERROR(EINVAL,
+ "IBS fetch ctl2 config 0x%jx has bits outside allowed"
+ " mask 0x%jx", (uint64_t)config, (uint64_t)allowed_mask));
+
+ if ((config & IBS_FETCH_CTL2_DISABLE) != 0)
+ return (EXTERROR(EINVAL,
+ "IBS fetch ctl2 DISABLE bit is reserved for the kernel"));
+
+ if ((config & IBS_FETCH_CTL2_ADDR63MASK) == IBS_FETCH_CTL2_ADDR63MASK)
+ return (EXTERROR(EINVAL,
+ "IBS fetch ctl2 cannot exclude both addr63=0 and"
+ " addr63=1"));
+
+ return (0);
+}
+
+static int
+ibs_validate_op_ctl2_config(uint64_t config)
+{
+ uint64_t allowed_mask;
+
+ if (config == 0)
+ return (0);
+
+ if (!ibs_op_ctl2_supported)
+ return (EXTERROR(EINVAL,
+ "IBS op ctl2 features are not supported on this CPU"));
+
+ allowed_mask = ibs_op_ctl2_allowed_mask | ibs_op_ctl2_extra_mask;
+
+ if ((config & ~allowed_mask) != 0)
+ return (EXTERROR(EINVAL,
+ "IBS op ctl2 config 0x%jx has bits outside allowed mask"
+ " 0x%jx", (uint64_t)config, (uint64_t)allowed_mask));
+
+ if ((config & IBS_OP_CTL2_DISABLE) != 0)
+ return (EXTERROR(EINVAL,
+ "IBS op ctl2 DISABLE bit is reserved for the kernel"));
+
+ if ((config & IBS_OP_CTL2_RIP63MASK) == IBS_OP_CTL2_RIP63MASK)
+ return (EXTERROR(EINVAL,
+ "IBS op ctl2 cannot exclude both addr63=0 and addr63=1"));
+
+ return (0);
+}
+
+static int
+ibs_validate_pmc_config(int ri, uint64_t config, uint64_t config2)
{
+ int error;
switch (ri) {
case IBS_PMC_FETCH:
- return (ibs_validate_fetch_config(config));
+ error = ibs_validate_fetch_config(config);
+ if (error != 0)
+ return (error);
+ return (ibs_validate_fetch_ctl2_config(config2));
case IBS_PMC_OP:
- return (ibs_validate_op_config(config));
+ error = ibs_validate_op_config(config);
+ if (error != 0)
+ return (error);
+ return (ibs_validate_op_ctl2_config(config2));
default:
- return (EINVAL);
+ return (EXTERROR(EINVAL, "invalid IBS PMC index %d", ri));
}
}
@@ -266,7 +378,7 @@
ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
- uint64_t caps, config;
+ uint64_t caps, config, config2;
int error;
KASSERT(ri >= 0 && ri < IBS_NPMCS,
@@ -291,13 +403,16 @@
return (EINVAL);
config = a->pm_md.pm_ibs.ibs_ctl;
- error = ibs_validate_pmc_config(ri, config);
+ config2 = a->pm_md.pm_ibs.ibs_ctl2;
+ error = ibs_validate_pmc_config(ri, config, config2);
if (error != 0)
return (error);
pm->pm_md.pm_ibs.ibs_ctl = config;
+ pm->pm_md.pm_ibs.ibs_ctl2 = config2;
- PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%jx", ri,
- config);
+ PMCDBG3(MDP, ALL, 2,
+ "ibs-allocate ri=%d -> config=0x%jx config2=0x%jx", ri,
+ config, config2);
return (0);
}
@@ -326,13 +441,52 @@
return (0);
}
+/*
+ * Strip the alternate disable bit from a stored ctl2 config so it can be
+ * written for a running counter.
+ */
+static uint64_t
+ibs_fetch_ctl2_start_config(struct pmc *pm)
+{
+ return (pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_FETCH_CTL2_DISABLE);
+}
+
+static uint64_t
+ibs_op_ctl2_start_config(struct pmc *pm)
+{
+ return (pm->pm_md.pm_ibs.ibs_ctl2 & ~IBS_OP_CTL2_DISABLE);
+}
+
+static void
+ibs_write_fetch_start(struct pmc *pm)
+{
+ uint64_t config;
+
+ wrmsr(IBS_FETCH_CTL, 0);
+ if (ibs_fetch_ctl2_supported)
+ wrmsr(IBS_FETCH_CTL2, ibs_fetch_ctl2_start_config(pm));
+ config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE;
+ wrmsr(IBS_FETCH_CTL, config);
+}
+
+static void
+ibs_write_op_start(struct pmc *pm)
+{
+ uint64_t config;
+
+ wrmsr(IBS_OP_CTL, 0);
+ if (ibs_op_ctl2_supported)
+ wrmsr(IBS_OP_CTL2, ibs_op_ctl2_start_config(pm));
+ config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE;
+ wrmsr(IBS_OP_CTL, config);
+}
+
/*
* Start a PMC.
*/
static int
ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm)
{
- uint64_t config;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
@@ -347,20 +501,12 @@
*/
atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING);
- /*
- * Turn on the ENABLE bit. Zeroing out the control register eliminates
- * stale valid bits from spurious NMIs and it resets the counter.
- */
switch (ri) {
case IBS_PMC_FETCH:
- wrmsr(IBS_FETCH_CTL, 0);
- config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE;
- wrmsr(IBS_FETCH_CTL, config);
+ ibs_write_fetch_start(pm);
break;
case IBS_PMC_OP:
- wrmsr(IBS_OP_CTL, 0);
- config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE;
- wrmsr(IBS_OP_CTL, config);
+ ibs_write_op_start(pm);
break;
}
@@ -374,7 +520,8 @@
ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm)
{
int i;
- uint64_t config;
+ uint64_t config, config2;
+ bool use_alt_disable;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[ibs,%d] illegal CPU value %d", __LINE__, cpu));
@@ -394,23 +541,47 @@
* are stopping and discard spurious NMIs. We then retry clearing the
* control register for 50us. This gives us enough time and ensures
* that the valid bit is not accidently stuck after a spurious NMI.
+ *
+ * On Zen6 with the alternate disable bit (CPUID IbsDis), assert the
+ * ctl2 DISABLE bit first. This avoids an RMW hazard in ctl1 that the
+ * processor may update concurrently while sampling.
*/
config = pm->pm_md.pm_ibs.ibs_ctl;
+ config2 = pm->pm_md.pm_ibs.ibs_ctl2;
+ use_alt_disable = (ibs_features & CPUID_IBSID_IBSDIS) != 0;
atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING);
+ /*
+ * On Zen6, ctl2 DISABLE is the authoritative stop switch; skip
+ * the legacy ctl1 RMW and clear it directly
+ */
switch (ri) {
case IBS_PMC_FETCH:
- wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK);
- DELAY(1);
- config &= ~IBS_FETCH_CTL_ENABLE;
- wrmsr(IBS_FETCH_CTL, config);
+ if (use_alt_disable) {
+ wrmsr(IBS_FETCH_CTL2,
+ config2 | IBS_FETCH_CTL2_DISABLE);
+ wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_ENABLE);
+ } else {
+ wrmsr(IBS_FETCH_CTL,
+ config & ~IBS_FETCH_CTL_MAXCNTMASK);
+ DELAY(1);
+ config &= ~IBS_FETCH_CTL_ENABLE;
+ wrmsr(IBS_FETCH_CTL, config);
+ }
break;
case IBS_PMC_OP:
- wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_MAXCNTMASK);
- DELAY(1);
- config &= ~IBS_OP_CTL_ENABLE;
- wrmsr(IBS_OP_CTL, config);
+ if (use_alt_disable) {
+ wrmsr(IBS_OP_CTL2,
+ config2 | IBS_OP_CTL2_DISABLE);
+ wrmsr(IBS_OP_CTL, config & ~IBS_OP_CTL_ENABLE);
+ } else {
+ wrmsr(IBS_OP_CTL,
+ config & ~IBS_OP_CTL_MAXCNTMASK);
+ DELAY(1);
+ config &= ~IBS_OP_CTL_ENABLE;
+ wrmsr(IBS_OP_CTL, config);
+ }
break;
}
@@ -420,9 +591,13 @@
switch (ri) {
case IBS_PMC_FETCH:
wrmsr(IBS_FETCH_CTL, 0);
+ if (ibs_fetch_ctl2_supported)
+ wrmsr(IBS_FETCH_CTL2, 0);
break;
case IBS_PMC_OP:
wrmsr(IBS_OP_CTL, 0);
+ if (ibs_op_ctl2_supported)
+ wrmsr(IBS_OP_CTL2, 0);
break;
}
}
@@ -446,7 +621,8 @@
memset(&mpd, 0, sizeof(mpd));
mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH;
- mpd.pl_length = PMC_MPIDX_FETCH_MAX;
+ mpd.pl_length = ibs_fetch_ctl2_supported ?
+ (PMC_MPIDX_FETCH_CTL2 + 1) : PMC_MPIDX_FETCH_MAX;
mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config;
if ((ibs_features & CPUID_IBSID_IBSFETCHCTLEXTD) != 0) {
mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL);
@@ -456,10 +632,13 @@
mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] =
rdmsr(IBS_FETCH_PHYSADDR);
}
+ if (ibs_fetch_ctl2_supported) {
+ mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL2] = rdmsr(IBS_FETCH_CTL2);
+ }
pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
- wrmsr(IBS_FETCH_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE);
+ ibs_write_fetch_start(pm);
}
static void
@@ -476,7 +655,8 @@
memset(&mpd, 0, sizeof(mpd));
mpd.pl_type = PMC_CC_MULTIPART_IBS_OP;
- mpd.pl_length = PMC_MPIDX_OP_MAX;
+ mpd.pl_length = ibs_op_ctl2_supported ?
+ (PMC_MPIDX_OP_CTL2 + 1) : PMC_MPIDX_OP_MAX;
mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config;
mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP);
mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA);
@@ -490,10 +670,13 @@
if ((ibs_features & CPUID_IBSID_IBSOPDATA4) != 0) {
mpd.pl_mpdata[PMC_MPIDX_OP_DATA4] = rdmsr(IBS_OP_DATA4);
}
+ if (ibs_op_ctl2_supported) {
+ mpd.pl_mpdata[PMC_MPIDX_OP_CTL2] = rdmsr(IBS_OP_CTL2);
+ }
pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd);
- wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE);
+ ibs_write_op_start(pm);
}
/*
@@ -644,6 +827,10 @@
*/
wrmsr(IBS_FETCH_CTL, 0);
wrmsr(IBS_OP_CTL, 0);
+ if (ibs_fetch_ctl2_supported)
+ wrmsr(IBS_FETCH_CTL2, 0);
+ if (ibs_op_ctl2_supported)
+ wrmsr(IBS_OP_CTL2, 0);
/*
* Free up allocated space.
diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c
--- a/usr.sbin/pmcstat/pmcstat_log.c
+++ b/usr.sbin/pmcstat/pmcstat_log.c
@@ -371,10 +371,10 @@
#if defined(__amd64__) || defined(__i386__)
static void
-pmcstat_print_ibs_fetch(struct pmclog_ev_callchain *cc, int offset)
+pmcstat_print_ibs_fetch(struct pmclog_ev_callchain *cc, int offset, int len64)
{
uint64_t *ibsbuf = (uint64_t *)&cc->pl_pc[offset];
- uint64_t ctl;
+ uint64_t ctl, ctl2;
ctl = ibsbuf[PMC_MPIDX_FETCH_CTL];
PMCSTAT_PRINT_ENTRY("ibs-fetch", "%s%s%s%s",
@@ -390,15 +390,28 @@
PMCSTAT_PRINT_ENTRY("IBS", "Physical Address %" PRIx64,
ibsbuf[PMC_MPIDX_FETCH_PHYSADDR]);
}
+ if (len64 > PMC_MPIDX_FETCH_CTL2) {
+ ctl2 = ibsbuf[PMC_MPIDX_FETCH_CTL2];
+ if ((ctl2 & IBS_FETCH_CTL2_EXCLADDR63EQ1) != 0)
+ PMCSTAT_PRINT_ENTRY("ibs-fetch", "addr63=0");
+ if ((ctl2 & IBS_FETCH_CTL2_EXCLADDR63EQ0) != 0)
+ PMCSTAT_PRINT_ENTRY("ibs-fetch", "addr63=1");
+ if ((ctl2 & IBS_FETCH_CTL2_LATFILTERMASK) != 0) {
+ PMCSTAT_PRINT_ENTRY("ibs-fetch",
+ "fetchlat>=%" PRIu64,
+ (uint64_t)IBS_FETCH_CTL2_CTL_TO_LAT(ctl2));
+ }
+ }
}
static void
-pmcstat_print_ibs_op(struct pmclog_ev_callchain *cc, int offset)
+pmcstat_print_ibs_op(struct pmclog_ev_callchain *cc, int offset, int len64)
{
uint64_t *ibsbuf = (uint64_t *)&cc->pl_pc[offset];
- uint64_t data, data3;
+ uint64_t data, data2, data3, ctl2;
data = ibsbuf[PMC_MPIDX_OP_DATA];
+ data2 = ibsbuf[PMC_MPIDX_OP_DATA2];
data3 = ibsbuf[PMC_MPIDX_OP_DATA3];
if ((data & IBS_OP_DATA_RIPINVALID) == 0) {
@@ -416,6 +429,11 @@
(data3 & IBS_OP_DATA3_LOCKEDOP) ? "lock " : "",
(data3 & IBS_OP_DATA3_DCL1TLBMISS) ? "l1tlbmiss " : "",
(data3 & IBS_OP_DATA3_DCMISS) ? "dcmiss " : "");
+ if ((data2 & (IBS_OP_DATA2_STRMST | IBS_OP_DATA2_RMTSOCKET)) != 0) {
+ PMCSTAT_PRINT_ENTRY("ibs-op", "%s%s",
+ (data2 & IBS_OP_DATA2_STRMST) ? "streamstore " : "",
+ (data2 & IBS_OP_DATA2_RMTSOCKET) ? "remotesocket" : "");
+ }
PMCSTAT_PRINT_ENTRY("ibs-op", "Latency %" PRIu64,
IBS_OP_DATA3_TO_DCLAT(data3));
if ((data3 & IBS_OP_DATA3_DCLINADDRVALID) != 0) {
@@ -426,6 +444,15 @@
PMCSTAT_PRINT_ENTRY("ibs-op", "Physical Address %" PRIx64,
ibsbuf[PMC_MPIDX_OP_DC_PHYSADDR]);
}
+ if (len64 > PMC_MPIDX_OP_CTL2) {
+ ctl2 = ibsbuf[PMC_MPIDX_OP_CTL2];
+ if ((ctl2 & IBS_OP_CTL2_EXCLRIP63EQ1) != 0)
+ PMCSTAT_PRINT_ENTRY("ibs-op", "addr63=0");
+ if ((ctl2 & IBS_OP_CTL2_EXCLRIP63EQ0) != 0)
+ PMCSTAT_PRINT_ENTRY("ibs-op", "addr63=1");
+ if ((ctl2 & IBS_OP_CTL2_STRMSTFILTER) != 0)
+ PMCSTAT_PRINT_ENTRY("ibs-op", "streamstore");
+ }
}
#endif
@@ -446,9 +473,11 @@
return (offset);
#if defined(__amd64__) || defined(__i386__)
} else if (type == PMC_CC_MULTIPART_IBS_FETCH) {
- pmcstat_print_ibs_fetch(cc, offset);
+ pmcstat_print_ibs_fetch(cc, offset,
+ len / (sizeof(uint64_t) / sizeof(uintptr_t)));
} else if (type == PMC_CC_MULTIPART_IBS_OP) {
- pmcstat_print_ibs_op(cc, offset);
+ pmcstat_print_ibs_op(cc, offset,
+ len / (sizeof(uint64_t) / sizeof(uintptr_t)));
#endif
} else {
PMCSTAT_PRINT_ENTRY("unsupported multipart type!");

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 22, 10:27 PM (16 h, 38 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35384187
Default Alt Text
D56914.id177522.diff (25 KB)

Event Timeline