Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F177733216
D61053.1795154554.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Flag For Later
Award Token
Authored By
Unknown
Size
21 KB
Referenced Files
None
Subscribers
None
D61053.1795154554.diff
View Options
diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h
--- a/sys/dev/hwpmc/hwpmc_amd.h
+++ b/sys/dev/hwpmc/hwpmc_amd.h
@@ -60,9 +60,14 @@
*/
/* CORE */
-#define AMD_PMC_CORE_BASE 0xC0010200
-#define AMD_PMC_CORE_DEFAULT 6
-#define AMD_PMC_CORE_MAX 16
+#define AMD_PMC_CORE_BASE 0xC0010200
+#define AMD_PMC_CORE_DEFAULT 6
+#define AMD_PMC_CORE_MAX 16
+
+/* MergeEvent Support */
+#define AMD_PMC_FPRETSSSEAVXOPS 3 /* needs a merge pair */
+#define AMD_PMC_MERGE_EVENT 0xfff /* MergeEvent */
+#define AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING 0xffff /* odd ctr preload */
/*
* These are the PerfMonV2 global-control MSRs (Fam 19h Zen3+ / 1Ah).
@@ -235,6 +240,8 @@
/* MD extension for 'struct pmc' */
struct pmc_md_amd_pmc {
uint64_t pm_amd_evsel;
+ struct pmc *pmc_merge_event; /* its placeholder */
+ struct pmc *pmc_merge_parent; /* its merge PMC */
};
#endif /* _KERNEL */
diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -74,6 +74,10 @@
static u_int amd_umc_pkg_shift;
static u_int amd_umc_npkgs;
static u_int *amd_umc_gate_refcnt; /* [amd_umc_npkgs] */
+/* First row index of the AMD core counters in the hwpmc row space. */
+static int amd_core_pmc_baseri;
+/* True if the CPU supports merging counter pairs (family 17h and later). */
+static bool amd_merge_event_supported;
struct amd_event_code_map {
enum pmc_event pe_ev; /* enum value */
@@ -216,6 +220,13 @@
/* Mask with one bit for each core counter */
static uint64_t amd_global_cntr_mask;
+/*
+ * Number of process-mode merge PMCs using each odd core row as their
+ * partner counter. Lets amd_can_assign_pmc() keep other events off a
+ * reserved odd row. Protected by pmc_sx.
+ */
+static u_int amd_process_mode_merge_table[AMD_PMC_CORE_MAX];
+
SYSCTL_DECL(_kern_hwpmc);
SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_core_extra_mask, CTLFLAG_RDTUN,
@@ -238,6 +249,91 @@
&amd_perfmon_v2, 0,
"AMD PerfMonV2 global-control path selected (read-only)");
+/*
+ * Return true if 'config' on row 'ri' selects an event that needs a
+ * merged counter pair. Used only when deciding at allocation time.
+ */
+static __inline bool
+amd_is_merge_event(int ri, uint64_t config)
+{
+ return (amd_merge_event_supported &&
+ amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE &&
+ (config & AMD_PMC_EVENTMASK) == AMD_PMC_FPRETSSSEAVXOPS);
+}
+
+/*
+ * Return true if 'pm' was allocated as a merge event and owns a
+ * placeholder for its odd counter. False for the placeholder itself.
+ */
+static __inline bool
+amd_pmc_is_merge(const struct pmc *pm)
+{
+ return ((pm->pm_md.pm_amd.pmc_merge_event != NULL) &&
+ (pm->pm_flags & PMC_F_MERGE_EVENT) == 0);
+}
+
+/* True if pm's placeholder holds the odd counter on this CPU. */
+static __inline bool
+amd_merge_installed(int cpu, int ri, const struct pmc *pm)
+{
+ return (amd_pmc_is_merge(pm) &&
+ amd_pcpu[cpu]->pc_amdpmcs[ri + 1].phw_pmc ==
+ pm->pm_md.pm_amd.pmc_merge_event);
+}
+
+/*
+ * Check whether a merge event on even core row 'ri' may also use the
+ * odd counter ri + 1. A system-mode merge needs that counter free on
+ * its own CPU. A process-mode merge can run on any CPU, so the whole
+ * odd row may only be shared with other process-mode merges.
+ */
+static int
+amd_merge_can_reserve(int cpu, int ri, struct pmc *pm)
+{
+ struct pmc_hw *phw_odd;
+ int disp, odd;
+
+ odd = ri + 1;
+ if ((ri & 1) != 0 || odd >= amd_core_npmcs)
+ return (EINVAL);
+
+ disp = pmc_getrowdisp(amd_core_pmc_baseri + odd);
+
+ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
+ phw_odd = &amd_pcpu[cpu]->pc_amdpmcs[odd];
+ if ((phw_odd->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
+ phw_odd->phw_pmc != NULL ||
+ disp > 0 ||
+ amd_process_mode_merge_table[odd] != 0)
+ return (EINVAL);
+ } else {
+ /*
+ * Each process-mode merge adds one thread mark and
+ * one table count, so the row is free for us only if
+ * every thread mark belongs to a merge.
+ */
+ if (disp != (int)amd_process_mode_merge_table[odd])
+ return (EINVAL);
+ }
+ return (0);
+}
+
+static void
+amd_merge_reserve(int ri, struct pmc *pm)
+{
+ pmc_markrowdisp(amd_core_pmc_baseri + ri + 1, PMC_TO_MODE(pm));
+ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
+ amd_process_mode_merge_table[ri + 1]++;
+}
+
+static void
+amd_merge_unreserve(int ri, struct pmc *pm)
+{
+ pmc_unmarkrowdisp(amd_core_pmc_baseri + ri + 1, PMC_TO_MODE(pm));
+ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
+ amd_process_mode_merge_table[ri + 1]--;
+}
+
static void
amd_init_policy(void)
{
@@ -454,7 +550,7 @@
static int
amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
enum pmc_mode mode;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
@@ -463,11 +559,23 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
+
mode = PMC_TO_MODE(pm);
/* use 2's complement of the count for sampling mode PMCs */
- if (PMC_IS_SAMPLING_MODE(mode))
+ if (PMC_IS_SAMPLING_MODE(mode)) {
+ if (amd_merge_installed(cpu, ri, pm)) {
+ wrmsr(pd_merge->pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
+ }
+
v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
+ } else {
+ if (amd_merge_installed(cpu, ri, pm)) {
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+ }
PMCDBG3(MDP, WRI, 1, "amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
@@ -482,7 +590,7 @@
static int
amd_config_pmc(int cpu, int ri, struct pmc *pm)
{
- struct pmc_hw *phw;
+ struct pmc_hw *phw, *phw_merge;
PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
@@ -492,6 +600,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
+ phw_merge = &amd_pcpu[cpu]->pc_amdpmcs[ri + 1];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
@@ -501,6 +610,48 @@
amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE)
amd_v2_forget_core(cpu, ri, phw->phw_pmc);
+ if (pm) {
+ if (amd_pmc_is_merge(pm)) {
+ KASSERT(phw_merge->phw_pmc == NULL,
+ ("[amd,%d] merge pair ri %d odd counter busy: %p",
+ __LINE__, ri, phw_merge->phw_pmc));
+ if (phw_merge->phw_pmc == NULL) {
+ phw_merge->phw_pmc =
+ pm->pm_md.pm_amd.pmc_merge_event;
+ /*
+ * Hold the PMC until this CPU removes the
+ * placeholder: switch-out drops the core's
+ * runcount before it deconfigures, and
+ * release only waits for zero.
+ */
+ counter_u64_add(pm->pm_runcount, 1);
+ }
+ }
+ } else {
+ if (phw->phw_pmc != NULL &&
+ amd_pmc_is_merge(phw->phw_pmc) &&
+ phw_merge->phw_pmc ==
+ phw->phw_pmc->pm_md.pm_amd.pmc_merge_event) {
+ /*
+ * Clear the odd counter on every deconfigure, not only
+ * in stop: a stalled PMC is deconfigured without being
+ * stopped. A leftover MergeEvent would make the next
+ * user of the even counter count in merged mode.
+ */
+ if (amd_perfmon_v2) {
+ amd_v2_forget_core(cpu, ri + 1, phw->phw_pmc);
+ }
+ wrmsr(amd_pmcdesc[ri + 1].pm_evsel, 0);
+ wrmsr(amd_pmcdesc[ri + 1].pm_perfctr, 0);
+ phw_merge->phw_pmc = pm;
+ /*
+ * Last use of the old PMC: release may free it
+ * after this.
+ */
+ counter_u64_add(phw->phw_pmc->pm_runcount, -1);
+ }
+ }
+
phw->phw_pmc = pm;
return (0);
}
@@ -512,6 +663,12 @@
amd_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
+ /*
+ * Hide the merge placeholder: it is not a real PMC and has no run
+ * count or per-CPU state for the MI code to use.
+ */
+ if (*ppm != NULL && ((*ppm)->pm_flags & PMC_F_MERGE_EVENT) != 0)
+ *ppm = NULL;
return (0);
}
@@ -553,7 +710,8 @@
* Check if an event can use row index 'ri'.
*/
static int
-amd_can_assign_pmc(int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a)
+amd_can_assign_pmc(int ri, int cpu, struct pmc *pm,
+ const struct pmc_op_pmcallocate *a)
{
const struct pmc_descr *pd;
uint64_t allowed_unitmask, caps, config, unitmask;
@@ -578,6 +736,24 @@
((pd->pd_caps & PMC_CAP_PRECISE) == 0))
return (EINVAL);
+ /*
+ * Only the driver may program the MergeEvent. Selected directly,
+ * it would merge this counter with its even neighbor and corrupt
+ * another PMC's count.
+ */
+ if ((amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE) &&
+ (a->pm_md.pm_amd.pm_amd_config & AMD_PMC_EVENTMASK) ==
+ AMD_PMC_TO_EVENTMASK(AMD_PMC_MERGE_EVENT))
+ return (EINVAL);
+
+ /*
+ * An odd core row reserved by a process-mode merge is busy on
+ * every CPU, because the merge PMC can run on any of them.
+ */
+ if ((ri & 1) != 0 && ri < amd_core_npmcs &&
+ amd_process_mode_merge_table[ri] != 0)
+ return (EINVAL);
+
/* PMC_F_EV_PMU: config comes from pmu-events tables. */
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
@@ -587,6 +763,22 @@
"AMD PMU config has unsupported bits %#jx",
(uintmax_t)(config & ~amd_config_mask(
amd_pmcdesc[ri].pm_subclass, caps))));
+
+ /*
+ * Large Increment per Cycle events need two counters: the
+ * even counter counts the event, and the next odd counter
+ * runs the MergeEvent as its upper 16 bits. hwpmc assigns
+ * one row per PMC, so the driver reserves the odd row itself;
+ * see amd_merge_can_reserve().
+ */
+ if (amd_is_merge_event(ri, config)) {
+ if (amd_merge_can_reserve(cpu, ri, pm) != 0)
+ return (EINVAL);
+ }
+
+ pm->pm_md.pm_amd.pm_amd_evsel = config;
+ PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
+ ri, (uintmax_t)config);
return (0);
}
@@ -619,14 +811,15 @@
* Check if a given PMC allocation is feasible.
*/
static int
-amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
+amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
+ struct pmc *pmc_merge;
uint64_t allowed_unitmask, caps, config, unitmask;
enum pmc_event pe;
int error, i;
- error = amd_can_assign_pmc(ri, pm, a);
+ error = amd_can_assign_pmc(ri, cpu, pm, a);
if (error != 0)
return (error);
@@ -637,6 +830,30 @@
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
pm->pm_md.pm_amd.pm_amd_evsel = config;
+
+ /*
+ * Allocate a placeholder PMC for the odd counter. It fills
+ * the odd slot while the merge PMC is configured on a CPU,
+ * and is marked with PMC_F_MERGE_EVENT so the interrupt
+ * handlers and amd_get_config() skip it.
+ * amd_can_assign_pmc() has already checked the reservation.
+ */
+ if (amd_is_merge_event(ri, config)) {
+ pmc_merge = malloc(sizeof(struct pmc), M_PMC,
+ M_WAITOK | M_ZERO);
+ pmc_merge->pm_id = PMC_ID_MAKE_ID(a->pm_cpu, a->pm_mode,
+ a->pm_class, amd_core_pmc_baseri + ri + 1);
+ pmc_merge->pm_event = pm->pm_event;
+ pmc_merge->pm_state = PMC_STATE_ALLOCATED;
+ pmc_merge->pm_caps = pm->pm_caps;
+ pmc_merge->pm_flags = pm->pm_flags | PMC_F_MERGE_EVENT;
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel =
+ AMD_PMC_TO_EVENTMASK(AMD_PMC_MERGE_EVENT);
+ pmc_merge->pm_md.pm_amd.pmc_merge_parent = pm;
+ pm->pm_md.pm_amd.pmc_merge_event = pmc_merge;
+ amd_merge_reserve(ri, pm);
+ }
+
PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
ri, (uintmax_t)config);
return (0);
@@ -683,11 +900,11 @@
}
/*
- * Release machine dependent state associated with a PMC. This is a
- * no-op on this architecture.
+ * Release machine dependent state associated with a PMC. For a merge
+ * PMC, drop the odd counter's reservation and free its placeholder.
*/
static int
-amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
+amd_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw __diagused;
@@ -701,6 +918,16 @@
KASSERT(phw->phw_pmc == NULL,
("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
+ /*
+ * The PMC has already been deconfigured on every CPU, so the
+ * placeholder is no longer in any slot and can be freed.
+ */
+ if (amd_pmc_is_merge(pmc)) {
+ amd_merge_unreserve(ri, pmc);
+ free(pmc->pm_md.pm_amd.pmc_merge_event, M_PMC);
+ pmc->pm_md.pm_amd.pmc_merge_event = NULL;
+ }
+
return (0);
}
@@ -708,10 +935,11 @@
* Start a PMC.
*/
static int
-amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm)
+amd_start_pmc(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
- uint64_t config;
+ const struct amd_descr *pd, *pd_merge;
+ const struct pmc *pmc_merge;
+ uint64_t config, config_merge;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
@@ -719,6 +947,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri);
@@ -744,17 +973,26 @@
PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config);
+ /* Enable the MergeEvent on the odd counter before the even one. */
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pmc_merge = pm->pm_md.pm_amd.pmc_merge_event;
+ config_merge =
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
+ wrmsr(pd_merge->pm_evsel, config_merge);
+ }
+
wrmsr(pd->pm_evsel, config);
return (0);
}
/* Start one PMC. pcd_start_all commits the virtual GLOBAL_CTL bits. */
static int
-amd_start_pmc_v2(int cpu __diagused, int ri, struct pmc *pm)
+amd_start_pmc_v2(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
+ const struct pmc *pmc_merge;
enum pmc_mode mode;
- uint64_t config;
+ uint64_t config, config_merge;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
@@ -762,6 +1000,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
mode = PMC_TO_MODE(pm);
PMCDBG2(MDP, STA, 1, "amd-start-v2 cpu=%d ri=%d", cpu, ri);
@@ -796,6 +1035,23 @@
DPCPU_SET(nmi_counter, 1);
}
+ /*
+ * Enable the MergeEvent on the odd counter before the even one.
+ * Its global bit follows the even counter's: staged for process
+ * mode, set directly for system mode.
+ */
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pmc_merge = pm->pm_md.pm_amd.pmc_merge_event;
+ config_merge =
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
+ wrmsr(pd_merge->pm_evsel, config_merge);
+ if (PMC_IS_VIRTUAL_MODE(mode))
+ amd_v2_stage_virtual(cpu, ri + 1);
+ else
+ atomic_set_int(&amd_pcpu[cpu]->pc_global_mask,
+ amd_v2_counter_mask(ri + 1));
+ }
+
/* Enable EVSEL while the virtual slot's global bit is off. */
if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC)
config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_UMC_ENABLE;
@@ -817,9 +1073,9 @@
* Stop a PMC.
*/
static int
-amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm)
+amd_stop_pmc(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
uint64_t config;
int i;
@@ -829,6 +1085,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri);
@@ -846,8 +1103,21 @@
config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE;
}
+
wrmsr(pd->pm_evsel, config);
+ if (amd_merge_installed(cpu, ri, pm)) {
+ /*
+ * Clear the odd counter after the even one is disabled, so
+ * no carry is lost. A leftover MergeEvent would make the
+ * next user of the even counter count in merged mode, and
+ * a leftover count would appear as the upper 16 bits of its
+ * value.
+ */
+ wrmsr(pd_merge->pm_evsel, 0);
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+
/*
* Due to NMI latency on newer AMD processors
* NMI interrupts are ignored, which leads to
@@ -867,9 +1137,9 @@
/* Stop one PMC. pcd_stop_all already zeroed the virtual GLOBAL_CTL bits. */
static int
-amd_stop_pmc_v2(int cpu __diagused, int ri, struct pmc *pm)
+amd_stop_pmc_v2(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
enum pmc_mode mode;
uint64_t config;
int i;
@@ -891,6 +1161,21 @@
amd_v2_publish_mask(cpu);
}
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pd_merge = &amd_pmcdesc[ri + 1];
+
+ atomic_clear_int(&amd_pcpu[cpu]->pc_global_mask,
+ amd_v2_counter_mask(ri + 1));
+ amd_v2_publish_mask(cpu);
+ /*
+ * Clear the odd counter's global bit, MergeEvent and count.
+ * The even counter's global bit is already off, so no carry
+ * is lost. See amd_stop_pmc() for why this is needed.
+ */
+ wrmsr(pd_merge->pm_evsel, 0);
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+
/* Disable EVSEL after the counter's global bit is off. */
if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC)
config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE;
@@ -958,7 +1243,8 @@
break;
if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
- !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
+ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) ||
+ (pm->pm_flags & PMC_F_MERGE_EVENT)) {
continue;
}
@@ -979,7 +1265,6 @@
v = pm->pm_sc.pm_reloadcount;
config = rdmsr(evsel);
-
if (amd_pmcdesc[i].pm_subclass == PMC_AMD_SUB_CLASS_UMC) {
KASSERT((config & ~AMD_PMC_UMC_ENABLE) ==
(pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE),
@@ -996,6 +1281,14 @@
wrmsr(evsel, config & ~AMD_PMC_ENABLE);
}
wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
+ /*
+ * Preload the odd counter again so the merged counter
+ * overflows at the next even-counter wrap. Do it before
+ * logging: after a stall, the restart does not reload it.
+ */
+ if (amd_merge_installed(cpu, i, pm))
+ wrmsr(amd_pmcdesc[i + 1].pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
/* Restart the counter if logging succeeded. */
error = pmc_process_interrupt(PMC_HR, pm, tf);
@@ -1072,7 +1365,9 @@
*/
for (i = 0; i < amd_core_npmcs; i++) {
pm = pac->pc_amdpmcs[i].phw_pmc;
- if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
+ /* The merge placeholder is not a sampling PMC of its own. */
+ if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
+ !(pm->pm_flags & PMC_F_MERGE_EVENT))
active++;
}
@@ -1082,7 +1377,8 @@
pending &= ~(1ULL << i);
if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
- !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
+ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) ||
+ (pm->pm_flags & PMC_F_MERGE_EVENT)) {
continue;
}
@@ -1091,6 +1387,17 @@
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
+ /*
+ * Preload the odd counter again so the merged counter
+ * overflows at the next even-counter wrap. Without it, the
+ * pair would not interrupt again. A merged overflow sets
+ * only the even counter's GLOBAL_STATUS bit.
+ */
+ if (amd_merge_installed(cpu, i, pm)) {
+ wrmsr(amd_pmcdesc[i + 1].pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
+ }
+
/* Reload the counter. */
v = pm->pm_sc.pm_reloadcount;
wrmsr(amd_pmcdesc[i].pm_perfctr,
@@ -1158,6 +1465,10 @@
if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) != 0) {
pi->pm_enabled = true;
*ppmc = phw->phw_pmc;
+ /* Report the merge PMC that owns this odd counter. */
+ if (*ppmc != NULL &&
+ ((*ppmc)->pm_flags & PMC_F_MERGE_EVENT) != 0)
+ *ppmc = phw->phw_pmc->pm_md.pm_amd.pmc_merge_parent;
} else {
pi->pm_enabled = false;
*ppmc = NULL;
@@ -1553,6 +1864,14 @@
model = CPUID_TO_MODEL(cpu_id);
stepping = CPUID_TO_STEPPING(cpu_id);
+ /*
+ * Counter merging for Large Increment per Cycle events is
+ * available from family 17h on.
+ */
+ if (family >= 0x17) {
+ amd_merge_event_supported = true;
+ }
+
if (family == 0x18)
snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X",
family, model, stepping);
@@ -1815,6 +2134,8 @@
pmc_mdep->pmd_npmc += amd_npmcs;
+ amd_core_pmc_baseri = pcd->pcd_ri;
+
amd_init_policy();
PMCDBG0(MDP, INI, 0, "amd-initialize");
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -2984,9 +2984,37 @@
int
pmc_getrowdisp(int ri)
{
+ sx_assert(&pmc_sx, SX_XLOCKED);
return (pmc_pmcdisp[ri]);
}
+/*
+ * Add or remove one mark on row 'ri' for a PMC of the given mode.
+ * For drivers whose PMCs also use another row, such as AMD merge
+ * events. The caller must hold pmc_sx exclusively.
+ */
+void
+pmc_markrowdisp(int ri, enum pmc_mode mode)
+{
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ if (PMC_IS_SYSTEM_MODE(mode))
+ PMC_MARK_ROW_STANDALONE(ri);
+ else
+ PMC_MARK_ROW_THREAD(ri);
+}
+
+void
+pmc_unmarkrowdisp(int ri, enum pmc_mode mode)
+{
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ if (PMC_IS_SYSTEM_MODE(mode))
+ PMC_UNMARK_ROW_STANDALONE(ri);
+ else
+ PMC_UNMARK_ROW_THREAD(ri);
+}
+
/*
* Check if a PMC at row index 'ri' can be allocated to the current
* process.
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -386,6 +386,7 @@
#define PMC_F_ATTACHED_TO_OWNER 0x00010000 /*attached to owner*/
#define PMC_F_NEEDS_LOGFILE 0x00020000 /*needs log file */
#define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */
+#define PMC_F_MERGE_EVENT 0x00080000 /*merge pair placeholder */
#define PMC_CALLCHAIN_DEPTH_MAX 512
@@ -1262,6 +1263,8 @@
struct pmc_mdep *pmc_md_initialize(void); /* MD init function */
void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */
int pmc_getrowdisp(int _ri);
+void pmc_markrowdisp(int _ri, enum pmc_mode _mode);
+void pmc_unmarkrowdisp(int _ri, enum pmc_mode _mode);
int pmc_process_interrupt_mp(int _ring, struct pmc *_pm,
struct trapframe *_tf, struct pmc_multipart *mp);
int pmc_process_interrupt(int _ring, struct pmc *_pm,
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Fri, Nov 20, 6:02 AM (4 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41831385
Default Alt Text
D61053.1795154554.diff (21 KB)
Attached To
Mode
D61053: hwpmc_amd: Support Large Increment per Cycle events
Attached
Detach File
Event Timeline