Page MenuHomeFreeBSD

D61053.1795154554.diff
No OneTemporary

Authored By
Unknown
Size
21 KB
Referenced Files
None
Subscribers
None

D61053.1795154554.diff

diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h
--- a/sys/dev/hwpmc/hwpmc_amd.h
+++ b/sys/dev/hwpmc/hwpmc_amd.h
@@ -60,9 +60,14 @@
*/
/* CORE */
-#define AMD_PMC_CORE_BASE 0xC0010200
-#define AMD_PMC_CORE_DEFAULT 6
-#define AMD_PMC_CORE_MAX 16
+#define AMD_PMC_CORE_BASE 0xC0010200
+#define AMD_PMC_CORE_DEFAULT 6
+#define AMD_PMC_CORE_MAX 16
+
+/* MergeEvent Support */
+#define AMD_PMC_FPRETSSSEAVXOPS 3 /* needs a merge pair */
+#define AMD_PMC_MERGE_EVENT 0xfff /* MergeEvent */
+#define AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING 0xffff /* odd ctr preload */
/*
* These are the PerfMonV2 global-control MSRs (Fam 19h Zen3+ / 1Ah).
@@ -235,6 +240,8 @@
/* MD extension for 'struct pmc' */
struct pmc_md_amd_pmc {
uint64_t pm_amd_evsel;
+ struct pmc *pmc_merge_event; /* its placeholder */
+ struct pmc *pmc_merge_parent; /* its merge PMC */
};
#endif /* _KERNEL */
diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c
--- a/sys/dev/hwpmc/hwpmc_amd.c
+++ b/sys/dev/hwpmc/hwpmc_amd.c
@@ -74,6 +74,10 @@
static u_int amd_umc_pkg_shift;
static u_int amd_umc_npkgs;
static u_int *amd_umc_gate_refcnt; /* [amd_umc_npkgs] */
+/* First row index of the AMD core counters in the hwpmc row space. */
+static int amd_core_pmc_baseri;
+/* True if the CPU supports merging counter pairs (family 17h and later). */
+static bool amd_merge_event_supported;
struct amd_event_code_map {
enum pmc_event pe_ev; /* enum value */
@@ -216,6 +220,13 @@
/* Mask with one bit for each core counter */
static uint64_t amd_global_cntr_mask;
+/*
+ * Number of process-mode merge PMCs using each odd core row as their
+ * partner counter. Lets amd_can_assign_pmc() keep other events off a
+ * reserved odd row. Protected by pmc_sx.
+ */
+static u_int amd_process_mode_merge_table[AMD_PMC_CORE_MAX];
+
SYSCTL_DECL(_kern_hwpmc);
SYSCTL_U64(_kern_hwpmc, OID_AUTO, amd_core_extra_mask, CTLFLAG_RDTUN,
@@ -238,6 +249,91 @@
&amd_perfmon_v2, 0,
"AMD PerfMonV2 global-control path selected (read-only)");
+/*
+ * Return true if 'config' on row 'ri' selects an event that needs a
+ * merged counter pair. Used only when deciding at allocation time.
+ */
+static __inline bool
+amd_is_merge_event(int ri, uint64_t config)
+{
+ return (amd_merge_event_supported &&
+ amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE &&
+ (config & AMD_PMC_EVENTMASK) == AMD_PMC_FPRETSSSEAVXOPS);
+}
+
+/*
+ * Return true if 'pm' was allocated as a merge event and owns a
+ * placeholder for its odd counter. False for the placeholder itself.
+ */
+static __inline bool
+amd_pmc_is_merge(const struct pmc *pm)
+{
+ return ((pm->pm_md.pm_amd.pmc_merge_event != NULL) &&
+ (pm->pm_flags & PMC_F_MERGE_EVENT) == 0);
+}
+
+/* True if pm's placeholder holds the odd counter on this CPU. */
+static __inline bool
+amd_merge_installed(int cpu, int ri, const struct pmc *pm)
+{
+ return (amd_pmc_is_merge(pm) &&
+ amd_pcpu[cpu]->pc_amdpmcs[ri + 1].phw_pmc ==
+ pm->pm_md.pm_amd.pmc_merge_event);
+}
+
+/*
+ * Check whether a merge event on even core row 'ri' may also use the
+ * odd counter ri + 1. A system-mode merge needs that counter free on
+ * its own CPU. A process-mode merge can run on any CPU, so the whole
+ * odd row may only be shared with other process-mode merges.
+ */
+static int
+amd_merge_can_reserve(int cpu, int ri, struct pmc *pm)
+{
+ struct pmc_hw *phw_odd;
+ int disp, odd;
+
+ odd = ri + 1;
+ if ((ri & 1) != 0 || odd >= amd_core_npmcs)
+ return (EINVAL);
+
+ disp = pmc_getrowdisp(amd_core_pmc_baseri + odd);
+
+ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
+ phw_odd = &amd_pcpu[cpu]->pc_amdpmcs[odd];
+ if ((phw_odd->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
+ phw_odd->phw_pmc != NULL ||
+ disp > 0 ||
+ amd_process_mode_merge_table[odd] != 0)
+ return (EINVAL);
+ } else {
+ /*
+ * Each process-mode merge adds one thread mark and
+ * one table count, so the row is free for us only if
+ * every thread mark belongs to a merge.
+ */
+ if (disp != (int)amd_process_mode_merge_table[odd])
+ return (EINVAL);
+ }
+ return (0);
+}
+
+static void
+amd_merge_reserve(int ri, struct pmc *pm)
+{
+ pmc_markrowdisp(amd_core_pmc_baseri + ri + 1, PMC_TO_MODE(pm));
+ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
+ amd_process_mode_merge_table[ri + 1]++;
+}
+
+static void
+amd_merge_unreserve(int ri, struct pmc *pm)
+{
+ pmc_unmarkrowdisp(amd_core_pmc_baseri + ri + 1, PMC_TO_MODE(pm));
+ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
+ amd_process_mode_merge_table[ri + 1]--;
+}
+
static void
amd_init_policy(void)
{
@@ -454,7 +550,7 @@
static int
amd_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
enum pmc_mode mode;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
@@ -463,11 +559,23 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
+
mode = PMC_TO_MODE(pm);
/* use 2's complement of the count for sampling mode PMCs */
- if (PMC_IS_SAMPLING_MODE(mode))
+ if (PMC_IS_SAMPLING_MODE(mode)) {
+ if (amd_merge_installed(cpu, ri, pm)) {
+ wrmsr(pd_merge->pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
+ }
+
v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
+ } else {
+ if (amd_merge_installed(cpu, ri, pm)) {
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+ }
PMCDBG3(MDP, WRI, 1, "amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
@@ -482,7 +590,7 @@
static int
amd_config_pmc(int cpu, int ri, struct pmc *pm)
{
- struct pmc_hw *phw;
+ struct pmc_hw *phw, *phw_merge;
PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
@@ -492,6 +600,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
+ phw_merge = &amd_pcpu[cpu]->pc_amdpmcs[ri + 1];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
@@ -501,6 +610,48 @@
amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE)
amd_v2_forget_core(cpu, ri, phw->phw_pmc);
+ if (pm) {
+ if (amd_pmc_is_merge(pm)) {
+ KASSERT(phw_merge->phw_pmc == NULL,
+ ("[amd,%d] merge pair ri %d odd counter busy: %p",
+ __LINE__, ri, phw_merge->phw_pmc));
+ if (phw_merge->phw_pmc == NULL) {
+ phw_merge->phw_pmc =
+ pm->pm_md.pm_amd.pmc_merge_event;
+ /*
+ * Hold the PMC until this CPU removes the
+ * placeholder: switch-out drops the core's
+ * runcount before it deconfigures, and
+ * release only waits for zero.
+ */
+ counter_u64_add(pm->pm_runcount, 1);
+ }
+ }
+ } else {
+ if (phw->phw_pmc != NULL &&
+ amd_pmc_is_merge(phw->phw_pmc) &&
+ phw_merge->phw_pmc ==
+ phw->phw_pmc->pm_md.pm_amd.pmc_merge_event) {
+ /*
+ * Clear the odd counter on every deconfigure, not only
+ * in stop: a stalled PMC is deconfigured without being
+ * stopped. A leftover MergeEvent would make the next
+ * user of the even counter count in merged mode.
+ */
+ if (amd_perfmon_v2) {
+ amd_v2_forget_core(cpu, ri + 1, phw->phw_pmc);
+ }
+ wrmsr(amd_pmcdesc[ri + 1].pm_evsel, 0);
+ wrmsr(amd_pmcdesc[ri + 1].pm_perfctr, 0);
+ phw_merge->phw_pmc = pm;
+ /*
+ * Last use of the old PMC: release may free it
+ * after this.
+ */
+ counter_u64_add(phw->phw_pmc->pm_runcount, -1);
+ }
+ }
+
phw->phw_pmc = pm;
return (0);
}
@@ -512,6 +663,12 @@
amd_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
+ /*
+ * Hide the merge placeholder: it is not a real PMC and has no run
+ * count or per-CPU state for the MI code to use.
+ */
+ if (*ppm != NULL && ((*ppm)->pm_flags & PMC_F_MERGE_EVENT) != 0)
+ *ppm = NULL;
return (0);
}
@@ -553,7 +710,8 @@
* Check if an event can use row index 'ri'.
*/
static int
-amd_can_assign_pmc(int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a)
+amd_can_assign_pmc(int ri, int cpu, struct pmc *pm,
+ const struct pmc_op_pmcallocate *a)
{
const struct pmc_descr *pd;
uint64_t allowed_unitmask, caps, config, unitmask;
@@ -578,6 +736,24 @@
((pd->pd_caps & PMC_CAP_PRECISE) == 0))
return (EINVAL);
+ /*
+ * Only the driver may program the MergeEvent. Selected directly,
+ * it would merge this counter with its even neighbor and corrupt
+ * another PMC's count.
+ */
+ if ((amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE) &&
+ (a->pm_md.pm_amd.pm_amd_config & AMD_PMC_EVENTMASK) ==
+ AMD_PMC_TO_EVENTMASK(AMD_PMC_MERGE_EVENT))
+ return (EINVAL);
+
+ /*
+ * An odd core row reserved by a process-mode merge is busy on
+ * every CPU, because the merge PMC can run on any of them.
+ */
+ if ((ri & 1) != 0 && ri < amd_core_npmcs &&
+ amd_process_mode_merge_table[ri] != 0)
+ return (EINVAL);
+
/* PMC_F_EV_PMU: config comes from pmu-events tables. */
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
@@ -587,6 +763,22 @@
"AMD PMU config has unsupported bits %#jx",
(uintmax_t)(config & ~amd_config_mask(
amd_pmcdesc[ri].pm_subclass, caps))));
+
+ /*
+ * Large Increment per Cycle events need two counters: the
+ * even counter counts the event, and the next odd counter
+ * runs the MergeEvent as its upper 16 bits. hwpmc assigns
+ * one row per PMC, so the driver reserves the odd row itself;
+ * see amd_merge_can_reserve().
+ */
+ if (amd_is_merge_event(ri, config)) {
+ if (amd_merge_can_reserve(cpu, ri, pm) != 0)
+ return (EINVAL);
+ }
+
+ pm->pm_md.pm_amd.pm_amd_evsel = config;
+ PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
+ ri, (uintmax_t)config);
return (0);
}
@@ -619,14 +811,15 @@
* Check if a given PMC allocation is feasible.
*/
static int
-amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
+amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
+ struct pmc *pmc_merge;
uint64_t allowed_unitmask, caps, config, unitmask;
enum pmc_event pe;
int error, i;
- error = amd_can_assign_pmc(ri, pm, a);
+ error = amd_can_assign_pmc(ri, cpu, pm, a);
if (error != 0)
return (error);
@@ -637,6 +830,30 @@
if ((a->pm_flags & PMC_F_EV_PMU) != 0) {
config = a->pm_md.pm_amd.pm_amd_config;
pm->pm_md.pm_amd.pm_amd_evsel = config;
+
+ /*
+ * Allocate a placeholder PMC for the odd counter. It fills
+ * the odd slot while the merge PMC is configured on a CPU,
+ * and is marked with PMC_F_MERGE_EVENT so the interrupt
+ * handlers and amd_get_config() skip it.
+ * amd_can_assign_pmc() has already checked the reservation.
+ */
+ if (amd_is_merge_event(ri, config)) {
+ pmc_merge = malloc(sizeof(struct pmc), M_PMC,
+ M_WAITOK | M_ZERO);
+ pmc_merge->pm_id = PMC_ID_MAKE_ID(a->pm_cpu, a->pm_mode,
+ a->pm_class, amd_core_pmc_baseri + ri + 1);
+ pmc_merge->pm_event = pm->pm_event;
+ pmc_merge->pm_state = PMC_STATE_ALLOCATED;
+ pmc_merge->pm_caps = pm->pm_caps;
+ pmc_merge->pm_flags = pm->pm_flags | PMC_F_MERGE_EVENT;
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel =
+ AMD_PMC_TO_EVENTMASK(AMD_PMC_MERGE_EVENT);
+ pmc_merge->pm_md.pm_amd.pmc_merge_parent = pm;
+ pm->pm_md.pm_amd.pmc_merge_event = pmc_merge;
+ amd_merge_reserve(ri, pm);
+ }
+
PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
ri, (uintmax_t)config);
return (0);
@@ -683,11 +900,11 @@
}
/*
- * Release machine dependent state associated with a PMC. This is a
- * no-op on this architecture.
+ * Release machine dependent state associated with a PMC. For a merge
+ * PMC, drop the odd counter's reservation and free its placeholder.
*/
static int
-amd_release_pmc(int cpu, int ri, struct pmc *pmc __unused)
+amd_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw __diagused;
@@ -701,6 +918,16 @@
KASSERT(phw->phw_pmc == NULL,
("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
+ /*
+ * The PMC has already been deconfigured on every CPU, so the
+ * placeholder is no longer in any slot and can be freed.
+ */
+ if (amd_pmc_is_merge(pmc)) {
+ amd_merge_unreserve(ri, pmc);
+ free(pmc->pm_md.pm_amd.pmc_merge_event, M_PMC);
+ pmc->pm_md.pm_amd.pmc_merge_event = NULL;
+ }
+
return (0);
}
@@ -708,10 +935,11 @@
* Start a PMC.
*/
static int
-amd_start_pmc(int cpu __diagused, int ri, struct pmc *pm)
+amd_start_pmc(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
- uint64_t config;
+ const struct amd_descr *pd, *pd_merge;
+ const struct pmc *pmc_merge;
+ uint64_t config, config_merge;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
@@ -719,6 +947,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
PMCDBG2(MDP, STA, 1, "amd-start cpu=%d ri=%d", cpu, ri);
@@ -744,17 +973,26 @@
PMCDBG1(MDP, STA, 2, "amd-start config=0x%x", config);
+ /* Enable the MergeEvent on the odd counter before the even one. */
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pmc_merge = pm->pm_md.pm_amd.pmc_merge_event;
+ config_merge =
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
+ wrmsr(pd_merge->pm_evsel, config_merge);
+ }
+
wrmsr(pd->pm_evsel, config);
return (0);
}
/* Start one PMC. pcd_start_all commits the virtual GLOBAL_CTL bits. */
static int
-amd_start_pmc_v2(int cpu __diagused, int ri, struct pmc *pm)
+amd_start_pmc_v2(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
+ const struct pmc *pmc_merge;
enum pmc_mode mode;
- uint64_t config;
+ uint64_t config, config_merge;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
@@ -762,6 +1000,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
mode = PMC_TO_MODE(pm);
PMCDBG2(MDP, STA, 1, "amd-start-v2 cpu=%d ri=%d", cpu, ri);
@@ -796,6 +1035,23 @@
DPCPU_SET(nmi_counter, 1);
}
+ /*
+ * Enable the MergeEvent on the odd counter before the even one.
+ * Its global bit follows the even counter's: staged for process
+ * mode, set directly for system mode.
+ */
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pmc_merge = pm->pm_md.pm_amd.pmc_merge_event;
+ config_merge =
+ pmc_merge->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
+ wrmsr(pd_merge->pm_evsel, config_merge);
+ if (PMC_IS_VIRTUAL_MODE(mode))
+ amd_v2_stage_virtual(cpu, ri + 1);
+ else
+ atomic_set_int(&amd_pcpu[cpu]->pc_global_mask,
+ amd_v2_counter_mask(ri + 1));
+ }
+
/* Enable EVSEL while the virtual slot's global bit is off. */
if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC)
config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_UMC_ENABLE;
@@ -817,9 +1073,9 @@
* Stop a PMC.
*/
static int
-amd_stop_pmc(int cpu __diagused, int ri, struct pmc *pm)
+amd_stop_pmc(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
uint64_t config;
int i;
@@ -829,6 +1085,7 @@
("[amd,%d] illegal row-index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri];
+ pd_merge = &amd_pmcdesc[ri + 1];
PMCDBG1(MDP, STO, 1, "amd-stop ri=%d", ri);
@@ -846,8 +1103,21 @@
config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE;
}
+
wrmsr(pd->pm_evsel, config);
+ if (amd_merge_installed(cpu, ri, pm)) {
+ /*
+ * Clear the odd counter after the even one is disabled, so
+ * no carry is lost. A leftover MergeEvent would make the
+ * next user of the even counter count in merged mode, and
+ * a leftover count would appear as the upper 16 bits of its
+ * value.
+ */
+ wrmsr(pd_merge->pm_evsel, 0);
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+
/*
* Due to NMI latency on newer AMD processors
* NMI interrupts are ignored, which leads to
@@ -867,9 +1137,9 @@
/* Stop one PMC. pcd_stop_all already zeroed the virtual GLOBAL_CTL bits. */
static int
-amd_stop_pmc_v2(int cpu __diagused, int ri, struct pmc *pm)
+amd_stop_pmc_v2(int cpu, int ri, struct pmc *pm)
{
- const struct amd_descr *pd;
+ const struct amd_descr *pd, *pd_merge;
enum pmc_mode mode;
uint64_t config;
int i;
@@ -891,6 +1161,21 @@
amd_v2_publish_mask(cpu);
}
+ if (amd_merge_installed(cpu, ri, pm)) {
+ pd_merge = &amd_pmcdesc[ri + 1];
+
+ atomic_clear_int(&amd_pcpu[cpu]->pc_global_mask,
+ amd_v2_counter_mask(ri + 1));
+ amd_v2_publish_mask(cpu);
+ /*
+ * Clear the odd counter's global bit, MergeEvent and count.
+ * The even counter's global bit is already off, so no carry
+ * is lost. See amd_stop_pmc() for why this is needed.
+ */
+ wrmsr(pd_merge->pm_evsel, 0);
+ wrmsr(pd_merge->pm_perfctr, 0);
+ }
+
/* Disable EVSEL after the counter's global bit is off. */
if (pd->pm_subclass == PMC_AMD_SUB_CLASS_UMC)
config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE;
@@ -958,7 +1243,8 @@
break;
if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
- !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
+ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) ||
+ (pm->pm_flags & PMC_F_MERGE_EVENT)) {
continue;
}
@@ -979,7 +1265,6 @@
v = pm->pm_sc.pm_reloadcount;
config = rdmsr(evsel);
-
if (amd_pmcdesc[i].pm_subclass == PMC_AMD_SUB_CLASS_UMC) {
KASSERT((config & ~AMD_PMC_UMC_ENABLE) ==
(pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_UMC_ENABLE),
@@ -996,6 +1281,14 @@
wrmsr(evsel, config & ~AMD_PMC_ENABLE);
}
wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
+ /*
+ * Preload the odd counter again so the merged counter
+ * overflows at the next even-counter wrap. Do it before
+ * logging: after a stall, the restart does not reload it.
+ */
+ if (amd_merge_installed(cpu, i, pm))
+ wrmsr(amd_pmcdesc[i + 1].pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
/* Restart the counter if logging succeeded. */
error = pmc_process_interrupt(PMC_HR, pm, tf);
@@ -1072,7 +1365,9 @@
*/
for (i = 0; i < amd_core_npmcs; i++) {
pm = pac->pc_amdpmcs[i].phw_pmc;
- if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
+ /* The merge placeholder is not a sampling PMC of its own. */
+ if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
+ !(pm->pm_flags & PMC_F_MERGE_EVENT))
active++;
}
@@ -1082,7 +1377,8 @@
pending &= ~(1ULL << i);
if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
- !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
+ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) ||
+ (pm->pm_flags & PMC_F_MERGE_EVENT)) {
continue;
}
@@ -1091,6 +1387,17 @@
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
+ /*
+ * Preload the odd counter again so the merged counter
+ * overflows at the next even-counter wrap. Without it, the
+ * pair would not interrupt again. A merged overflow sets
+ * only the even counter's GLOBAL_STATUS bit.
+ */
+ if (amd_merge_installed(cpu, i, pm)) {
+ wrmsr(amd_pmcdesc[i + 1].pm_perfctr,
+ AMD_PMC_MERGE_EVENT_PRELOAD_SAMPLING);
+ }
+
/* Reload the counter. */
v = pm->pm_sc.pm_reloadcount;
wrmsr(amd_pmcdesc[i].pm_perfctr,
@@ -1158,6 +1465,10 @@
if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) != 0) {
pi->pm_enabled = true;
*ppmc = phw->phw_pmc;
+ /* Report the merge PMC that owns this odd counter. */
+ if (*ppmc != NULL &&
+ ((*ppmc)->pm_flags & PMC_F_MERGE_EVENT) != 0)
+ *ppmc = phw->phw_pmc->pm_md.pm_amd.pmc_merge_parent;
} else {
pi->pm_enabled = false;
*ppmc = NULL;
@@ -1553,6 +1864,14 @@
model = CPUID_TO_MODEL(cpu_id);
stepping = CPUID_TO_STEPPING(cpu_id);
+ /*
+ * Counter merging for Large Increment per Cycle events is
+ * available from family 17h on.
+ */
+ if (family >= 0x17) {
+ amd_merge_event_supported = true;
+ }
+
if (family == 0x18)
snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X",
family, model, stepping);
@@ -1815,6 +2134,8 @@
pmc_mdep->pmd_npmc += amd_npmcs;
+ amd_core_pmc_baseri = pcd->pcd_ri;
+
amd_init_policy();
PMCDBG0(MDP, INI, 0, "amd-initialize");
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -2984,9 +2984,37 @@
int
pmc_getrowdisp(int ri)
{
+ sx_assert(&pmc_sx, SX_XLOCKED);
return (pmc_pmcdisp[ri]);
}
+/*
+ * Add or remove one mark on row 'ri' for a PMC of the given mode.
+ * For drivers whose PMCs also use another row, such as AMD merge
+ * events. The caller must hold pmc_sx exclusively.
+ */
+void
+pmc_markrowdisp(int ri, enum pmc_mode mode)
+{
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ if (PMC_IS_SYSTEM_MODE(mode))
+ PMC_MARK_ROW_STANDALONE(ri);
+ else
+ PMC_MARK_ROW_THREAD(ri);
+}
+
+void
+pmc_unmarkrowdisp(int ri, enum pmc_mode mode)
+{
+ sx_assert(&pmc_sx, SX_XLOCKED);
+
+ if (PMC_IS_SYSTEM_MODE(mode))
+ PMC_UNMARK_ROW_STANDALONE(ri);
+ else
+ PMC_UNMARK_ROW_THREAD(ri);
+}
+
/*
* Check if a PMC at row index 'ri' can be allocated to the current
* process.
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -386,6 +386,7 @@
#define PMC_F_ATTACHED_TO_OWNER 0x00010000 /*attached to owner*/
#define PMC_F_NEEDS_LOGFILE 0x00020000 /*needs log file */
#define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */
+#define PMC_F_MERGE_EVENT 0x00080000 /*merge pair placeholder */
#define PMC_CALLCHAIN_DEPTH_MAX 512
@@ -1262,6 +1263,8 @@
struct pmc_mdep *pmc_md_initialize(void); /* MD init function */
void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */
int pmc_getrowdisp(int _ri);
+void pmc_markrowdisp(int _ri, enum pmc_mode _mode);
+void pmc_unmarkrowdisp(int _ri, enum pmc_mode _mode);
int pmc_process_interrupt_mp(int _ring, struct pmc *_pm,
struct trapframe *_tf, struct pmc_multipart *mp);
int pmc_process_interrupt(int _ring, struct pmc *_pm,

File Metadata

Mime Type
text/plain
Expires
Fri, Nov 20, 6:02 AM (4 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41831385
Default Alt Text
D61053.1795154554.diff (21 KB)

Event Timeline