diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -35,6 +35,7 @@ #define CPUID_EXTPERFMON 0x80000022 #define EXTPERFMON_CORE_PMCS(x) ((x) & 0x0F) #define EXTPERFMON_DF_PMCS(x) (((x) >> 10) & 0x3F) +#define EXTPERFMON_PERFMONV2(x) ((x) & 0x1) /* leaf 0x80000022 EAX bit 0 */ /* AMD K8 PMCs */ #define AMD_PMC_EVSEL_0 0xC0010000 @@ -62,6 +63,15 @@ #define AMD_PMC_CORE_DEFAULT 6 #define AMD_PMC_CORE_MAX 16 +/* + * PerfMonV2 global control MSRs (Family 19h Zen3+ / Family 1Ah). + * These cover CORE counters only; L3/DF stay on the classic per-counter path. + */ +#define AMD_PMC_GLOBAL_STATUS 0xC0000300 /* RO */ +#define AMD_PMC_GLOBAL_CTL 0xC0000301 /* RW */ +#define AMD_PMC_GLOBAL_STATUS_CLR 0xC0000302 /* WO */ +/* GLOBAL_STATUS.LBRS_FROZEN (bit 58) goes here in the follow-up LBR v2 task. */ + #define AMD_PMC_COUNTERMASK 0xFF000000 #define AMD_PMC_PRECISERETIRE (1ULL << 43) /* Only valid for PERF_CTL2 */ #define AMD_PMC_HOST (1ULL << 41) diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -62,6 +62,8 @@ static int amd_npmcs; static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; +static bool amd_perfmon_v2; /* PerfMonV2 global-control path selected */ +static uint64_t amd_global_cntr_mask; /* (1 << amd_core_npmcs) - 1 */ static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ @@ -202,6 +204,10 @@ &amd_df_extra_mask, 0, "Extra allowed bits in AMD DF PMU control (override; default 0)"); +SYSCTL_BOOL(_kern_hwpmc, OID_AUTO, amd_perfmon_v2, CTLFLAG_RD, + &amd_perfmon_v2, 0, + "AMD PerfMonV2 global-control path selected (read-only)"); + static void amd_init_policy(void) { @@ -236,6 +242,19 @@ } } +/* PerfMonV2: write GLOBAL_CTL to start or stop all core counters at once. */ +static __inline void +amd_v2_enable_all(void) +{ + wrmsr(AMD_PMC_GLOBAL_CTL, amd_global_cntr_mask); +} + +static __inline void +amd_v2_disable_all(void) +{ + wrmsr(AMD_PMC_GLOBAL_CTL, 0); +} + /* * Read a PMC value from the MSR. */ @@ -535,6 +554,32 @@ return (0); } +/* Start a PMC (PerfMonV2 path): arm EVSEL, then assert the global counter mask. */ +static int +amd_start_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) +{ + const struct amd_descr *pd; + uint64_t config; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < amd_npmcs, + ("[amd,%d] illegal row-index %d", __LINE__, ri)); + + pd = &amd_pmcdesc[ri]; + + PMCDBG2(MDP, STA, 1, "amd-start-v2 cpu=%d ri=%d", cpu, ri); + + /* Arm the counter; GLOBAL_CTL gates when it actually counts. */ + config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; + wrmsr(pd->pm_evsel, config); + + /* Assert the global counter mask to (re)start core counters. */ + amd_v2_enable_all(); + + return (0); +} + /* * Stop a PMC. */ @@ -579,6 +624,39 @@ return (0); } +/* Stop a PMC (PerfMonV2 path): clear EVSEL ENABLE; wait out a pending overflow NMI via GLOBAL_STATUS. */ +static int +amd_stop_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) +{ + const struct amd_descr *pd; + int i; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < amd_npmcs, + ("[amd,%d] illegal row-index %d", __LINE__, ri)); + + pd = &amd_pmcdesc[ri]; + + PMCDBG1(MDP, STO, 1, "amd-stop-v2 ri=%d", ri); + + /* Disarm this counter's event; leaves other counters' GLOBAL_CTL bits. */ + wrmsr(pd->pm_evsel, pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); + + /* NMI latency: wait for an in-flight overflow NMI; the handler's ack clears the status bit. */ + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { + for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { + if ((rdmsr(AMD_PMC_GLOBAL_STATUS) & (1ULL << ri)) == 0) + break; + + DELAY(1); + } + } + + return (0); +} + /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed @@ -689,6 +767,94 @@ return (retval); } +/* Interrupt handler (PerfMonV2 path): freeze counters, read GLOBAL_STATUS once, reload, thaw. */ +static int +amd_intr_v2(struct trapframe *tf) +{ + struct amd_cpu *pac; + struct pmc *pm; + pmc_value_t v; + uint64_t status, mask; + uint32_t active = 0, count = 0; + int i, error, retval, cpu; + + cpu = curcpu; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] out of range CPU %d", __LINE__, cpu)); + + PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); + + retval = 0; + pac = amd_pcpu[cpu]; + + retval = pmc_ibs_intr(tf); + if (retval) + goto done; + + amd_v2_disable_all(); + + /* Single read of the overflow bitmap. */ + status = rdmsr(AMD_PMC_GLOBAL_STATUS); + status &= amd_global_cntr_mask; + + for (i = 0; i < amd_npmcs; i++) { + if (amd_pmcdesc[i].pm_subclass != PMC_AMD_SUB_CLASS_CORE) + break; + + if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || + !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { + continue; + } + + active++; + + mask = (1ULL << i); + if ((status & mask) == 0) + continue; + + retval = 1; + + if (pm->pm_state != PMC_STATE_RUNNING) + continue; + + /* Reload the counter. */ + v = pm->pm_sc.pm_reloadcount; + wrmsr(amd_pmcdesc[i].pm_perfctr, + AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); + + /* On log failure, leave the counter disarmed. MI restarts via pcd_start_pmc. */ + error = pmc_process_interrupt(PMC_HR, pm, tf); + if (error != 0) + wrmsr(amd_pmcdesc[i].pm_evsel, + pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); + } + + /* Ack overflow bits. */ + wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, status); + + /* Resume core counters. */ + amd_v2_enable_all(); + + /* NMI latency: an earlier NMI may have already handled the overflow; absorb the stray. */ + if (retval) { + DPCPU_SET(nmi_counter, min(2, active)); + } else { + if ((count = DPCPU_GET(nmi_counter))) { + retval = 1; + DPCPU_SET(nmi_counter, --count); + } + } + +done: + if (retval) + counter_u64_add(pmc_stats.pm_intr_processed, 1); + else + counter_u64_add(pmc_stats.pm_intr_ignored, 1); + + PMCDBG1(MDP, INT, 2, "retval=%d", retval); + return (retval); +} + /* * Describe a PMC. */ @@ -995,6 +1161,9 @@ amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); } + /* EAX bit 0 is the PerfMonV2 flag. */ + if (EXTPERFMON_PERFMONV2(regs[0]) && family >= 0x19) + amd_perfmon_v2 = true; } /* Enable the newer core counters */ @@ -1024,6 +1193,9 @@ } amd_npmcs = amd_core_npmcs; + if (amd_perfmon_v2) + amd_global_cntr_mask = (1ULL << amd_core_npmcs) - 1; + if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { /* Enable the LLC/L3 counters */ for (i = 0; i < amd_l3_npmcs; i++) { @@ -1118,6 +1290,13 @@ pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; + /* PerfMonV2: override start/stop/intr. L3 and DF keep the classic path. */ + if (amd_perfmon_v2) { + pcd->pcd_start_pmc = amd_start_pmc_v2; + pcd->pcd_stop_pmc = amd_stop_pmc_v2; + pmc_mdep->pmd_intr = amd_intr_v2; + } + pmc_mdep->pmd_npmc += amd_npmcs; amd_init_policy();