diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -35,6 +35,7 @@ #define CPUID_EXTPERFMON 0x80000022 #define EXTPERFMON_CORE_PMCS(x) ((x) & 0x0F) #define EXTPERFMON_DF_PMCS(x) (((x) >> 10) & 0x3F) +#define EXTPERFMON_PERFMONV2(x) ((x) & 0x1) /* EAX bit 0: PerfMonV2 flag */ /* AMD K8 PMCs */ #define AMD_PMC_EVSEL_0 0xC0010000 @@ -62,6 +63,16 @@ #define AMD_PMC_CORE_DEFAULT 6 #define AMD_PMC_CORE_MAX 16 +/* + * These are the PerfMonV2 global-control MSRs (Fam 19h Zen3+ / 1Ah). + * They control core counters only. L3 and DF counters keep the + * classic per-counter path. + */ +#define AMD_PMC_GLOBAL_STATUS 0xC0000300 /* RO */ +#define AMD_PMC_GLOBAL_CTL 0xC0000301 /* RW */ +#define AMD_PMC_GLOBAL_STATUS_CLR 0xC0000302 /* WO */ +/* A future LBR v2 change adds GLOBAL_STATUS.LBRS_FROZEN (bit 58) here. */ + #define AMD_PMC_COUNTERMASK 0xFF000000 #define AMD_PMC_PRECISERETIRE (1ULL << 43) /* Only valid for PERF_CTL2 */ #define AMD_PMC_HOST (1ULL << 41) diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -46,6 +46,7 @@ #define EXTERR_CATEGORY EXTERR_CAT_HWPMC_AMD #include +#include #include #include #include @@ -65,6 +66,8 @@ static int amd_npmcs; static int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; +static bool amd_perfmon_v2; /* Is the PerfMonV2 global-control path active? */ +static uint64_t amd_global_cntr_mask; /* Mask with one bit for each core counter */ static struct amd_descr amd_pmcdesc[AMD_NPMCS_MAX]; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ @@ -175,10 +178,20 @@ const int amd_event_codes_size = nitems(amd_event_codes); /* - * Per-processor information + * This struct holds per-processor information. + * + * pc_global_mask marks the core counters that must run on this CPU now. + * pc_virtual_mask marks per-process counters that wait to start at the + * next context switch. The switch clears this mask each time, so it + * never holds a system-wide counter. System counters and per-process + * counters always use different row indices. They never share one + * GLOBAL_CTL bit. */ struct amd_cpu { struct pmc_hw pc_amdpmcs[AMD_NPMCS_MAX]; + volatile u_int pc_global_mask; + volatile u_int pc_virtual_mask; + volatile u_int pc_gate_depth; }; static struct amd_cpu **amd_pcpu; @@ -205,6 +218,10 @@ &amd_df_extra_mask, 0, "Extra allowed bits in AMD DF PMU control (override; default 0)"); +SYSCTL_BOOL(_kern_hwpmc, OID_AUTO, amd_perfmon_v2, CTLFLAG_RD, + &amd_perfmon_v2, 0, + "AMD PerfMonV2 global-control path selected (read-only)"); + static void amd_init_policy(void) { @@ -239,6 +256,132 @@ } } +static __inline u_int +amd_v2_counter_mask(int ri) +{ + KASSERT(ri >= 0 && ri < amd_core_npmcs, + ("[amd,%d] illegal core row-index %d", __LINE__, ri)); + return (1U << ri); +} + +static __inline void +amd_v2_assert_mask(u_int mask) +{ + KASSERT((mask & ~(u_int)amd_global_cntr_mask) == 0, + ("[amd,%d] invalid GLOBAL_CTL mask %#x", __LINE__, mask)); +} + +static __inline void +amd_v2_publish_mask(int cpu) +{ + struct amd_cpu *pac; + u_int mask; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + pac = amd_pcpu[cpu]; + KASSERT(pac != NULL, + ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); + mask = atomic_load_acq_int(&pac->pc_global_mask); + amd_v2_assert_mask(mask); + if (atomic_load_acq_int(&pac->pc_gate_depth) == 0) + wrmsr(AMD_PMC_GLOBAL_CTL, mask); +} + +static void +amd_v2_stage_virtual(int cpu, int ri) +{ + struct amd_cpu *pac; + u_int mask; + + pac = amd_pcpu[cpu]; + mask = amd_v2_counter_mask(ri); + atomic_set_int(&pac->pc_virtual_mask, mask); +} + +static int +amd_start_pmc_all_v2(int cpu) +{ + struct amd_cpu *pac; + u_int mask; + + pac = amd_pcpu[cpu]; + mask = atomic_load_acq_int(&pac->pc_virtual_mask); + if (mask == 0) + return (0); + amd_v2_assert_mask(mask); + atomic_set_int(&pac->pc_global_mask, mask); + amd_v2_publish_mask(cpu); + return (0); +} + +/* Clear only the staged virtual bits. System-wide counters keep running. */ +static int +amd_stop_pmc_all_v2(int cpu) +{ + struct amd_cpu *pac; + u_int mask; + + pac = amd_pcpu[cpu]; + mask = atomic_readandclear_int(&pac->pc_virtual_mask); + if (mask == 0) + return (0); + amd_v2_assert_mask(mask); + atomic_clear_int(&pac->pc_global_mask, mask); + amd_v2_publish_mask(cpu); + return (0); +} + +static void +amd_v2_forget_core(int cpu, int ri, struct pmc *pm) +{ + struct amd_cpu *pac; + u_int mask; + + pac = amd_pcpu[cpu]; + mask = amd_v2_counter_mask(ri); + if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) + atomic_clear_int(&pac->pc_virtual_mask, mask); + if ((atomic_load_acq_int(&pac->pc_global_mask) & mask) != 0) { + atomic_clear_int(&pac->pc_global_mask, mask); + amd_v2_publish_mask(cpu); + } +} + +static __inline void +amd_v2_disable_all(void) +{ + wrmsr(AMD_PMC_GLOBAL_CTL, 0); +} + +static void +amd_v2_freeze_core(int cpu) +{ + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + atomic_add_int(&amd_pcpu[cpu]->pc_gate_depth, 1); + amd_v2_disable_all(); +} + +static void +amd_v2_thaw_core(int cpu) +{ + u_int old; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + old = atomic_fetchadd_int(&amd_pcpu[cpu]->pc_gate_depth, -1); + KASSERT(old > 0, + ("AMD PMC gate-depth underflow on CPU %d", cpu)); + if (old == 1) { + u_int mask; + + mask = atomic_load_acq_int(&amd_pcpu[cpu]->pc_global_mask); + amd_v2_assert_mask(mask); + wrmsr(AMD_PMC_GLOBAL_CTL, mask); + } +} + /* * Read a PMC value from the MSR. */ @@ -334,6 +477,10 @@ ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); + if (amd_perfmon_v2 && pm == NULL && phw->phw_pmc != NULL && + amd_pmcdesc[ri].pm_subclass == PMC_AMD_SUB_CLASS_CORE) + amd_v2_forget_core(cpu, ri, phw->phw_pmc); + phw->phw_pmc = pm; return (0); } @@ -383,11 +530,10 @@ } /* - * Check if a given PMC allocation is feasible. + * Check if an event can use row index 'ri'. */ static int -amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, - const struct pmc_op_pmcallocate *a) +amd_can_assign_pmc(int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { const struct pmc_descr *pd; uint64_t allowed_unitmask, caps, config, unitmask; @@ -399,14 +545,11 @@ pd = &amd_pmcdesc[ri].pm_descr; - /* check class match */ if (pd->pd_class != a->pm_class) return (EINVAL); caps = pm->pm_caps; - PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); - /* Validate sub-class. */ if (amd_pmcdesc[ri].pm_subclass != a->pm_md.pm_amd.pm_amd_sub_class) return (EINVAL); @@ -424,18 +567,10 @@ "AMD PMU config has unsupported bits %#jx", (uintmax_t)(config & ~amd_config_mask( amd_pmcdesc[ri].pm_subclass, caps)))); - pm->pm_md.pm_amd.pm_amd_evsel = config; - PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx", - ri, (uintmax_t)config); return (0); } - /* - * Everything below this is for supporting older processors. - */ pe = a->pm_ev; - - /* map ev to the correct event mask code */ config = allowed_unitmask = 0; for (i = 0; i < amd_event_codes_size; i++) { if (amd_event_codes[i].pe_ev == pe) { @@ -457,13 +592,55 @@ "AMD unitmask %#jx exceeds allowed mask %#jx", (uintmax_t)unitmask, (uintmax_t)allowed_unitmask)); + return (0); +} + +/* + * Check if a given PMC allocation is feasible. + */ +static int +amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + uint64_t allowed_unitmask, caps, config, unitmask; + enum pmc_event pe; + int error, i; + + error = amd_can_assign_pmc(ri, pm, a); + if (error != 0) + return (error); + + caps = pm->pm_caps; + PMCDBG2(MDP, ALL, 1,"amd-allocate ri=%d caps=0x%x", ri, caps); + + /* PMC_F_EV_PMU: config comes from pmu-events tables. */ + if ((a->pm_flags & PMC_F_EV_PMU) != 0) { + config = a->pm_md.pm_amd.pm_amd_config; + pm->pm_md.pm_amd.pm_amd_evsel = config; + PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx", + ri, (uintmax_t)config); + return (0); + } + + pe = a->pm_ev; + config = allowed_unitmask = 0; + for (i = 0; i < amd_event_codes_size; i++) { + if (amd_event_codes[i].pe_ev == pe) { + config = + AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); + allowed_unitmask = + AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); + break; + } + } + + unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; if (unitmask && (caps & PMC_CAP_QUALIFIER) != 0) config |= unitmask; if ((caps & PMC_CAP_THRESHOLD) != 0) config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; - /* Set at least one of the 'usr' or 'os' caps. */ if ((caps & PMC_CAP_USER) != 0) config |= AMD_PMC_USR; if ((caps & PMC_CAP_SYSTEM) != 0) @@ -478,7 +655,7 @@ if ((caps & PMC_CAP_INTERRUPT) != 0) config |= AMD_PMC_INT; - pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ + pm->pm_md.pm_amd.pm_amd_evsel = config; PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%x", ri, config); @@ -543,6 +720,68 @@ return (0); } +/* Start one PMC. pcd_start_all commits the virtual GLOBAL_CTL bits. */ +static int +amd_start_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) +{ + const struct amd_descr *pd; + enum pmc_mode mode; + uint64_t config; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < amd_npmcs, + ("[amd,%d] illegal row-index %d", __LINE__, ri)); + + pd = &amd_pmcdesc[ri]; + mode = PMC_TO_MODE(pm); + + PMCDBG2(MDP, STA, 1, "amd-start-v2 cpu=%d ri=%d", cpu, ri); + + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_VIRTUAL_MODE(mode)) + amd_v2_stage_virtual(cpu, ri); + + /* + * Drop a stale overflow status bit before the row goes live. + * amd_stop_pmc_v2() waits only OVERFLOW_WAIT_COUNT microseconds + * for the old occupant's in-flight NMI. Group rotation reuses + * rows often, so a surviving GLOBAL_STATUS bit would reach the + * late handler after the row restarts. The handler would then + * blame this PMC: a false sample, plus a mid-window reload for + * the wrong event. With the bit clear, the late handler finds + * no work and treats the NMI as spurious. Legacy AMD has no + * status plane to check, so it only relies on the bounded wait. + */ + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_SAMPLING_MODE(mode) && + (rdmsr(AMD_PMC_GLOBAL_STATUS) & (1ULL << ri)) != 0) { + wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, 1ULL << ri); + + /* + * That NMI may still be in flight. No status bit is left + * to claim it, so amd_intr_v2() would return 0 and the NMI + * would go unhandled (machdep.panic_on_nmi). Give the stray + * absorber one credit on this CPU. + */ + if (DPCPU_GET(nmi_counter) == 0) + DPCPU_SET(nmi_counter, 1); + } + + /* Enable EVSEL while the virtual slot's global bit is off. */ + config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; + wrmsr(pd->pm_evsel, config); + + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_SYSTEM_MODE(mode)) { + atomic_set_int(&amd_pcpu[cpu]->pc_global_mask, + amd_v2_counter_mask(ri)); + amd_v2_publish_mask(cpu); + } + + return (0); +} + /* * Stop a PMC. */ @@ -587,6 +826,49 @@ return (0); } +/* Stop one PMC. pcd_stop_all already zeroed the virtual GLOBAL_CTL bits. */ +static int +amd_stop_pmc_v2(int cpu __diagused, int ri, struct pmc *pm) +{ + const struct amd_descr *pd; + enum pmc_mode mode; + int i; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < amd_npmcs, + ("[amd,%d] illegal row-index %d", __LINE__, ri)); + + pd = &amd_pmcdesc[ri]; + mode = PMC_TO_MODE(pm); + + PMCDBG1(MDP, STO, 1, "amd-stop-v2 ri=%d", ri); + + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_SYSTEM_MODE(mode)) { + atomic_clear_int(&amd_pcpu[cpu]->pc_global_mask, + amd_v2_counter_mask(ri)); + amd_v2_publish_mask(cpu); + } + + /* Disable EVSEL after the counter's global bit is off. */ + wrmsr(pd->pm_evsel, + pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); + + /* Wait out an in-flight overflow NMI. The handler clears the status bit. */ + if (pd->pm_subclass == PMC_AMD_SUB_CLASS_CORE && + PMC_IS_SAMPLING_MODE(mode)) { + for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { + if ((rdmsr(AMD_PMC_GLOBAL_STATUS) & (1ULL << ri)) == 0) + break; + + DELAY(1); + } + } + + return (0); +} + /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed @@ -697,6 +979,100 @@ return (retval); } +/* This is the v2 interrupt handler: freeze counters, read GLOBAL_STATUS once, reload, thaw. */ +static int +amd_intr_v2(struct trapframe *tf) +{ + struct amd_cpu *pac; + struct pmc *pm; + pmc_value_t v; + uint64_t status, pending; + uint32_t active = 0, count = 0; + int i, error, retval, cpu; + + cpu = curcpu; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[amd,%d] out of range CPU %d", __LINE__, cpu)); + + PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); + + retval = 0; + pac = amd_pcpu[cpu]; + + retval = pmc_ibs_intr(tf); + if (retval) + goto done; + + amd_v2_freeze_core(cpu); + + /* Read the overflow bitmap once. */ + status = rdmsr(AMD_PMC_GLOBAL_STATUS); + status &= amd_global_cntr_mask; + + /* + * Count all active sampling PMCs, not just the ones that + * overflowed. Count the in-flight NMI too, its counter has + * not overflowed yet. + */ + for (i = 0; i < amd_core_npmcs; i++) { + pm = pac->pc_amdpmcs[i].phw_pmc; + if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + active++; + } + + /* ffsll() returns a 1-based bit index, or 0 when no bits are set. */ + pending = status; + while ((i = ffsll(pending) - 1) != -1) { + pending &= ~(1ULL << i); + + if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || + !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { + continue; + } + + retval = 1; + + if (pm->pm_state != PMC_STATE_RUNNING) + continue; + + /* Reload the counter. */ + v = pm->pm_sc.pm_reloadcount; + wrmsr(amd_pmcdesc[i].pm_perfctr, + AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); + + /* On a log failure, leave the PMC disabled. MI code restarts it via pcd_start_pmc. */ + error = pmc_process_interrupt(PMC_HR, pm, tf); + if (error != 0) + wrmsr(amd_pmcdesc[i].pm_evsel, + pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE); + } + + /* Acknowledge the overflow bits through GLOBAL_STATUS_CLR. */ + wrmsr(AMD_PMC_GLOBAL_STATUS_CLR, status); + + /* Thaw the core counters. */ + amd_v2_thaw_core(cpu); + + /* An earlier NMI may have serviced this overflow. Absorb the stray NMI. */ + if (retval) { + DPCPU_SET(nmi_counter, min(2, active)); + } else { + if ((count = DPCPU_GET(nmi_counter))) { + retval = 1; + DPCPU_SET(nmi_counter, --count); + } + } + +done: + if (retval) + counter_u64_add(pmc_stats.pm_intr_processed, 1); + else + counter_u64_add(pmc_stats.pm_intr_ignored, 1); + + PMCDBG1(MDP, INT, 2, "retval=%d", retval); + return (retval); +} + /* * Describe a PMC. */ @@ -804,6 +1180,18 @@ amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, M_WAITOK | M_ZERO); + if (amd_perfmon_v2) { + KASSERT(atomic_load_acq_int(&pac->pc_gate_depth) == 0, + ("[amd,%d] nonzero initial gate depth on CPU %d", + __LINE__, cpu)); + KASSERT(atomic_load_acq_int(&pac->pc_global_mask) == 0, + ("[amd,%d] nonzero initial desired mask on CPU %d", + __LINE__, cpu)); + KASSERT(atomic_load_acq_int(&pac->pc_virtual_mask) == 0, + ("[amd,%d] nonzero initial virtual mask on CPU %d", + __LINE__, cpu)); + amd_v2_disable_all(); + } /* * Set the content of the hardware descriptors to a known @@ -845,6 +1233,15 @@ if ((pac = amd_pcpu[cpu]) == NULL) return (0); + if (amd_perfmon_v2) { + KASSERT(atomic_load_acq_int(&pac->pc_gate_depth) == 0, + ("[amd,%d] nonzero gate depth on CPU %d", __LINE__, cpu)); + KASSERT(atomic_load_acq_int(&pac->pc_global_mask) == 0, + ("[amd,%d] nonzero desired mask on CPU %d", __LINE__, cpu)); + KASSERT(atomic_load_acq_int(&pac->pc_virtual_mask) == 0, + ("[amd,%d] nonzero virtual mask on CPU %d", __LINE__, cpu)); + amd_v2_disable_all(); + } amd_pcpu[cpu] = NULL; #ifdef HWPMC_DEBUG @@ -871,6 +1268,30 @@ return (0); } +struct amd_v2_hwcheck_state { + volatile u_int avh_read_error; + volatile u_int avh_enabled; +}; + +static void +amd_v2_hwcheck_cpu(void *arg) +{ + struct amd_v2_hwcheck_state *state; + uint64_t reg; + int error, i; + + state = arg; + for (i = 0; i < amd_core_npmcs; i++) { + error = rdmsr_safe(amd_pmcdesc[i].pm_evsel, ®); + if (error != 0) { + atomic_set_int(&state->avh_read_error, 1); + continue; + } + if ((reg & AMD_PMC_ENABLE) != 0) + atomic_set_int(&state->avh_enabled, 1); + } +} + /* * Check that the PMC hardware is safe to use. First, we check that the PMCs * are not in use by firmware or another module. Second, if none of the PMC @@ -880,9 +1301,29 @@ static int amd_hwcheck(void) { + struct amd_v2_hwcheck_state state; uint64_t reg; int error, i; + if (amd_perfmon_v2) { + state.avh_read_error = 0; + state.avh_enabled = 0; + smp_rendezvous_cpus(all_cpus, smp_no_rendezvous_barrier, + amd_v2_hwcheck_cpu, smp_no_rendezvous_barrier, &state); + if (state.avh_read_error != 0) { + printf("hwpmc: AMD PerfMonV2 EVSEL read failed on one " + "or more CPUs!\n"); + return (-1); + } + if (state.avh_enabled != 0) { + printf("hwpmc: PMCs maybe in use by firmware!\n"); + printf("hwpmc: Disable the PMC use in the BIOS before " + "loading\n"); + return (-1); + } + return (0); + } + /* * Some PC vendors enable the core counters in firmware to track * performance. The best guess is that this is being used to control @@ -1003,6 +1444,9 @@ amd_core_npmcs = EXTPERFMON_CORE_PMCS(regs[1]); amd_df_npmcs = EXTPERFMON_DF_PMCS(regs[1]); } + /* EAX bit 0 holds the PerfMonV2 flag. */ + if (EXTPERFMON_PERFMONV2(regs[0]) && family >= 0x19) + amd_perfmon_v2 = true; } /* Enable the newer core counters */ @@ -1032,6 +1476,9 @@ } amd_npmcs = amd_core_npmcs; + if (amd_perfmon_v2) + amd_global_cntr_mask = (1ULL << amd_core_npmcs) - 1; + if ((amd_feature2 & AMDID2_PTSCEL2I) != 0) { /* Enable the LLC/L3 counters */ for (i = 0; i < amd_l3_npmcs; i++) { @@ -1127,6 +1574,15 @@ pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; + /* For v2, override core control. L3 and DF keep the classic path. */ + if (amd_perfmon_v2) { + pcd->pcd_start_pmc = amd_start_pmc_v2; + pcd->pcd_stop_pmc = amd_stop_pmc_v2; + pcd->pcd_start_all = amd_start_pmc_all_v2; + pcd->pcd_stop_all = amd_stop_pmc_all_v2; + pmc_mdep->pmd_intr = amd_intr_v2; + } + pmc_mdep->pmd_npmc += amd_npmcs; amd_init_policy(); diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -255,6 +255,9 @@ static void pmc_process_fork(void *arg, struct proc *p1, struct proc *p2, int n); static void pmc_process_proccreate(struct proc *p); +static void pmc_process_csw_start_all(int cpu); +static void pmc_process_csw_stop_all(int cpu); +static void pmc_process_csw_out_prepare(int cpu); static void pmc_process_samples(int cpu, ring_type_t soft); static void pmc_process_threadcreate(struct thread *td); static void pmc_process_threadexit(struct thread *td); @@ -1472,6 +1475,63 @@ } } +/* + * Execute optional context-switch batch operations for all classes. + */ +static void +pmc_process_csw_start_all(int cpu) +{ + struct pmc_classdep *pcd; + u_int class; + + for (class = 0; class < md->pmd_nclass; class++) { + pcd = &md->pmd_classdep[class]; + if (pcd->pcd_start_all != NULL) + (void)pcd->pcd_start_all(cpu); + } +} + +static void +pmc_process_csw_stop_all(int cpu) +{ + struct pmc_classdep *pcd; + u_int class; + + pmc_process_csw_out_prepare(cpu); + + for (class = 0; class < md->pmd_nclass; class++) { + pcd = &md->pmd_classdep[class]; + if (pcd->pcd_stop_all != NULL) + (void)pcd->pcd_stop_all(cpu); + } +} + +/* + * Mark virtual PMCs stopped, before you close hardware gates. + */ +static void +pmc_process_csw_out_prepare(int cpu) +{ + struct pmc *pm; + struct pmc_classdep *pcd; + u_int class; + int adjri; + + for (class = 0; class < md->pmd_nclass; class++) { + pcd = &md->pmd_classdep[class]; + if (pcd->pcd_stop_all == NULL) + continue; + for (adjri = 0; adjri < pcd->pcd_num; adjri++) { + pm = NULL; + (void)pcd->pcd_get_config(cpu, adjri, &pm); + if (pm == NULL || + !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) + continue; + pm->pm_pcpu_state[cpu].pps_cpustate = 0; + } + } +} + /* * Thread context switch IN. */ @@ -1627,6 +1687,9 @@ */ (void)(*md->pmd_switch_in)(pc, pp); + /* Commit all class PMC start updates at one boundary. */ + pmc_process_csw_start_all(cpu); + critical_exit(); } @@ -1748,6 +1811,9 @@ pc = pmc_pcpu[cpu]; + /* Close shared class gates before any PMC stop or read. */ + pmc_process_csw_stop_all(cpu); + /* * When a PMC gets unlinked from a target PMC, it will * be removed from the target's pp_pmc[] array. @@ -5278,6 +5344,9 @@ PMCDBG2(PRC,EXT,2, "process-exit proc=%p pmc-process=%p", p, pp); + /* Run the context-switch-out steps for process exit. */ + pmc_process_csw_stop_all(cpu); + /* * The exiting process could be the target of some PMCs which will be * running on currently executing CPU. diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h --- a/sys/sys/pmc.h +++ b/sys/sys/pmc.h @@ -1053,6 +1053,11 @@ /* starting and stopping PMCs */ int (*pcd_start_pmc)(int _cpu, int _ri, struct pmc *_pm); int (*pcd_stop_pmc)(int _cpu, int _ri, struct pmc *_pm); + /* + * Optional context-switch batch operations. + */ + int (*pcd_start_all)(int _cpu); + int (*pcd_stop_all)(int _cpu); /* description */ int (*pcd_describe)(int _cpu, int _ri, struct pmc_info *_pi,