diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index 1b3c59163141..a2607ca1d9e3 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -1,1323 +1,1324 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Core PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #define CORE_CPUID_REQUEST 0xA #define CORE_CPUID_REQUEST_SIZE 0x4 #define CORE_CPUID_EAX 0x0 #define CORE_CPUID_EBX 0x1 #define CORE_CPUID_ECX 0x2 #define CORE_CPUID_EDX 0x3 #define IAF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30)) #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define EV_IS_NOTARCH 0 #define EV_IS_ARCH_SUPP 1 #define EV_IS_ARCH_NOTSUPP -1 /* * "Architectural" events defined by Intel. The values of these * symbols correspond to positions in the bitmask returned by * the CPUID.0AH instruction. */ enum core_arch_events { CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5, CORE_AE_BRANCH_MISSES_RETIRED = 6, CORE_AE_INSTRUCTION_RETIRED = 1, CORE_AE_LLC_MISSES = 4, CORE_AE_LLC_REFERENCE = 3, CORE_AE_UNHALTED_REFERENCE_CYCLES = 2, CORE_AE_UNHALTED_CORE_CYCLES = 0 }; static enum pmc_cputype core_cputype; static int core_version; struct core_cpu { volatile uint32_t pc_iafctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_corepmcs[]; }; static struct core_cpu **core_pcpu; static uint32_t core_architectural_events; static uint64_t core_pmcmask; static int core_iaf_ri; /* relative index of fixed counters */ static int core_iaf_width; static int core_iaf_npmc; static int core_iap_width; static int core_iap_npmc; static int core_iap_wroffset; static u_int pmc_alloc_refs; static bool pmc_tsx_force_abort_set; static int core_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int core_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct core_cpu *cc; struct pmc_hw *phw; int core_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; if (core_version >= 2) npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); core_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + core_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + core_ri] = phw; } if (core_version >= 2) { /* Enable Freezing PMCs on PMI. */ wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000); } return (0); } static int core_pcpu_fini(struct pmc_mdep *md, int cpu) { int core_ri, n, npmc; struct pmc_cpu *pc; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); if ((cc = core_pcpu[cpu]) == NULL) return (0); core_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(IAP_EVSEL0 + n, 0); if (core_version >= 2) { wrmsr(IAF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; } for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + core_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t iaf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iaf_width - 1))) == 0) return (0); v &= (1ULL << core_iaf_width) - 1; return (1ULL << core_iaf_width) - v; } static pmc_value_t iaf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iaf_width) - rlc; } static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev, umask; - uint32_t caps, flags, config; + uint32_t caps; + uint64_t config, flags; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > core_iaf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_IAF) return (EINVAL); iap = &a->pm_md.pm_iap; config = iap->pm_iap_config; ev = IAP_EVSEL_GET(config); umask = IAP_UMASK_GET(config); if (ev == 0x0) { if (umask != ri + 1) return (EINVAL); } else { switch (ri) { case 0: /* INST_RETIRED.ANY */ if (ev != 0xC0 || umask != 0x00) return (EINVAL); break; case 1: /* CPU_CLK_UNHALTED.THREAD */ if (ev != 0x3C || umask != 0x00) return (EINVAL); break; case 2: /* CPU_CLK_UNHALTED.REF */ if (ev != 0x3C || umask != 0x01) return (EINVAL); break; case 3: /* TOPDOWN.SLOTS */ if (ev != 0xA4 || umask != 0x01) return (EINVAL); break; default: return (EINVAL); } } pmc_alloc_refs++; if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 && !pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = true; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 1, NULL); } flags = 0; if (config & IAP_OS) flags |= IAF_OS; if (config & IAP_USR) flags |= IAF_USR; if (config & IAP_ANY) flags |= IAF_ANY; if (config & IAP_INT) flags |= IAF_PMI; caps = a->pm_caps; if (caps & PMC_CAP_INTERRUPT) flags |= IAF_PMI; if (caps & PMC_CAP_SYSTEM) flags |= IAF_OS; if (caps & PMC_CAP_USER) flags |= IAF_USR; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) flags |= (IAF_OS | IAF_USR); pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); return (0); } static int iaf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; return (0); } static int iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iaf_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri]; (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri); if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iaf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; return (0); } static int iaf_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[iaf,%d] ri %d out of range", __LINE__, ri)); *msr = IAF_RI_TO_MSR(ri); return (0); } static int iaf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + core_iaf_ri)); tmp = rdpmc(IAF_RI_TO_MSR(ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iaf_width) - 1); PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); return (0); } static int iaf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); MPASS(pmc_alloc_refs > 0); if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = false; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 0, NULL); } return (0); } static int iaf_start_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; wrmsr(IAF_CTRL, cc->pc_iafctrl); cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_stop_pmc(int cpu, int ri) { struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4)); wrmsr(IAF_CTRL, cc->pc_iafctrl); cc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET)); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_write_pmc(int cpu, int ri, pmc_value_t v) { struct core_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iaf_reload_count_to_perfctr_value(v); /* Turn off the fixed counter */ wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4))); wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); /* Turn on fixed counters */ wrmsr(IAF_CTRL, cc->pc_iafctrl); PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); return (0); } static void iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iaf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; pcd->pcd_caps = IAF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iaf_allocate_pmc; pcd->pcd_config_pmc = iaf_config_pmc; pcd->pcd_describe = iaf_describe; pcd->pcd_get_config = iaf_get_config; pcd->pcd_get_msr = iaf_get_msr; pcd->pcd_pcpu_fini = core_pcpu_noop; pcd->pcd_pcpu_init = core_pcpu_noop; pcd->pcd_read_pmc = iaf_read_pmc; pcd->pcd_release_pmc = iaf_release_pmc; pcd->pcd_start_pmc = iaf_start_pmc; pcd->pcd_stop_pmc = iaf_stop_pmc; pcd->pcd_write_pmc = iaf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* Sub fields of UMASK that this event supports. */ #define IAP_M_CORE (1 << 0) /* Core specificity */ #define IAP_M_AGENT (1 << 1) /* Agent specificity */ #define IAP_M_PREFETCH (1 << 2) /* Prefetch */ #define IAP_M_MESI (1 << 3) /* MESI */ #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */ #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */ #define IAP_M_TRANSITION (1 << 6) /* Transition */ #define IAP_F_CORE (0x3 << 14) /* Core specificity */ #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */ #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */ #define IAP_F_MESI (0xF << 8) /* MESI */ #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */ #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */ #define IAP_F_TRANSITION (0x1 << 12) /* Transition */ #define IAP_PREFETCH_RESERVED (0x2 << 12) #define IAP_CORE_THIS (0x1 << 14) #define IAP_CORE_ALL (0x3 << 14) #define IAP_F_CMASK 0xFF000000 static pmc_value_t iap_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iap_width - 1))) == 0) return (0); v &= (1ULL << core_iap_width) - 1; return (1ULL << core_iap_width) - v; } static pmc_value_t iap_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iap_width) - rlc; } static int iap_pmc_has_overflowed(int ri) { uint64_t v; /* * We treat a Core (i.e., Intel architecture v1) PMC as has * having overflowed if its MSB is zero. */ v = rdpmc(ri); return ((v & (1ULL << (core_iap_width - 1))) == 0); } static int iap_event_corei7_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0, 1. */ case 0x40: case 0x41: case 0x42: case 0x43: case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x53: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_westmere_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0x60: case 0xB3: mask = 0x1; break; /* Events valid only on counter 0, 1. */ case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0xB7: mask = 0x1; break; /* Events valid only on counter 1. */ case 0xC0: mask = 0x2; break; /* Events valid only on counter 2. */ case 0x48: case 0xA2: case 0xA3: mask = 0x4; break; /* Events valid only on counter 3. */ case 0xBB: case 0xCD: mask = 0x8; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_core_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0. */ case 0x10: case 0x14: case 0x18: case 0xB3: case 0xC1: case 0xCB: mask = (1 << 0); break; /* * Events valid only on counter 1. */ case 0x11: case 0x12: case 0x13: mask = (1 << 1); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_IAP) return (EINVAL); iap = &a->pm_md.pm_iap; ev = IAP_EVSEL_GET(iap->pm_iap_config); switch (core_cputype) { case PMC_CPU_INTEL_CORE: case PMC_CPU_INTEL_CORE2: case PMC_CPU_INTEL_CORE2EXTREME: if (iap_event_core_ok_on_counter(ev, ri) == 0) return (EINVAL); case PMC_CPU_INTEL_COREI7: case PMC_CPU_INTEL_NEHALEM_EX: if (iap_event_corei7_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_WESTMERE: case PMC_CPU_INTEL_WESTMERE_EX: if (iap_event_westmere_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_SANDYBRIDGE: case PMC_CPU_INTEL_SANDYBRIDGE_XEON: case PMC_CPU_INTEL_IVYBRIDGE: case PMC_CPU_INTEL_IVYBRIDGE_XEON: case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_HASWELL_XEON: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_ATOM: case PMC_CPU_INTEL_ATOM_SILVERMONT: case PMC_CPU_INTEL_ATOM_GOLDMONT: case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_ICELAKE: case PMC_CPU_INTEL_ICELAKE_XEON: case PMC_CPU_INTEL_ALDERLAKE: default: break; } pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config; return (0); } static int iap_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; return (0); } static int iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iap_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri]; (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri); if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iap_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; return (0); } static int iap_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iap_npmc, ("[iap,%d] ri %d out of range", __LINE__, ri)); *msr = ri; return (0); } static int iap_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdpmc(ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iap_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iap_width) - 1); PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAP_PMC0 + ri, *v); return (0); } static int iap_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iap_start_pmc(int cpu, int ri) { struct pmc *pm; - uint32_t evsel; + uint64_t evsel; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_iap.pm_iap_evsel; PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, IAP_EVSEL0 + ri, evsel); /* Event specific configuration. */ switch (IAP_EVSEL_GET(evsel)) { case 0xB7: wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); break; case 0xBB: wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp); break; default: break; } wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN); if (core_version >= 2) { cc->pc_globalctrl |= (1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } return (0); } static int iap_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); wrmsr(IAP_EVSEL0 + ri, 0); if (core_version >= 2) { cc->pc_globalctrl &= ~(1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } return (0); } static int iap_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iap_reload_count_to_perfctr_value(v); v &= (1ULL << core_iap_width) - 1; PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, IAP_PMC0 + ri, v); /* * Write the new value to the counter (or it's alias). The * counter will be in a stopped state when the pcd_write() * entry point is called. */ wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); return (0); } static void iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, int flags) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iap-initialize"); /* Remember the set of architectural events supported. */ core_architectural_events = ~flags; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; pcd->pcd_caps = IAP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iap_allocate_pmc; pcd->pcd_config_pmc = iap_config_pmc; pcd->pcd_describe = iap_describe; pcd->pcd_get_config = iap_get_config; pcd->pcd_get_msr = iap_get_msr; pcd->pcd_pcpu_fini = core_pcpu_fini; pcd->pcd_pcpu_init = core_pcpu_init; pcd->pcd_read_pmc = iap_read_pmc; pcd->pcd_release_pmc = iap_release_pmc; pcd->pcd_start_pmc = iap_start_pmc; pcd->pcd_stop_pmc = iap_stop_pmc; pcd->pcd_write_pmc = iap_write_pmc; md->pmd_npmc += npmc; } static int core_intr(struct trapframe *tf) { pmc_value_t v; struct pmc *pm; struct core_cpu *cc; int error, found_interrupt, ri; PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf, TRAPF_USERMODE(tf)); found_interrupt = 0; cc = core_pcpu[curcpu]; for (ri = 0; ri < core_iap_npmc; ri++) { if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (!iap_pmc_has_overflowed(ri)) continue; found_interrupt = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); v = pm->pm_sc.pm_reloadcount; v = iap_reload_count_to_perfctr_value(v); /* * Stop the counter, reload it but only restart it if * the PMC is not stalled. */ wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel); wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); if (__predict_false(error)) continue; wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); if (found_interrupt) lapic_reenable_pmc(); return (found_interrupt); } static int core2_intr(struct trapframe *tf) { int error, found_interrupt = 0, n, cpu; uint64_t flag, intrstatus, intrdisable = 0; struct pmc *pm; struct core_cpu *cc; pmc_value_t v; cpu = curcpu; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); /* * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which * PMCs have a pending PMI interrupt. We take a 'snapshot' of * the current set of interrupting PMCs and process these * after stopping them. */ intrstatus = rdmsr(IA_GLOBAL_STATUS); PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, (uintmax_t) intrstatus); /* * Stop PMCs unless hardware already done it. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) wrmsr(IA_GLOBAL_CTRL, 0); cc = core_pcpu[cpu]; KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); /* * Look for interrupts from fixed function PMCs. */ for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); /* Reload sampling count. */ wrmsr(IAF_CTR0 + n, v); PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu, error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); } /* * Process interrupts from the programmable counters. */ for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, (uintmax_t) v); /* Reload sampling count. */ wrmsr(core_iap_wroffset + IAP_PMC0 + n, v); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); /* * Reenable all non-stalled PMCs. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) { wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } else { if (__predict_false(intrdisable)) { cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); } PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_STATUS)); if (found_interrupt) lapic_reenable_pmc(); return (found_interrupt); } int pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) { int cpuid[CORE_CPUID_REQUEST_SIZE]; int flags, nflags; do_cpuid(CORE_CPUID_REQUEST, cpuid); core_cputype = md->pmd_cputype; core_version = (version_override > 0) ? version_override : cpuid[CORE_CPUID_EAX] & 0xFF; PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d", core_cputype, maxcpu, core_version); if (core_version < 1 || core_version > 5 || (core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) { /* Unknown PMC architecture. */ printf("hwpc_core: unknown PMC architecture: %d\n", core_version); return (EPROGMISMATCH); } core_iap_wroffset = 0; if (cpu_feature2 & CPUID2_PDCM) { if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) { PMCDBG0(MDP, INI, 1, "core-init full-width write supported"); core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0; } else PMCDBG0(MDP, INI, 1, "core-init full-width write NOT supported"); } else PMCDBG0(MDP, INI, 1, "core-init pdcm not supported"); core_pmcmask = 0; /* * Initialize programmable counters. */ core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF; core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF; core_pmcmask |= ((1ULL << core_iap_npmc) - 1); nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF; flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1); iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags); /* * Initialize fixed function counters, if present. */ if (core_version >= 2) { core_iaf_ri = core_iap_npmc; core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; } PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, core_iaf_ri); core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); /* * Choose the appropriate interrupt handler. */ if (core_version >= 2) md->pmd_intr = core2_intr; else md->pmd_intr = core_intr; md->pmd_pcpu_fini = NULL; md->pmd_pcpu_init = NULL; return (0); } void pmc_core_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "core-finalize"); free(core_pcpu, M_PMC); core_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_core.h b/sys/dev/hwpmc/hwpmc_core.h index 5b42182a876c..b75146d8b590 100644 --- a/sys/dev/hwpmc/hwpmc_core.h +++ b/sys/dev/hwpmc/hwpmc_core.h @@ -1,206 +1,210 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_HWPMC_CORE_H_ #define _DEV_HWPMC_CORE_H_ 1 #define IA32_PERF_CAPABILITIES 0x345 #define PERFCAP_LBR_FORMAT 0x003f #define PERFCAP_PEBS_TRAP 0x0040 #define PERFCAP_PEBS_SAVEARCH 0x0080 #define PERFCAP_PEBS_RECFORMAT 0x0f00 #define PERFCAP_SMM_FREEZE 0x1000 #define PERFCAP_FW_WRITE 0x2000 /* full width write aliases */ #define IAF_OS 0x1 #define IAF_USR 0x2 #define IAF_ANY 0x4 #define IAF_PMI 0x8 /* * Programmable PMCs. */ struct pmc_md_iap_op_pmcallocate { - uint32_t pm_iap_config; + uint64_t pm_iap_config; uint64_t pm_iap_rsp; }; #define IAP_EVSEL(C) ((C) & 0xFF) #define IAP_UMASK(C) (((C) & 0xFF) << 8) #define IAP_USR (1 << 16) #define IAP_OS (1 << 17) #define IAP_EDGE (1 << 18) #define IAP_INT (1 << 20) #define IAP_ANY (1 << 21) #define IAP_EN (1 << 22) #define IAP_INV (1 << 23) #define IAP_CMASK(C) (((C) & 0xFF) << 24) #define IAP_EVSEL_GET(C) ((C) & 0xFF) #define IAP_UMASK_GET(C) (((C) & 0xFF00) >> 8) #define IA_OFFCORE_RSP_MASK_I7WM 0x000000F7FF #define IA_OFFCORE_RSP_MASK_SBIB 0x3F807F8FFF #ifdef _KERNEL /* * Fixed-function counters. */ -#define IAF_MASK 0xF +#define IAF_MASK 0x000000010000000f -#define IAF_COUNTER_MASK 0x0000ffffffffffff #define IAF_CTR0 0x309 #define IAF_CTR1 0x30A #define IAF_CTR2 0x30B /* * The IAF_CTRL MSR is laid out in the following way. * * Bit Position Use - * 63 - 12 Reserved (do not touch) + * 63 - 45 Reserved (do not touch) + * 44 Ctr 3 Adaptive Record (v5) + * 43 - 41 Reserved (do not touch) + * 40 Ctr 2 Adaptive Record (v5) + * 39 - 37 Reserved (do not touch) + * 36 Ctr 1 Adaptive Record (v5) + * 35 - 33 Reserved (do not touch) + * 32 Ctr 0 Adaptive Record (v5) + * 15 Ctr 3 PMI + * 14 Ctr 3 Any Thread (v3) + * 13-12 Ctr 3 Enable * 11 Ctr 2 PMI * 10 Ctr 2 Any Thread (v3) * 9-8 Ctr 2 Enable * 7 Ctr 1 PMI * 6 Ctr 1 Any Thread (v3) * 5-4 Ctr 1 Enable * 3 Ctr 0 PMI * 2 Ctr 0 Any Thread (v3) * 1-0 Ctr 0 Enable (3: All Levels, 2: User, 1: OS, 0: Disable) */ #define IAF_OFFSET 32 #define IAF_CTRL 0x38D -#define IAF_CTRL_MASK 0x0000000000000fff /* * Programmable counters. */ #define IAP_PMC0 0x0C1 #define IAP_A_PMC0 0x4C1 /* * IAP_EVSEL(n) is laid out in the following way. * * Bit Position Use - * 63-31 Reserved (do not touch) + * 63-35 Reserved (do not touch) + * 34 Adaptive Record (v5) + * 33 IN_TX (v3) + * 32 IN_TXCP (v3) * 31-24 Counter Mask * 23 Invert * 22 Enable * 21 Any Thread (v3) * 20 APIC Interrupt Enable * 19 Pin Control * 18 Edge Detect * 17 OS * 16 User * 15-8 Unit Mask * 7-0 Event Select */ -#define IAP_EVSEL_MASK 0x00000000ffffffff #define IAP_EVSEL0 0x186 /* * Simplified programming interface in Intel Performance Architecture * v2 and later. */ #define IA_GLOBAL_STATUS 0x38E #define IA_GLOBAL_CTRL 0x38F /* * IA_GLOBAL_CTRL is laid out in the following way. * * Bit Position Use - * 63-35 Reserved (do not touch) + * 63-49 Reserved (do not touch) + * 48 Perf Metrics Enable (v5) + * 47-36 Reserved (do not touch) + * 35 IAF Counter 3 Enable * 34 IAF Counter 2 Enable * 33 IAF Counter 1 Enable * 32 IAF Counter 0 Enable * 31-0 Depends on programmable counters */ -/* The mask is only for the fixed porttion of the register. */ -#define IAF_GLOBAL_CTRL_MASK 0x0000000700000000 - -/* The mask is only for the programmable porttion of the register. */ -#define IAP_GLOBAL_CTRL_MASK 0x00000000ffffffff - -/* The mask is for both the fixed and programmable porttions of the register. */ -#define IA_GLOBAL_CTRL_MASK 0x00000007ffffffff - #define IA_GLOBAL_OVF_CTRL 0x390 #define IA_GLOBAL_STATUS_RESET 0x390 #define IA_GLOBAL_STATUS_SET 0x391 /* v4 */ #define IA_GLOBAL_STATUS_FLAG_CONDCHG (1ULL << 63) #define IA_GLOBAL_STATUS_FLAG_OVFBUF (1ULL << 62) #define IA_GLOBAL_STATUS_FLAG_OVFUNCORE (1ULL << 61) /* v3 */ #define IA_GLOBAL_STATUS_FLAG_ASCI (1ULL << 60) /* v4 */ #define IA_GLOBAL_STATUS_FLAG_CTR_FRZ (1ULL << 59) /* v4 */ #define IA_GLOBAL_STATUS_FLAG_LBP_FRZ (1ULL << 58) /* v4 */ #define IA_GLOBAL_STATUS_FLAG_TRACETOPAPMI (1ULL << 55) /* v4 */ #define IA_GLOBAL_INUSE 0x392 /* v4 */ /* * Offcore response configuration. */ #define IA_OFFCORE_RSP0 0x1A6 #define IA_OFFCORE_RSP1 0x1A7 struct pmc_md_iaf_pmc { uint64_t pm_iaf_ctrl; }; struct pmc_md_iap_pmc { - uint32_t pm_iap_evsel; + uint64_t pm_iap_evsel; uint64_t pm_iap_rsp; }; /* * Prototypes. */ int pmc_core_initialize(struct pmc_mdep *_md, int _maxcpu, int _version_override); void pmc_core_finalize(struct pmc_mdep *_md); int pmc_iaf_initialize(struct pmc_mdep *_md, int _maxcpu, int _npmc, int _width); void pmc_iaf_finalize(struct pmc_mdep *_md); int pmc_iap_initialize(struct pmc_mdep *_md, int _maxcpu, int _npmc, int _width, int _flags); void pmc_iap_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _DEV_HWPMC_CORE_H */ diff --git a/sys/dev/hwpmc/hwpmc_uncore.c b/sys/dev/hwpmc/hwpmc_uncore.c index 93e41e8d91f1..f69b21e173d5 100644 --- a/sys/dev/hwpmc/hwpmc_uncore.c +++ b/sys/dev/hwpmc/hwpmc_uncore.c @@ -1,844 +1,844 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Uncore PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if (__FreeBSD_version >= 1100000) #include #else #include #endif #include #include #include #define UCF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE) #define UCP_PMC_CAPS \ (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define SELECTSEL(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCP_CB0_EVSEL0 : UCP_EVSEL0) #define SELECTOFF(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCF_OFFSET_SB : UCF_OFFSET) static enum pmc_cputype uncore_cputype; struct uncore_cpu { volatile uint32_t pc_ucfctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_uncorepmcs[]; }; static struct uncore_cpu **uncore_pcpu; static uint64_t uncore_pmcmask; static int uncore_ucf_ri; /* relative index of fixed counters */ static int uncore_ucf_width; static int uncore_ucf_npmc; static int uncore_ucp_width; static int uncore_ucp_npmc; static int uncore_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int uncore_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct uncore_cpu *cc; struct pmc_hw *phw; int uncore_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); uncore_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + uncore_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + uncore_ri] = phw; } return (0); } static int uncore_pcpu_fini(struct pmc_mdep *md, int cpu) { int uncore_ri, n, npmc; struct pmc_cpu *pc; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); if ((cc = uncore_pcpu[cpu]) == NULL) return (0); uncore_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(SELECTSEL(uncore_cputype) + n, 0); wrmsr(UCF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + uncore_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t ucf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (uncore_ucf_width - 1))) == 0) return (0); v &= (1ULL << uncore_ucf_width) - 1; return (1ULL << uncore_ucf_width) - v; } static pmc_value_t ucf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucf_width) - rlc; } static int ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t flags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > uncore_ucf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_UCF) return (EINVAL); flags = UCF_EN; pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); return (0); } static int ucf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; return (0); } static int ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucf_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; return (0); } static int ucf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + uncore_ucf_ri)); tmp = rdmsr(UCF_CTR0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); return (0); } static int ucf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); } static void ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; pcd->pcd_caps = UCF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucf_allocate_pmc; pcd->pcd_config_pmc = ucf_config_pmc; pcd->pcd_describe = ucf_describe; pcd->pcd_get_config = ucf_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_noop; pcd->pcd_pcpu_init = uncore_pcpu_noop; pcd->pcd_read_pmc = ucf_read_pmc; pcd->pcd_release_pmc = ucf_release_pmc; pcd->pcd_start_pmc = ucf_start_pmc; pcd->pcd_stop_pmc = ucf_stop_pmc; pcd->pcd_write_pmc = ucf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct ucp_event_descr { enum pmc_event ucp_ev; unsigned char ucp_evcode; unsigned char ucp_umask; unsigned char ucp_flags; }; #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ #define UCP_F_WM (1 << 1) /* CPU: Westmere */ #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ #define UCP_F_HW (1 << 3) /* CPU: Haswell */ #define UCP_F_FM (1 << 4) /* Fixed mask */ #define UCP_F_ALLCPUS \ (UCP_F_I7 | UCP_F_WM) #define UCP_F_CMASK 0xFF000000 static pmc_value_t ucp_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucp_width) - 1; return (1ULL << uncore_ucp_width) - v; } static pmc_value_t ucp_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucp_width) - rlc; } /* * Counter specific event information for Sandybridge and Haswell */ static int ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) { uint32_t mask; switch (ev) { /* * Events valid only on counter 0. */ case 0x80: case 0x83: mask = (1 << 0); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_ucp_op_pmcallocate *ucp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_UCP) return (EINVAL); ucp = &a->pm_md.pm_ucp; ev = UCP_EVSEL(ucp->pm_ucp_config); switch (uncore_cputype) { case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_SANDYBRIDGE: if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: break; } pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; return (0); } static int ucp_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; return (0); } static int ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucp_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucp_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; return (0); } static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); } static int ucp_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; - uint32_t evsel; + uint64_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); /* Event specific configuration. */ switch (pm->pm_event) { case PMC_EV_UCP_EVENT_0CH_04H_E: case PMC_EV_UCP_EVENT_0CH_08H_E: wrmsr(MSR_GQ_SNOOP_MESF,0x2); break; case PMC_EV_UCP_EVENT_0CH_04H_F: case PMC_EV_UCP_EVENT_0CH_08H_F: wrmsr(MSR_GQ_SNOOP_MESF,0x8); break; case PMC_EV_UCP_EVENT_0CH_04H_M: case PMC_EV_UCP_EVENT_0CH_08H_M: wrmsr(MSR_GQ_SNOOP_MESF,0x1); break; case PMC_EV_UCP_EVENT_0CH_04H_S: case PMC_EV_UCP_EVENT_0CH_08H_S: wrmsr(MSR_GQ_SNOOP_MESF,0x4); break; default: break; } wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); return (0); } static int ucp_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); /* stop hw. */ wrmsr(SELECTSEL(uncore_cputype) + ri, 0); cc->pc_globalctrl &= ~(1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); return (0); } static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); } static void ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucp-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; pcd->pcd_caps = UCP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucp_allocate_pmc; pcd->pcd_config_pmc = ucp_config_pmc; pcd->pcd_describe = ucp_describe; pcd->pcd_get_config = ucp_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_fini; pcd->pcd_pcpu_init = uncore_pcpu_init; pcd->pcd_read_pmc = ucp_read_pmc; pcd->pcd_release_pmc = ucp_release_pmc; pcd->pcd_start_pmc = ucp_start_pmc; pcd->pcd_stop_pmc = ucp_stop_pmc; pcd->pcd_write_pmc = ucp_write_pmc; md->pmd_npmc += npmc; } int pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) { uncore_cputype = md->pmd_cputype; uncore_pmcmask = 0; /* * Initialize programmable counters. */ uncore_ucp_npmc = 8; uncore_ucp_width = 48; uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); /* * Initialize fixed function counters, if present. */ uncore_ucf_ri = uncore_ucp_npmc; uncore_ucf_npmc = 1; uncore_ucf_width = 48; ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, uncore_ucf_ri); uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); return (0); } void pmc_uncore_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "uncore-finalize"); free(uncore_pcpu, M_PMC); uncore_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_uncore.h b/sys/dev/hwpmc/hwpmc_uncore.h index ac1623e7737e..5e8610f77ddf 100644 --- a/sys/dev/hwpmc/hwpmc_uncore.h +++ b/sys/dev/hwpmc/hwpmc_uncore.h @@ -1,127 +1,127 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_HWPMC_UNCORE_H_ #define _DEV_HWPMC_UNCORE_H_ 1 /* * Fixed-function PMCs. */ struct pmc_md_ucf_op_pmcallocate { uint16_t pm_ucf_flags; /* additional flags */ }; #define UCF_EN 0x1 #define UCF_PMI 0x4 /* * Programmable PMCs. */ struct pmc_md_ucp_op_pmcallocate { - uint32_t pm_ucp_config; + uint64_t pm_ucp_config; }; #define UCP_EVSEL(C) ((C) & 0xFF) #define UCP_UMASK(C) ((C) & 0xFF00) #define UCP_CTRR (1 << 17) #define UCP_EDGE (1 << 18) #define UCP_INT (1 << 20) #define UCP_EN (1 << 22) #define UCP_INV (1 << 23) #define UCP_CMASK(C) (((C) & 0xFF) << 24) #ifdef _KERNEL #define DCTL_FLAG_UNC_PMI (1ULL << 13) /* * Fixed-function counters. */ #define UCF_MASK 0xF #define UCF_CTR0 0x394 #define UCF_OFFSET 32 #define UCF_OFFSET_SB 29 #define UCF_CTRL 0x395 /* * Programmable counters. */ #define UCP_PMC0 0x3B0 #define UCP_EVSEL0 0x3C0 #define UCP_OPCODE_MATCH 0x396 #define UCP_CB0_EVSEL0 0x700 /* * Simplified programming interface in Intel Performance Architecture * v2 and later. */ #define UC_GLOBAL_STATUS 0x392 #define UC_GLOBAL_CTRL 0x391 #define UC_GLOBAL_OVF_CTRL 0x393 #define UC_GLOBAL_STATUS_FLAG_CLRCHG (1ULL << 63) #define UC_GLOBAL_STATUS_FLAG_OVFPMI (1ULL << 61) #define UC_GLOBAL_CTRL_FLAG_FRZ (1ULL << 63) #define UC_GLOBAL_CTRL_FLAG_ENPMICORE0 (1ULL << 48) /* * Model specific registers. */ #define MSR_GQ_SNOOP_MESF 0x301 struct pmc_md_ucf_pmc { uint64_t pm_ucf_ctrl; }; struct pmc_md_ucp_pmc { - uint32_t pm_ucp_evsel; + uint64_t pm_ucp_evsel; }; /* * Prototypes. */ int pmc_uncore_initialize(struct pmc_mdep *_md, int _maxcpu); void pmc_uncore_finalize(struct pmc_mdep *_md); int pmc_ucf_initialize(struct pmc_mdep *_md, int _maxcpu, int _npmc, int _width); void pmc_ucf_finalize(struct pmc_mdep *_md); int pmc_ucp_initialize(struct pmc_mdep *_md, int _maxcpu, int _npmc, int _width, int _flags); void pmc_ucp_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _DEV_HWPMC_UNCORE_H */