diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 024024ece19b..b26e31cc1543 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -1,1238 +1,1236 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* Support for the AMD K7 and later processors */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_DEBUG enum pmc_class amd_pmc_class; #endif #define OVERFLOW_WAIT_COUNT 50 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); /* AMD K7 & K8 PMCs */ struct amd_descr { struct pmc_descr pm_descr; /* "base class" */ uint32_t pm_evsel; /* address of EVSEL register */ uint32_t pm_perfctr; /* address of PERFCTR register */ }; static struct amd_descr amd_pmcdesc[AMD_NPMCS] = { { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_0, .pm_perfctr = AMD_PMC_PERFCTR_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_1, .pm_perfctr = AMD_PMC_PERFCTR_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_2, .pm_perfctr = AMD_PMC_PERFCTR_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_3, .pm_perfctr = AMD_PMC_PERFCTR_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_4, .pm_perfctr = AMD_PMC_PERFCTR_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_5, .pm_perfctr = AMD_PMC_PERFCTR_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_4, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_5, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_3 } }; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ uint16_t pe_code; /* encoded event mask */ uint8_t pe_mask; /* bits allowed in unit mask */ }; const struct amd_event_code_map amd_event_codes[] = { #if defined(__i386__) /* 32 bit Athlon (K7) only */ { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, { PMC_EV_K7_DC_MISSES, 0x41, 0 }, { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, { PMC_EV_K7_IC_MISSES, 0x81, 0 }, { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, #endif { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 0x21, 0x00 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 0xCA, 0x00 }, { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 0xCC, 0x07 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 0xD2, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 0xD5, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 0xD6, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 0xD9, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 0xDA, 0x00 }, { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 0xE2, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } }; const int amd_event_codes_size = nitems(amd_event_codes); /* * Per-processor information */ struct amd_cpu { struct pmc_hw pc_amdpmcs[AMD_NPMCS]; }; static struct amd_cpu **amd_pcpu; /* * read a pmc register */ static int amd_read_pmc(int cpu, int ri, pmc_value_t *v) { enum pmc_mode mode; const struct amd_descr *pd; struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); KASSERT(amd_pcpu[cpu], ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(mode)) { /* Sign extend 48 bit value to 64 bits. */ tmp = (pmc_value_t) (((int64_t) tmp << 16) >> 16); tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } *v = tmp; PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v); return 0; } /* * Write a PMC MSR. */ static int amd_write_pmc(int cpu, int ri, pmc_value_t v) { const struct amd_descr *pd; enum pmc_mode mode; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif /* use 2's complement of the count for sampling mode PMCs */ if (PMC_IS_SAMPLING_MODE(mode)) v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); /* write the PMC value */ wrmsr(pd->pm_perfctr, v); return 0; } /* * configure hardware pmc according to the configuration recorded in * pmc 'pm'. */ static int amd_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } /* * Retrieve a configured PMC pointer from hardware state. */ static int amd_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; return 0; } /* * Machine dependent actions taken during the context switch in of a * thread. */ static int amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); /* enable the RDPMC instruction if needed */ if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) load_cr4(rcr4() | CR4_PCE); return 0; } /* * Machine dependent actions taken during the context switch out of a * thread. */ static int amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; (void) pp; /* can be NULL */ PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); /* always turn off the RDPMC instruction */ load_cr4(rcr4() & ~CR4_PCE); return 0; } /* * Check if a given allocation is feasible. */ static int amd_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int i; uint64_t allowed_unitmask, caps, config, unitmask; enum pmc_event pe; const struct pmc_descr *pd; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri].pm_descr; /* check class match */ if (pd->pd_class != a->pm_class) return EINVAL; caps = pm->pm_caps; PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); if((ri >= 0 && ri < 6) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_CORE)) return EINVAL; if((ri >= 6 && ri < 12) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_L3_CACHE)) return EINVAL; if((ri >= 12 && ri < 16) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_DATA_FABRIC)) return EINVAL; - if ((pd->pd_caps & caps) != caps) - return EPERM; if (strlen(pmc_cpuid) != 0) { pm->pm_md.pm_amd.pm_amd_evsel = a->pm_md.pm_amd.pm_amd_config; PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, a->pm_md.pm_amd.pm_amd_config); return (0); } pe = a->pm_ev; /* map ev to the correct event mask code */ config = allowed_unitmask = 0; for (i = 0; i < amd_event_codes_size; i++) if (amd_event_codes[i].pe_ev == pe) { config = AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); allowed_unitmask = AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); break; } if (i == amd_event_codes_size) return EINVAL; unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ return EINVAL; if (unitmask && (caps & PMC_CAP_QUALIFIER)) config |= unitmask; if (caps & PMC_CAP_THRESHOLD) config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; /* set at least one of the 'usr' or 'os' caps */ if (caps & PMC_CAP_USER) config |= AMD_PMC_USR; if (caps & PMC_CAP_SYSTEM) config |= AMD_PMC_OS; if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) config |= (AMD_PMC_USR|AMD_PMC_OS); if (caps & PMC_CAP_EDGE) config |= AMD_PMC_EDGE; if (caps & PMC_CAP_INVERT) config |= AMD_PMC_INVERT; if (caps & PMC_CAP_INTERRUPT) config |= AMD_PMC_INT; pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); return 0; } /* * Release machine dependent state associated with a PMC. This is a * no-op on this architecture. * */ /* ARGSUSED0 */ static int amd_release_pmc(int cpu, int ri, struct pmc *pmc) { #ifdef HWPMC_DEBUG const struct amd_descr *pd; #endif struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); #ifdef HWPMC_DEBUG pd = &amd_pmcdesc[ri]; if (pd->pm_descr.pd_class == amd_pmc_class) KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC %d released while active", __LINE__, ri)); #endif return 0; } /* * start a PMC. */ static int amd_start_pmc(int cpu, int ri) { uint64_t config; struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, ri, cpu, pd->pm_descr.pd_name)); /* turn on the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config); wrmsr(pd->pm_evsel, config); return 0; } /* * Stop a PMC. */ static int amd_stop_pmc(int cpu, int ri) { struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; uint64_t config; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", __LINE__, ri, cpu, pd->pm_descr.pd_name)); PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri); /* turn off the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; wrmsr(pd->pm_evsel, config); /* * Due to NMI latency on newer AMD processors * NMI interrupts are ignored, which leads to * panic or messages based on kernel configuration */ /* Wait for the count to be reset */ for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) break; DELAY(1); } return 0; } /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed * to sleep or do anything a 'fast' interrupt handler is not allowed * to do. */ static int amd_intr(struct trapframe *tf) { int i, error, retval, cpu; uint64_t config, evsel, perfctr; struct pmc *pm; struct amd_cpu *pac; pmc_value_t v; uint32_t active = 0, count = 0; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pac = amd_pcpu[cpu]; /* * look for all PMCs that have interrupted: * - look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association * * If found, we call a helper to process the interrupt. * * PMCs interrupting at the same time are collapsed into * a single interrupt. Check all the valid pmcs for * overflow. */ for (i = 0; i < AMD_CORE_NPMCS; i++) { if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; } /* Consider pmc with valid handle as active */ active++; if (!AMD_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* Stop the PMC, reload count. */ evsel = amd_pmcdesc[i].pm_evsel; perfctr = amd_pmcdesc[i].pm_perfctr; v = pm->pm_sc.pm_reloadcount; config = rdmsr(evsel); KASSERT((config & ~AMD_PMC_ENABLE) == (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); wrmsr(evsel, config & ~AMD_PMC_ENABLE); wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); /* Restart the counter if logging succeeded. */ error = pmc_process_interrupt(PMC_HR, pm, tf); if (error == 0) wrmsr(evsel, config); } /* * Due to NMI latency, there can be a scenario in which * multiple pmcs gets serviced in an earlier NMI and we * do not find an overflow in the subsequent NMI. * * For such cases we keep a per-cpu count of active NMIs * and compare it with min(active pmcs, 2) to determine * if this NMI was for a pmc overflow which was serviced * in an earlier request or should be ignored. */ if (retval) { DPCPU_SET(nmi_counter, min(2, active)); } else { if ((count = DPCPU_GET(nmi_counter))) { retval = 1; DPCPU_SET(nmi_counter, --count); } } if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); PMCDBG1(MDP,INT,2, "retval=%d", retval); return (retval); } /* * describe a PMC */ static int amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; size_t copied; const struct amd_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] row-index %d out of range", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pd = &amd_pmcdesc[ri]; if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, PMC_NAME_MAX, &copied)) != 0) return error; pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return 0; } /* * i386 specific entry points */ /* * return the MSR address of the given PMC. */ static int amd_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] ri %d out of range", __LINE__, ri)); *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; return (0); } /* * processor dependent initialization. */ static int amd_pcpu_init(struct pmc_mdep *md, int cpu) { int classindex, first_ri, n; struct pmc_cpu *pc; struct amd_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu); amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, M_WAITOK|M_ZERO); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * processor dependent cleanup prior to the KLD * being unloaded */ static int amd_pcpu_fini(struct pmc_mdep *md, int cpu) { int classindex, first_ri, i; uint32_t evsel; struct pmc_cpu *pc; struct amd_cpu *pac; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu); /* * First, turn off all PMCs on this CPU. */ for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ evsel = rdmsr(AMD_PMC_EVSEL_0 + i); evsel &= ~AMD_PMC_ENABLE; wrmsr(AMD_PMC_EVSEL_0 + i, evsel); } /* * Next, free up allocated space. */ if ((pac = amd_pcpu[cpu]) == NULL) return (0); amd_pcpu[cpu] = NULL; #ifdef HWPMC_DEBUG for (i = 0; i < AMD_NPMCS; i++) { KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); } #endif pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; /* * Reset pointers in the MI 'per-cpu' state. */ for (i = 0; i < AMD_NPMCS; i++) { pc->pc_hwpmcs[i + first_ri] = NULL; } free(pac, M_PMC); return (0); } /* * Initialize ourselves. */ struct pmc_mdep * pmc_amd_initialize(void) { int classindex, error, i, ncpus; struct pmc_classdep *pcd; enum pmc_cputype cputype; struct pmc_mdep *pmc_mdep; enum pmc_class class; int family, model, stepping; char *name; /* * The presence of hardware performance counters on the AMD * Athlon, Duron or later processors, is _not_ indicated by * any of the processor feature flags set by the 'CPUID' * instruction, so we only check the 'instruction family' * field returned by CPUID for instruction family >= 6. */ name = NULL; family = CPUID_TO_FAMILY(cpu_id); model = CPUID_TO_MODEL(cpu_id); stepping = CPUID_TO_STEPPING(cpu_id); if (family == 0x18) snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", family, model, stepping); else snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", family, model, stepping); switch (cpu_id & 0xF00) { #if defined(__i386__) case 0x600: /* Athlon(tm) processor */ classindex = PMC_MDEP_CLASS_INDEX_K7; cputype = PMC_CPU_AMD_K7; class = PMC_CLASS_K7; name = "K7"; break; #endif case 0xF00: /* Athlon64/Opteron processor */ classindex = PMC_MDEP_CLASS_INDEX_K8; cputype = PMC_CPU_AMD_K8; class = PMC_CLASS_K8; name = "K8"; break; default: (void) printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, (cpu_id & 0xF00) >> 8, model); return NULL; } #ifdef HWPMC_DEBUG amd_pmc_class = class; #endif /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* * These processors have two classes of PMCs: the TSC and * programmable PMCs. */ pmc_mdep = pmc_mdep_alloc(2); pmc_mdep->pmd_cputype = cputype; ncpus = pmc_cpu_max(); /* Initialize the TSC. */ error = pmc_tsc_initialize(pmc_mdep, ncpus); if (error) goto error; /* Initialize AMD K7 and K8 PMC handling. */ pcd = &pmc_mdep->pmd_classdep[classindex]; pcd->pcd_caps = AMD_PMC_CAPS; pcd->pcd_class = class; pcd->pcd_num = AMD_NPMCS; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 48; /* fill in the correct pmc name and class */ for (i = 0; i < AMD_NPMCS; i++) { (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", name, i); amd_pmcdesc[i].pm_descr.pd_class = class; } pcd->pcd_allocate_pmc = amd_allocate_pmc; pcd->pcd_config_pmc = amd_config_pmc; pcd->pcd_describe = amd_describe; pcd->pcd_get_config = amd_get_config; pcd->pcd_get_msr = amd_get_msr; pcd->pcd_pcpu_fini = amd_pcpu_fini; pcd->pcd_pcpu_init = amd_pcpu_init; pcd->pcd_read_pmc = amd_read_pmc; pcd->pcd_release_pmc = amd_release_pmc; pcd->pcd_start_pmc = amd_start_pmc; pcd->pcd_stop_pmc = amd_stop_pmc; pcd->pcd_write_pmc = amd_write_pmc; pmc_mdep->pmd_pcpu_init = NULL; pmc_mdep->pmd_pcpu_fini = NULL; pmc_mdep->pmd_intr = amd_intr; pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; pmc_mdep->pmd_npmc += AMD_NPMCS; PMCDBG0(MDP,INI,0,"amd-initialize"); return (pmc_mdep); error: if (error) { free(pmc_mdep, M_PMC); pmc_mdep = NULL; } return (NULL); } /* * Finalization code for AMD CPUs. */ void pmc_amd_finalize(struct pmc_mdep *md) { #if defined(INVARIANTS) int classindex, i, ncpus, pmcclass; #endif pmc_tsc_finalize(md); KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer", __LINE__)); #if defined(INVARIANTS) switch (md->pmd_cputype) { #if defined(__i386__) case PMC_CPU_AMD_K7: classindex = PMC_MDEP_CLASS_INDEX_K7; pmcclass = PMC_CLASS_K7; break; #endif default: classindex = PMC_MDEP_CLASS_INDEX_K8; pmcclass = PMC_CLASS_K8; } KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass, ("[amd,%d] pmc class mismatch", __LINE__)); ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu", __LINE__)); #endif free(amd_pcpu, M_PMC); amd_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_arm64.c b/sys/dev/hwpmc/hwpmc_arm64.c index 8a149f5f508f..14283b22644d 100644 --- a/sys/dev/hwpmc/hwpmc_arm64.c +++ b/sys/dev/hwpmc/hwpmc_arm64.c @@ -1,599 +1,598 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by the University of Cambridge Computer * Laboratory with support from ARM Ltd. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int arm64_npmcs; struct arm64_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; /* * Per-processor information. */ struct arm64_cpu { struct pmc_hw *pc_arm64pmcs; }; static struct arm64_cpu **arm64_pcpu; /* * Interrupt Enable Set Register */ static __inline void arm64_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenset_el1, reg); isb(); } /* * Interrupt Clear Set Register */ static __inline void arm64_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenclr_el1, reg); isb(); } /* * Counter Set Enable Register */ static __inline void arm64_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenset_el0, reg); isb(); } /* * Counter Clear Enable Register */ static __inline void arm64_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenclr_el0, reg); isb(); } /* * Performance Monitors Control Register */ static uint32_t arm64_pmcr_read(void) { uint32_t reg; reg = READ_SPECIALREG(pmcr_el0); return (reg); } static void arm64_pmcr_write(uint32_t reg) { WRITE_SPECIALREG(pmcr_el0, reg); isb(); } /* * Performance Count Register N */ static uint32_t arm64_pmcn_read(unsigned int pmc) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); isb(); return (READ_SPECIALREG(pmxevcntr_el0)); } static void arm64_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); WRITE_SPECIALREG(pmxevcntr_el0, reg); isb(); } static int arm64_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { - uint32_t caps, config; + uint32_t config; struct arm64_cpu *pac; enum pmc_event pe; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pac = arm64_pcpu[cpu]; - caps = a->pm_caps; if (a->pm_class != PMC_CLASS_ARMV8) { return (EINVAL); } pe = a->pm_ev; /* Adjust the config value if needed. */ config = a->pm_md.pm_md_config; if ((a->pm_md.pm_md_flags & PM_MD_RAW_EVENT) == 0) { config = (uint32_t)pe - PMC_EV_ARMV8_FIRST; if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST)) return (EINVAL); } pm->pm_md.pm_arm64.pm_arm64_evsel = config; PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config); return (0); } static int arm64_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; /* * Ensure we don't get interrupted while updating the overflow count. */ s = intr_disable(); tmp = arm64_pmcn_read(ri); reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) { /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = arm64_pmcn_read(ri); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); else *v = tmp; return (0); } static int arm64_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; arm64_pmcn_write(ri, v); return (0); } static int arm64_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int arm64_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_arm64.pm_arm64_evsel; /* * Configure the event selection. */ WRITE_SPECIALREG(pmselr_el0, ri); WRITE_SPECIALREG(pmxevtyper_el0, config); isb(); /* * Enable the PMC. */ arm64_interrupt_enable(ri); arm64_counter_enable(ri); return (0); } static int arm64_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. */ arm64_counter_disable(ri); arm64_interrupt_disable(ri); return (0); } static int arm64_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } static int arm64_intr(struct trapframe *tf) { struct arm64_cpu *pc; int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] CPU %d out of range", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf, TRAPF_USERMODE(tf)); retval = 0; pc = arm64_pcpu[cpu]; for (ri = 0; ri < arm64_npmcs; ri++) { pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter is overflowed */ reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0) continue; /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); isb(); retval = 1; /* Found an interrupting PMC. */ if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; continue; } if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) arm64_stop_pmc(cpu, ri); /* Reload sampling count */ arm64_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { char arm64_name[PMC_NAME_MAX]; struct pmc_hw *phw; int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] row-index %d out of range", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; snprintf(arm64_name, sizeof(arm64_name), "ARMV8-%d", ri); if ((error = copystr(arm64_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_ARMV8; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int arm64_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; return (0); } /* * XXX don't know what we should do here. */ static int arm64_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_pcpu_init(struct pmc_mdep *md, int cpu) { struct arm64_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint64_t pmcr; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "arm64-init cpu=%d", cpu); arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC, M_WAITOK | M_ZERO); pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs, M_PMC, M_WAITOK | M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri; KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* Enable unit */ pmcr = arm64_pmcr_read(); pmcr |= PMCR_E; arm64_pmcr_write(pmcr); return (0); } static int arm64_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmcr; pmcr = arm64_pmcr_read(); pmcr &= ~PMCR_E; arm64_pmcr_write(pmcr); return (0); } struct pmc_mdep * pmc_arm64_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode, impcode; int reg; uint64_t midr; reg = arm64_pmcr_read(); arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT; impcode = (reg & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT; idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs); /* * Write the CPU model to kern.hwpmc.cpuid. * * We zero the variant and revision fields. * * TODO: how to handle differences between cores due to big.LITTLE? * For now, just use MIDR from CPU 0. */ midr = (uint64_t)(pcpu_find(0)->pc_midr); midr &= ~(CPU_VAR_MASK | CPU_REV_MASK); snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch(impcode) { case PMCR_IMP_ARM: switch (idcode) { case PMCR_IDCODE_CORTEX_A76: case PMCR_IDCODE_NEOVERSE_N1: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76; break; case PMCR_IDCODE_CORTEX_A57: case PMCR_IDCODE_CORTEX_A72: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57; break; default: case PMCR_IDCODE_CORTEX_A53: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } break; default: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8]; pcd->pcd_caps = ARMV8_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV8; pcd->pcd_num = arm64_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = arm64_allocate_pmc; pcd->pcd_config_pmc = arm64_config_pmc; pcd->pcd_pcpu_fini = arm64_pcpu_fini; pcd->pcd_pcpu_init = arm64_pcpu_init; pcd->pcd_describe = arm64_describe; pcd->pcd_get_config = arm64_get_config; pcd->pcd_read_pmc = arm64_read_pmc; pcd->pcd_release_pmc = arm64_release_pmc; pcd->pcd_start_pmc = arm64_start_pmc; pcd->pcd_stop_pmc = arm64_stop_pmc; pcd->pcd_write_pmc = arm64_write_pmc; pmc_mdep->pmd_intr = arm64_intr; pmc_mdep->pmd_switch_in = arm64_switch_in; pmc_mdep->pmd_switch_out = arm64_switch_out; pmc_mdep->pmd_npmc += arm64_npmcs; return (pmc_mdep); } void pmc_arm64_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_armv7.c b/sys/dev/hwpmc/hwpmc_armv7.c index 84a983bbc69c..12b3fff4d743 100644 --- a/sys/dev/hwpmc/hwpmc_armv7.c +++ b/sys/dev/hwpmc/hwpmc_armv7.c @@ -1,569 +1,567 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int armv7_npmcs; struct armv7_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; #define PMC_EV_CPU_CYCLES 0xFF /* * Per-processor information. */ struct armv7_cpu { struct pmc_hw *pc_armv7pmcs; }; static struct armv7_cpu **armv7_pcpu; /* * Interrupt Enable Set Register */ static __inline void armv7_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_set(reg); } /* * Interrupt Clear Set Register */ static __inline void armv7_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_clr(reg); } /* * Counter Set Enable Register */ static __inline void armv7_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_set(reg); } /* * Counter Clear Enable Register */ static __inline void armv7_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_clr(reg); } /* * Performance Count Register N */ static uint32_t armv7_pmcn_read(unsigned int pmc, uint32_t evsel) { if (evsel == PMC_EV_CPU_CYCLES) { return ((uint32_t)cp15_pmccntr_get()); } KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); return (cp15_pmxevcntr_get()); } static uint32_t armv7_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); cp15_pmxevcntr_set(reg); return (reg); } static int armv7_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { struct armv7_cpu *pac; enum pmc_event pe; uint32_t config; - uint32_t caps; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); pac = armv7_pcpu[cpu]; - caps = a->pm_caps; if (a->pm_class != PMC_CLASS_ARMV7) return (EINVAL); pe = a->pm_ev; config = (pe & EVENT_ID_MASK); pm->pm_md.pm_armv7.pm_armv7_evsel = config; PMCDBG2(MDP, ALL, 2, "armv7-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int armv7_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; u_int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; s = intr_disable(); tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) != 0) { /* Clear Overflow Flag */ cp15_pmovsr_set(reg); if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "armv7-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ARMV7_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); else *v = tmp; return 0; } static int armv7_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV7_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "armv7-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) cp15_pmccntr_set(v); else armv7_pmcn_write(ri, v); return 0; } static int armv7_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[armv7,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int armv7_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; /* * Configure the event selection. */ if (config != PMC_EV_CPU_CYCLES) { cp15_pmselr_set(ri); cp15_pmxevtyper_set(config); } else ri = 31; /* * Enable the PMC. */ armv7_interrupt_enable(ri); armv7_counter_enable(ri); return 0; } static int armv7_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; uint32_t config; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; if (config == PMC_EV_CPU_CYCLES) ri = 31; /* * Disable the PMCs. */ armv7_counter_disable(ri); armv7_interrupt_disable(ri); return 0; } static int armv7_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[armv7,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int armv7_intr(struct trapframe *tf) { struct armv7_cpu *pc; int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; pc = armv7_pcpu[cpu]; for (ri = 0; ri < armv7_npmcs; ri++) { pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) == 0) { continue; } /* Clear Overflow Flag */ cp15_pmovsr_set(reg); retval = 1; /* Found an interrupting PMC. */ if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; continue; } if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) armv7_stop_pmc(cpu, ri); /* Reload sampling count */ armv7_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int armv7_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { char armv7_name[PMC_NAME_MAX]; struct pmc_hw *phw; int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] row-index %d out of range", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; snprintf(armv7_name, sizeof(armv7_name), "ARMV7-%d", ri); if ((error = copystr(armv7_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return error; pi->pm_class = PMC_CLASS_ARMV7; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int armv7_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; return 0; } /* * XXX don't know what we should do here. */ static int armv7_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_pcpu_init(struct pmc_mdep *md, int cpu) { struct armv7_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint32_t pmnc; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "armv7-init cpu=%d", cpu); armv7_pcpu[cpu] = pac = malloc(sizeof(struct armv7_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_armv7pmcs = malloc(sizeof(struct pmc_hw) * armv7_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7].pcd_ri; KASSERT(pc != NULL, ("[armv7,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_armv7pmcs; i < armv7_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); /* Enable unit */ pmnc = cp15_pmcr_get(); pmnc |= ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); return 0; } static int armv7_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmnc; pmnc = cp15_pmcr_get(); pmnc &= ~ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); return 0; } struct pmc_mdep * pmc_armv7_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode; int reg; reg = cp15_pmcr_get(); armv7_npmcs = (reg >> ARMV7_PMNC_N_SHIFT) & \ ARMV7_PMNC_N_MASK; idcode = (reg & ARMV7_IDCODE_MASK) >> ARMV7_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "armv7-init npmcs=%d", armv7_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ armv7_pcpu = malloc(sizeof(struct armv7_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch (idcode) { case ARMV7_IDCODE_CORTEX_A9: pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A9; break; default: case ARMV7_IDCODE_CORTEX_A8: /* * On A8 we implemented common events only, * so use it for the rest of machines. */ pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A8; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7]; pcd->pcd_caps = ARMV7_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV7; pcd->pcd_num = armv7_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = armv7_allocate_pmc; pcd->pcd_config_pmc = armv7_config_pmc; pcd->pcd_pcpu_fini = armv7_pcpu_fini; pcd->pcd_pcpu_init = armv7_pcpu_init; pcd->pcd_describe = armv7_describe; pcd->pcd_get_config = armv7_get_config; pcd->pcd_read_pmc = armv7_read_pmc; pcd->pcd_release_pmc = armv7_release_pmc; pcd->pcd_start_pmc = armv7_start_pmc; pcd->pcd_stop_pmc = armv7_stop_pmc; pcd->pcd_write_pmc = armv7_write_pmc; pmc_mdep->pmd_intr = armv7_intr; pmc_mdep->pmd_switch_in = armv7_switch_in; pmc_mdep->pmd_switch_out = armv7_switch_out; pmc_mdep->pmd_npmc += armv7_npmcs; return (pmc_mdep); } void pmc_armv7_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index 507b20488132..84967e8a6248 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -1,1347 +1,1340 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Core PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #define CORE_CPUID_REQUEST 0xA #define CORE_CPUID_REQUEST_SIZE 0x4 #define CORE_CPUID_EAX 0x0 #define CORE_CPUID_EBX 0x1 #define CORE_CPUID_ECX 0x2 #define CORE_CPUID_EDX 0x3 #define IAF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30)) #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define EV_IS_NOTARCH 0 #define EV_IS_ARCH_SUPP 1 #define EV_IS_ARCH_NOTSUPP -1 /* * "Architectural" events defined by Intel. The values of these * symbols correspond to positions in the bitmask returned by * the CPUID.0AH instruction. */ enum core_arch_events { CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5, CORE_AE_BRANCH_MISSES_RETIRED = 6, CORE_AE_INSTRUCTION_RETIRED = 1, CORE_AE_LLC_MISSES = 4, CORE_AE_LLC_REFERENCE = 3, CORE_AE_UNHALTED_REFERENCE_CYCLES = 2, CORE_AE_UNHALTED_CORE_CYCLES = 0 }; static enum pmc_cputype core_cputype; struct core_cpu { volatile uint32_t pc_resync; volatile uint32_t pc_iafctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_corepmcs[]; }; static struct core_cpu **core_pcpu; static uint32_t core_architectural_events; static uint64_t core_pmcmask; static int core_iaf_ri; /* relative index of fixed counters */ static int core_iaf_width; static int core_iaf_npmc; static int core_iap_width; static int core_iap_npmc; static int core_iap_wroffset; static u_int pmc_alloc_refs; static bool pmc_tsx_force_abort_set; static int core_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int core_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct core_cpu *cc; struct pmc_hw *phw; int core_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; if (core_cputype != PMC_CPU_INTEL_CORE) npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); core_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + core_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + core_ri] = phw; } return (0); } static int core_pcpu_fini(struct pmc_mdep *md, int cpu) { int core_ri, n, npmc; struct pmc_cpu *pc; struct core_cpu *cc; uint64_t msr = 0; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); if ((cc = core_pcpu[cpu]) == NULL) return (0); core_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; for (n = 0; n < npmc; n++) { msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + n, msr); } if (core_cputype != PMC_CPU_INTEL_CORE) { msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; } for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + core_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t iaf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iaf_width - 1))) == 0) return (0); v &= (1ULL << core_iaf_width) - 1; return (1ULL << core_iaf_width) - v; } static pmc_value_t iaf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iaf_width) - rlc; } static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev, umask; uint32_t caps, flags, config; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > core_iaf_npmc) return (EINVAL); - caps = a->pm_caps; - - if (a->pm_class != PMC_CLASS_IAF || - (caps & IAF_PMC_CAPS) != caps) + if (a->pm_class != PMC_CLASS_IAF) return (EINVAL); iap = &a->pm_md.pm_iap; config = iap->pm_iap_config; ev = IAP_EVSEL_GET(config); umask = IAP_UMASK_GET(config); /* INST_RETIRED.ANY */ if (ev == 0xC0 && ri != 0) return (EINVAL); /* CPU_CLK_UNHALTED.THREAD */ if (ev == 0x3C && ri != 1) return (EINVAL); /* CPU_CLK_UNHALTED.REF */ if (ev == 0x0 && umask == 0x3 && ri != 2) return (EINVAL); pmc_alloc_refs++; if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 && !pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = true; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 1, NULL); } flags = 0; if (config & IAP_OS) flags |= IAF_OS; if (config & IAP_USR) flags |= IAF_USR; if (config & IAP_ANY) flags |= IAF_ANY; if (config & IAP_INT) flags |= IAF_PMI; + caps = a->pm_caps; if (caps & PMC_CAP_INTERRUPT) flags |= IAF_PMI; if (caps & PMC_CAP_SYSTEM) flags |= IAF_OS; if (caps & PMC_CAP_USER) flags |= IAF_USR; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) flags |= (IAF_OS | IAF_USR); pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); return (0); } static int iaf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; return (0); } static int iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iaf_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri]; (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri); if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iaf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; return (0); } static int iaf_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[iaf,%d] ri %d out of range", __LINE__, ri)); *msr = IAF_RI_TO_MSR(ri); return (0); } static int iaf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + core_iaf_ri)); tmp = rdpmc(IAF_RI_TO_MSR(ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iaf_width) - 1); PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); return (0); } static int iaf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); MPASS(pmc_alloc_refs > 0); if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = false; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 0, NULL); } return (0); } static int iaf_start_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *iafc; uint64_t msr = 0; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); iafc = core_pcpu[cpu]; pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); do { iafc->pc_resync = 0; iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & IAF_GLOBAL_CTRL_MASK)); } while (iafc->pc_resync != 0); PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_stop_pmc(int cpu, int ri) { uint32_t fc; struct core_cpu *iafc; uint64_t msr = 0; PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri); iafc = core_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); fc = (IAF_MASK << (ri * 4)); iafc->pc_iafctrl &= ~fc; PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl); msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); do { iafc->pc_resync = 0; iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET)); msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & IAF_GLOBAL_CTRL_MASK)); } while (iafc->pc_resync != 0); PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_write_pmc(int cpu, int ri, pmc_value_t v) { struct core_cpu *cc; struct pmc *pm; uint64_t msr; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iaf_reload_count_to_perfctr_value(v); /* Turn off fixed counters */ msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr); wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); /* Turn on fixed counters */ msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK)); PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); return (0); } static void iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iaf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; pcd->pcd_caps = IAF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iaf_allocate_pmc; pcd->pcd_config_pmc = iaf_config_pmc; pcd->pcd_describe = iaf_describe; pcd->pcd_get_config = iaf_get_config; pcd->pcd_get_msr = iaf_get_msr; pcd->pcd_pcpu_fini = core_pcpu_noop; pcd->pcd_pcpu_init = core_pcpu_noop; pcd->pcd_read_pmc = iaf_read_pmc; pcd->pcd_release_pmc = iaf_release_pmc; pcd->pcd_start_pmc = iaf_start_pmc; pcd->pcd_stop_pmc = iaf_stop_pmc; pcd->pcd_write_pmc = iaf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* Sub fields of UMASK that this event supports. */ #define IAP_M_CORE (1 << 0) /* Core specificity */ #define IAP_M_AGENT (1 << 1) /* Agent specificity */ #define IAP_M_PREFETCH (1 << 2) /* Prefetch */ #define IAP_M_MESI (1 << 3) /* MESI */ #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */ #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */ #define IAP_M_TRANSITION (1 << 6) /* Transition */ #define IAP_F_CORE (0x3 << 14) /* Core specificity */ #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */ #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */ #define IAP_F_MESI (0xF << 8) /* MESI */ #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */ #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */ #define IAP_F_TRANSITION (0x1 << 12) /* Transition */ #define IAP_PREFETCH_RESERVED (0x2 << 12) #define IAP_CORE_THIS (0x1 << 14) #define IAP_CORE_ALL (0x3 << 14) #define IAP_F_CMASK 0xFF000000 static pmc_value_t iap_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iap_width - 1))) == 0) return (0); v &= (1ULL << core_iap_width) - 1; return (1ULL << core_iap_width) - v; } static pmc_value_t iap_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iap_width) - rlc; } static int iap_pmc_has_overflowed(int ri) { uint64_t v; /* * We treat a Core (i.e., Intel architecture v1) PMC as has * having overflowed if its MSB is zero. */ v = rdpmc(ri); return ((v & (1ULL << (core_iap_width - 1))) == 0); } static int iap_event_corei7_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0, 1. */ case 0x40: case 0x41: case 0x42: case 0x43: case 0x51: case 0x63: mask = 0x3; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_westmere_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0. */ case 0x60: case 0xB3: mask = 0x1; break; /* * Events valid only on counter 0, 1. */ case 0x4C: case 0x4E: case 0x51: case 0x63: mask = 0x3; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0xB7: mask = 0x1; break; /* Events valid only on counter 1. */ case 0xC0: mask = 0x2; break; /* Events valid only on counter 2. */ case 0x48: case 0xA2: case 0xA3: mask = 0x4; break; /* Events valid only on counter 3. */ case 0xBB: case 0xCD: mask = 0x8; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0. */ case 0x10: case 0x14: case 0x18: case 0xB3: case 0xC1: case 0xCB: mask = (1 << 0); break; /* * Events valid only on counter 1. */ case 0x11: case 0x12: case 0x13: mask = (1 << 1); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; - uint32_t caps; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_IAP) return (EINVAL); - /* check requested capabilities */ - caps = a->pm_caps; - if ((IAP_PMC_CAPS & caps) != caps) - return (EPERM); iap = &a->pm_md.pm_iap; ev = IAP_EVSEL_GET(iap->pm_iap_config); switch (core_cputype) { case PMC_CPU_INTEL_COREI7: case PMC_CPU_INTEL_NEHALEM_EX: if (iap_event_corei7_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: case PMC_CPU_INTEL_SANDYBRIDGE: case PMC_CPU_INTEL_SANDYBRIDGE_XEON: case PMC_CPU_INTEL_IVYBRIDGE: case PMC_CPU_INTEL_IVYBRIDGE_XEON: case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_HASWELL_XEON: if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_WESTMERE: case PMC_CPU_INTEL_WESTMERE_EX: if (iap_event_westmere_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: if (iap_event_ok_on_counter(ev, ri) == 0) return (EINVAL); } pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config; return (0); } static int iap_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; return (0); } static int iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iap_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri]; (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri); if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iap_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; return (0); } static int iap_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iap_npmc, ("[iap,%d] ri %d out of range", __LINE__, ri)); *msr = ri; return (0); } static int iap_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdpmc(ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iap_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iap_width) - 1); PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAP_PMC0 + ri, *v); return (0); } static int iap_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iap_start_pmc(int cpu, int ri) { struct pmc *pm; uint32_t evsel; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_iap.pm_iap_evsel; PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, IAP_EVSEL0 + ri, evsel); /* Event specific configuration. */ switch (IAP_EVSEL_GET(evsel)) { case 0xB7: wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); break; case 0xBB: wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp); break; default: break; } wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN); if (core_cputype == PMC_CPU_INTEL_CORE) return (0); do { cc->pc_resync = 0; cc->pc_globalctrl |= (1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int iap_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct core_cpu *cc; uint64_t msr; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */ if (core_cputype == PMC_CPU_INTEL_CORE) return (0); do { cc->pc_resync = 0; cc->pc_globalctrl &= ~(1ULL << ri); msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int iap_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iap_reload_count_to_perfctr_value(v); v &= (1ULL << core_iap_width) - 1; PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, IAP_PMC0 + ri, v); /* * Write the new value to the counter (or it's alias). The * counter will be in a stopped state when the pcd_write() * entry point is called. */ wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); return (0); } static void iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, int flags) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iap-initialize"); /* Remember the set of architectural events supported. */ core_architectural_events = ~flags; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; pcd->pcd_caps = IAP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iap_allocate_pmc; pcd->pcd_config_pmc = iap_config_pmc; pcd->pcd_describe = iap_describe; pcd->pcd_get_config = iap_get_config; pcd->pcd_get_msr = iap_get_msr; pcd->pcd_pcpu_fini = core_pcpu_fini; pcd->pcd_pcpu_init = core_pcpu_init; pcd->pcd_read_pmc = iap_read_pmc; pcd->pcd_release_pmc = iap_release_pmc; pcd->pcd_start_pmc = iap_start_pmc; pcd->pcd_stop_pmc = iap_stop_pmc; pcd->pcd_write_pmc = iap_write_pmc; md->pmd_npmc += npmc; } static int core_intr(struct trapframe *tf) { pmc_value_t v; struct pmc *pm; struct core_cpu *cc; int error, found_interrupt, ri; uint64_t msr; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", curcpu, (void *) tf, TRAPF_USERMODE(tf)); found_interrupt = 0; cc = core_pcpu[curcpu]; for (ri = 0; ri < core_iap_npmc; ri++) { if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (!iap_pmc_has_overflowed(ri)) continue; found_interrupt = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); v = pm->pm_sc.pm_reloadcount; v = iap_reload_count_to_perfctr_value(v); /* * Stop the counter, reload it but only restart it if * the PMC is not stalled. */ msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + ri, msr); wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); if (error) continue; wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN)); } if (found_interrupt) lapic_reenable_pmc(); if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); return (found_interrupt); } static int core2_intr(struct trapframe *tf) { int error, found_interrupt, n, cpu; uint64_t flag, intrstatus, intrenable, msr; struct pmc *pm; struct core_cpu *cc; pmc_value_t v; cpu = curcpu; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); /* * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which * PMCs have a pending PMI interrupt. We take a 'snapshot' of * the current set of interrupting PMCs and process these * after stopping them. */ intrstatus = rdmsr(IA_GLOBAL_STATUS); intrenable = intrstatus & core_pmcmask; PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, (uintmax_t) intrstatus); found_interrupt = 0; cc = core_pcpu[cpu]; KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); cc->pc_globalctrl &= ~intrenable; cc->pc_resync = 1; /* MSRs now potentially out of sync. */ /* * Stop PMCs and clear overflow status bits. */ msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr); wrmsr(IA_GLOBAL_OVF_CTRL, intrenable | IA_GLOBAL_STATUS_FLAG_OVFBUF | IA_GLOBAL_STATUS_FLAG_CONDCHG); /* * Look for interrupts from fixed function PMCs. */ for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) intrenable &= ~flag; v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); /* Reload sampling count. */ wrmsr(IAF_CTR0 + n, v); PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu, error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); } /* * Process interrupts from the programmable counters. */ for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) intrenable &= ~flag; v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, (uintmax_t) v); /* Reload sampling count. */ wrmsr(core_iap_wroffset + IAP_PMC0 + n, v); } /* * Reenable all non-stalled PMCs. */ PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu, (uintmax_t) intrenable); cc->pc_globalctrl |= intrenable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK); PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx " "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_STATUS), (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL)); if (found_interrupt) lapic_reenable_pmc(); if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); return (found_interrupt); } int pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) { int cpuid[CORE_CPUID_REQUEST_SIZE]; int ipa_version, flags, nflags; do_cpuid(CORE_CPUID_REQUEST, cpuid); ipa_version = (version_override > 0) ? version_override : cpuid[CORE_CPUID_EAX] & 0xFF; core_cputype = md->pmd_cputype; PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d", core_cputype, maxcpu, ipa_version); if (ipa_version < 1 || ipa_version > 4 || (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) { /* Unknown PMC architecture. */ printf("hwpc_core: unknown PMC architecture: %d\n", ipa_version); return (EPROGMISMATCH); } core_iap_wroffset = 0; if (cpu_feature2 & CPUID2_PDCM) { if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) { PMCDBG0(MDP, INI, 1, "core-init full-width write supported"); core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0; } else PMCDBG0(MDP, INI, 1, "core-init full-width write NOT supported"); } else PMCDBG0(MDP, INI, 1, "core-init pdcm not supported"); core_pmcmask = 0; /* * Initialize programmable counters. */ core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF; core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF; core_pmcmask |= ((1ULL << core_iap_npmc) - 1); nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF; flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1); iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags); /* * Initialize fixed function counters, if present. */ if (core_cputype != PMC_CPU_INTEL_CORE) { core_iaf_ri = core_iap_npmc; core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; } PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, core_iaf_ri); core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); /* * Choose the appropriate interrupt handler. */ if (ipa_version == 1) md->pmd_intr = core_intr; else md->pmd_intr = core2_intr; md->pmd_pcpu_fini = NULL; md->pmd_pcpu_init = NULL; return (0); } void pmc_core_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "core-finalize"); free(core_pcpu, M_PMC); core_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_tsc.c b/sys/dev/hwpmc/hwpmc_tsc.c index 6cd098a8113b..d59c8908f4ca 100644 --- a/sys/dev/hwpmc/hwpmc_tsc.c +++ b/sys/dev/hwpmc/hwpmc_tsc.c @@ -1,387 +1,381 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * TSC support. */ #define TSC_CAPS PMC_CAP_READ struct tsc_descr { struct pmc_descr pm_descr; /* "base class" */ }; static struct tsc_descr tsc_pmcdesc[TSC_NPMCS] = { { .pm_descr = { .pd_name = "TSC", .pd_class = PMC_CLASS_TSC, .pd_caps = TSC_CAPS, .pd_width = 64 } } }; /* * Per-CPU data structure for TSCs. */ struct tsc_cpu { struct pmc_hw tc_hw; }; static struct tsc_cpu **tsc_pcpu; static int tsc_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_TSC) return (EINVAL); - if ((pm->pm_caps & TSC_CAPS) == 0) - return (EINVAL); - - if ((pm->pm_caps & ~TSC_CAPS) != 0) - return (EPERM); - if (a->pm_ev != PMC_EV_TSC_TSC || a->pm_mode != PMC_MODE_SC) return (EINVAL); return (0); } static int tsc_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[tsc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int tsc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; size_t copied; const struct tsc_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pd = &tsc_pmcdesc[ri]; if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, PMC_NAME_MAX, &copied)) != 0) return (error); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int tsc_get_config(int cpu, int ri, struct pmc **ppm) { (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); *ppm = tsc_pcpu[cpu]->tc_hw.phw_pmc; return (0); } static int tsc_get_msr(int ri, uint32_t *msr) { (void) ri; KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] ri %d out of range", __LINE__, ri)); *msr = MSR_TSC; return (0); } static int tsc_pcpu_fini(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu[cpu] != NULL, ("[tsc,%d] null pcpu", __LINE__)); free(tsc_pcpu[cpu], M_PMC); tsc_pcpu[cpu] = NULL; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; return (0); } static int tsc_pcpu_init(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; struct tsc_cpu *tsc_pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu, ("[tsc,%d] null pcpu", __LINE__)); KASSERT(tsc_pcpu[cpu] == NULL, ("[tsc,%d] non-null per-cpu", __LINE__)); tsc_pc = malloc(sizeof(struct tsc_cpu), M_PMC, M_WAITOK|M_ZERO); tsc_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | PMC_PHW_FLAG_IS_SHAREABLE; tsc_pcpu[cpu] = tsc_pc; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; KASSERT(pmc_pcpu, ("[tsc,%d] null generic pcpu", __LINE__)); pc = pmc_pcpu[cpu]; KASSERT(pc, ("[tsc,%d] null generic per-cpu", __LINE__)); pc->pc_hwpmcs[ri] = &tsc_pc->tc_hw; return (0); } static int tsc_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; enum pmc_mode mode __diagused; const struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal ri %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[tsc,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); KASSERT(mode == PMC_MODE_SC, ("[tsc,%d] illegal pmc mode %d", __LINE__, mode)); PMCDBG1(MDP,REA,1,"tsc-read id=%d", ri); *v = rdtsc(); return (0); } static int tsc_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(phw->phw_pmc == NULL, ("[tsc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); /* * Nothing to do. */ return (0); } static int tsc_start_pmc(int cpu, int ri) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* TSCs are always running. */ } static int tsc_stop_pmc(int cpu, int ri) { (void) cpu; (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* Cannot actually stop a TSC. */ } static int tsc_write_pmc(int cpu, int ri, pmc_value_t v) { (void) cpu; (void) ri; (void) v; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); /* * The TSCs are used as timecounters by the kernel, so even * though some i386 CPUs support writeable TSCs, we don't * support writing changing TSC values through the HWPMC API. */ return (0); } int pmc_tsc_initialize(struct pmc_mdep *md, int maxcpu) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[tsc,%d] md is NULL", __LINE__)); KASSERT(md->pmd_nclass >= 1, ("[tsc,%d] dubious md->nclass %d", __LINE__, md->pmd_nclass)); tsc_pcpu = malloc(sizeof(struct tsc_cpu *) * maxcpu, M_PMC, M_ZERO|M_WAITOK); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC]; pcd->pcd_caps = PMC_CAP_READ; pcd->pcd_class = PMC_CLASS_TSC; pcd->pcd_num = TSC_NPMCS; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 64; pcd->pcd_allocate_pmc = tsc_allocate_pmc; pcd->pcd_config_pmc = tsc_config_pmc; pcd->pcd_describe = tsc_describe; pcd->pcd_get_config = tsc_get_config; pcd->pcd_get_msr = tsc_get_msr; pcd->pcd_pcpu_init = tsc_pcpu_init; pcd->pcd_pcpu_fini = tsc_pcpu_fini; pcd->pcd_read_pmc = tsc_read_pmc; pcd->pcd_release_pmc = tsc_release_pmc; pcd->pcd_start_pmc = tsc_start_pmc; pcd->pcd_stop_pmc = tsc_stop_pmc; pcd->pcd_write_pmc = tsc_write_pmc; md->pmd_npmc += TSC_NPMCS; return (0); } void pmc_tsc_finalize(struct pmc_mdep *md) { #ifdef INVARIANTS int i, ncpus; ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(tsc_pcpu[i] == NULL, ("[tsc,%d] non-null pcpu cpu %d", __LINE__, i)); KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_class == PMC_CLASS_TSC, ("[tsc,%d] class mismatch", __LINE__)); #else (void) md; #endif free(tsc_pcpu, M_PMC); tsc_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_uncore.c b/sys/dev/hwpmc/hwpmc_uncore.c index 19017cabddd9..adb0e12c7454 100644 --- a/sys/dev/hwpmc/hwpmc_uncore.c +++ b/sys/dev/hwpmc/hwpmc_uncore.c @@ -1,862 +1,853 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Uncore PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if (__FreeBSD_version >= 1100000) #include #else #include #endif #include #include #include #define UCF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE) #define UCP_PMC_CAPS \ (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define SELECTSEL(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCP_CB0_EVSEL0 : UCP_EVSEL0) #define SELECTOFF(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCF_OFFSET_SB : UCF_OFFSET) static enum pmc_cputype uncore_cputype; struct uncore_cpu { volatile uint32_t pc_resync; volatile uint32_t pc_ucfctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_uncorepmcs[]; }; static struct uncore_cpu **uncore_pcpu; static uint64_t uncore_pmcmask; static int uncore_ucf_ri; /* relative index of fixed counters */ static int uncore_ucf_width; static int uncore_ucf_npmc; static int uncore_ucp_width; static int uncore_ucp_npmc; static int uncore_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int uncore_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct uncore_cpu *cc; struct pmc_hw *phw; int uncore_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); uncore_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + uncore_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + uncore_ri] = phw; } return (0); } static int uncore_pcpu_fini(struct pmc_mdep *md, int cpu) { int uncore_ri, n, npmc; struct pmc_cpu *pc; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); if ((cc = uncore_pcpu[cpu]) == NULL) return (0); uncore_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(SELECTSEL(uncore_cputype) + n, 0); wrmsr(UCF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + uncore_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t ucf_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucf_width) - 1; return (1ULL << uncore_ucf_width) - v; } static pmc_value_t ucf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucf_width) - rlc; } static int ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { - uint32_t caps, flags; + uint32_t flags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > uncore_ucf_npmc) return (EINVAL); - caps = a->pm_caps; - - if (a->pm_class != PMC_CLASS_UCF || - (caps & UCF_PMC_CAPS) != caps) + if (a->pm_class != PMC_CLASS_UCF) return (EINVAL); flags = UCF_EN; pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); return (0); } static int ucf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; return (0); } static int ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucf_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; return (0); } static int ucf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + uncore_ucf_ri)); tmp = rdmsr(UCF_CTR0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); return (0); } static int ucf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); } static void ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; pcd->pcd_caps = UCF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucf_allocate_pmc; pcd->pcd_config_pmc = ucf_config_pmc; pcd->pcd_describe = ucf_describe; pcd->pcd_get_config = ucf_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_noop; pcd->pcd_pcpu_init = uncore_pcpu_noop; pcd->pcd_read_pmc = ucf_read_pmc; pcd->pcd_release_pmc = ucf_release_pmc; pcd->pcd_start_pmc = ucf_start_pmc; pcd->pcd_stop_pmc = ucf_stop_pmc; pcd->pcd_write_pmc = ucf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct ucp_event_descr { enum pmc_event ucp_ev; unsigned char ucp_evcode; unsigned char ucp_umask; unsigned char ucp_flags; }; #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ #define UCP_F_WM (1 << 1) /* CPU: Westmere */ #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ #define UCP_F_HW (1 << 3) /* CPU: Haswell */ #define UCP_F_FM (1 << 4) /* Fixed mask */ #define UCP_F_ALLCPUS \ (UCP_F_I7 | UCP_F_WM) #define UCP_F_CMASK 0xFF000000 static pmc_value_t ucp_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucp_width) - 1; return (1ULL << uncore_ucp_width) - v; } static pmc_value_t ucp_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucp_width) - rlc; } /* * Counter specific event information for Sandybridge and Haswell */ static int ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) { uint32_t mask; switch (ev) { /* * Events valid only on counter 0. */ case 0x80: case 0x83: mask = (1 << 0); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; - uint32_t caps; const struct pmc_md_ucp_op_pmcallocate *ucp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_UCP) return (EINVAL); - /* check requested capabilities */ - caps = a->pm_caps; - if ((UCP_PMC_CAPS & caps) != caps) - return (EPERM); - ucp = &a->pm_md.pm_ucp; ev = UCP_EVSEL(ucp->pm_ucp_config); switch (uncore_cputype) { case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_SANDYBRIDGE: if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: break; } pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; return (0); } static int ucp_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; return (0); } static int ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucp_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucp_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; return (0); } static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); } static int ucp_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; uint32_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); /* Event specific configuration. */ switch (pm->pm_event) { case PMC_EV_UCP_EVENT_0CH_04H_E: case PMC_EV_UCP_EVENT_0CH_08H_E: wrmsr(MSR_GQ_SNOOP_MESF,0x2); break; case PMC_EV_UCP_EVENT_0CH_04H_F: case PMC_EV_UCP_EVENT_0CH_08H_F: wrmsr(MSR_GQ_SNOOP_MESF,0x8); break; case PMC_EV_UCP_EVENT_0CH_04H_M: case PMC_EV_UCP_EVENT_0CH_08H_M: wrmsr(MSR_GQ_SNOOP_MESF,0x1); break; case PMC_EV_UCP_EVENT_0CH_04H_S: case PMC_EV_UCP_EVENT_0CH_08H_S: wrmsr(MSR_GQ_SNOOP_MESF,0x4); break; default: break; } wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); do { cc->pc_resync = 0; cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int ucp_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); /* stop hw. */ wrmsr(SELECTSEL(uncore_cputype) + ri, 0); do { cc->pc_resync = 0; cc->pc_globalctrl &= ~(1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); } static void ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucp-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; pcd->pcd_caps = UCP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucp_allocate_pmc; pcd->pcd_config_pmc = ucp_config_pmc; pcd->pcd_describe = ucp_describe; pcd->pcd_get_config = ucp_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_fini; pcd->pcd_pcpu_init = uncore_pcpu_init; pcd->pcd_read_pmc = ucp_read_pmc; pcd->pcd_release_pmc = ucp_release_pmc; pcd->pcd_start_pmc = ucp_start_pmc; pcd->pcd_stop_pmc = ucp_stop_pmc; pcd->pcd_write_pmc = ucp_write_pmc; md->pmd_npmc += npmc; } int pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) { uncore_cputype = md->pmd_cputype; uncore_pmcmask = 0; /* * Initialize programmable counters. */ uncore_ucp_npmc = 8; uncore_ucp_width = 48; uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); /* * Initialize fixed function counters, if present. */ uncore_ucf_ri = uncore_ucp_npmc; uncore_ucf_npmc = 1; uncore_ucf_width = 48; ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, uncore_ucf_ri); uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); return (0); } void pmc_uncore_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "uncore-finalize"); free(uncore_pcpu, M_PMC); uncore_pcpu = NULL; }