diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 2b5356924ae6..05584ee1660d 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -1,1245 +1,1240 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* Support for the AMD K7 and later processors */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_DEBUG enum pmc_class amd_pmc_class; #endif #define OVERFLOW_WAIT_COUNT 50 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); /* AMD K7 & K8 PMCs */ struct amd_descr { struct pmc_descr pm_descr; /* "base class" */ uint32_t pm_evsel; /* address of EVSEL register */ uint32_t pm_perfctr; /* address of PERFCTR register */ }; static struct amd_descr amd_pmcdesc[AMD_NPMCS] = { { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_0, .pm_perfctr = AMD_PMC_PERFCTR_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_1, .pm_perfctr = AMD_PMC_PERFCTR_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_2, .pm_perfctr = AMD_PMC_PERFCTR_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_3, .pm_perfctr = AMD_PMC_PERFCTR_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_4, .pm_perfctr = AMD_PMC_PERFCTR_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_5, .pm_perfctr = AMD_PMC_PERFCTR_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_4, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_5, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_3 } }; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ uint16_t pe_code; /* encoded event mask */ uint8_t pe_mask; /* bits allowed in unit mask */ }; const struct amd_event_code_map amd_event_codes[] = { #if defined(__i386__) /* 32 bit Athlon (K7) only */ { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, { PMC_EV_K7_DC_MISSES, 0x41, 0 }, { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, { PMC_EV_K7_IC_MISSES, 0x81, 0 }, { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, #endif { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 0x21, 0x00 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 0xCA, 0x00 }, { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 0xCC, 0x07 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 0xD2, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 0xD5, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 0xD6, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 0xD9, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 0xDA, 0x00 }, { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 0xE2, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } }; const int amd_event_codes_size = nitems(amd_event_codes); /* * Per-processor information */ struct amd_cpu { struct pmc_hw pc_amdpmcs[AMD_NPMCS]; }; static struct amd_cpu **amd_pcpu; /* * read a pmc register */ static int amd_read_pmc(int cpu, int ri, pmc_value_t *v) { enum pmc_mode mode; const struct amd_descr *pd; struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); KASSERT(amd_pcpu[cpu], ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(mode)) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ULL << 47)) == 0) tmp = 0; else { /* Sign extend 48 bit value to 64 bits. */ tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16); tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } } *v = tmp; PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v); return 0; } /* * Write a PMC MSR. */ static int amd_write_pmc(int cpu, int ri, pmc_value_t v) { const struct amd_descr *pd; enum pmc_mode mode; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif /* use 2's complement of the count for sampling mode PMCs */ if (PMC_IS_SAMPLING_MODE(mode)) v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); /* write the PMC value */ wrmsr(pd->pm_perfctr, v); return 0; } /* * configure hardware pmc according to the configuration recorded in * pmc 'pm'. */ static int amd_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } /* * Retrieve a configured PMC pointer from hardware state. */ static int amd_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; return 0; } /* * Machine dependent actions taken during the context switch in of a * thread. */ static int amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); /* enable the RDPMC instruction if needed */ if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) load_cr4(rcr4() | CR4_PCE); return 0; } /* * Machine dependent actions taken during the context switch out of a * thread. */ static int amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; (void) pp; /* can be NULL */ PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); /* always turn off the RDPMC instruction */ load_cr4(rcr4() & ~CR4_PCE); return 0; } /* * Check if a given allocation is feasible. */ static int amd_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int i; uint64_t allowed_unitmask, caps, config, unitmask; enum pmc_event pe; const struct pmc_descr *pd; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri].pm_descr; /* check class match */ if (pd->pd_class != a->pm_class) return EINVAL; caps = pm->pm_caps; PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); if((ri >= 0 && ri < 6) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_CORE)) return EINVAL; if((ri >= 6 && ri < 12) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_L3_CACHE)) return EINVAL; if((ri >= 12 && ri < 16) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_DATA_FABRIC)) return EINVAL; if (strlen(pmc_cpuid) != 0) { pm->pm_md.pm_amd.pm_amd_evsel = a->pm_md.pm_amd.pm_amd_config; PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, a->pm_md.pm_amd.pm_amd_config); return (0); } pe = a->pm_ev; /* map ev to the correct event mask code */ config = allowed_unitmask = 0; for (i = 0; i < amd_event_codes_size; i++) if (amd_event_codes[i].pe_ev == pe) { config = AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); allowed_unitmask = AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); break; } if (i == amd_event_codes_size) return EINVAL; unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ return EINVAL; if (unitmask && (caps & PMC_CAP_QUALIFIER)) config |= unitmask; if (caps & PMC_CAP_THRESHOLD) config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; /* set at least one of the 'usr' or 'os' caps */ if (caps & PMC_CAP_USER) config |= AMD_PMC_USR; if (caps & PMC_CAP_SYSTEM) config |= AMD_PMC_OS; if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) config |= (AMD_PMC_USR|AMD_PMC_OS); if (caps & PMC_CAP_EDGE) config |= AMD_PMC_EDGE; if (caps & PMC_CAP_INVERT) config |= AMD_PMC_INVERT; if (caps & PMC_CAP_INTERRUPT) config |= AMD_PMC_INT; pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); return 0; } /* * Release machine dependent state associated with a PMC. This is a * no-op on this architecture. * */ /* ARGSUSED0 */ static int amd_release_pmc(int cpu, int ri, struct pmc *pmc) { #ifdef HWPMC_DEBUG const struct amd_descr *pd; #endif struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); #ifdef HWPMC_DEBUG pd = &amd_pmcdesc[ri]; if (pd->pm_descr.pd_class == amd_pmc_class) KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC %d released while active", __LINE__, ri)); #endif return 0; } /* * start a PMC. */ static int amd_start_pmc(int cpu, int ri) { uint64_t config; struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, ri, cpu, pd->pm_descr.pd_name)); /* turn on the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config); wrmsr(pd->pm_evsel, config); return 0; } /* * Stop a PMC. */ static int amd_stop_pmc(int cpu, int ri) { struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; uint64_t config; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", __LINE__, ri, cpu, pd->pm_descr.pd_name)); PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri); /* turn off the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; wrmsr(pd->pm_evsel, config); /* * Due to NMI latency on newer AMD processors * NMI interrupts are ignored, which leads to * panic or messages based on kernel configuraiton */ /* Wait for the count to be reset */ for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) break; DELAY(1); } return 0; } /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed * to sleep or do anything a 'fast' interrupt handler is not allowed * to do. */ static int amd_intr(struct trapframe *tf) { int i, error, retval, cpu; uint64_t config, evsel, perfctr; struct pmc *pm; struct amd_cpu *pac; pmc_value_t v; uint32_t active = 0, count = 0; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pac = amd_pcpu[cpu]; /* * look for all PMCs that have interrupted: * - look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association * * If found, we call a helper to process the interrupt. * * PMCs interrupting at the same time are collapsed into * a single interrupt. Check all the valid pmcs for * overflow. */ for (i = 0; i < AMD_CORE_NPMCS; i++) { if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; } /* Consider pmc with valid handle as active */ active++; if (!AMD_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* Stop the PMC, reload count. */ evsel = amd_pmcdesc[i].pm_evsel; perfctr = amd_pmcdesc[i].pm_perfctr; v = pm->pm_sc.pm_reloadcount; config = rdmsr(evsel); KASSERT((config & ~AMD_PMC_ENABLE) == (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); wrmsr(evsel, config & ~AMD_PMC_ENABLE); wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); /* Restart the counter if logging succeeded. */ error = pmc_process_interrupt(PMC_HR, pm, tf); if (error == 0) wrmsr(evsel, config); } /* * Due to NMI latency, there can be a scenario in which * multiple pmcs gets serviced in an earlier NMI and we * do not find an overflow in the subsequent NMI. * * For such cases we keep a per-cpu count of active NMIs * and compare it with min(active pmcs, 2) to determine * if this NMI was for a pmc overflow which was serviced * in an earlier request or should be ignored. */ if (retval) { DPCPU_SET(nmi_counter, min(2, active)); } else { if ((count = DPCPU_GET(nmi_counter))) { retval = 1; DPCPU_SET(nmi_counter, --count); } } if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); PMCDBG1(MDP,INT,2, "retval=%d", retval); return (retval); } /* * describe a PMC */ static int amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct amd_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] row-index %d out of range", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pd = &amd_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return error; - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return 0; } /* * i386 specific entry points */ /* * return the MSR address of the given PMC. */ static int amd_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] ri %d out of range", __LINE__, ri)); *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; return (0); } /* * processor dependent initialization. */ static int amd_pcpu_init(struct pmc_mdep *md, int cpu) { int classindex, first_ri, n; struct pmc_cpu *pc; struct amd_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu); amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, M_WAITOK|M_ZERO); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * processor dependent cleanup prior to the KLD * being unloaded */ static int amd_pcpu_fini(struct pmc_mdep *md, int cpu) { int classindex, first_ri, i; uint32_t evsel; struct pmc_cpu *pc; struct amd_cpu *pac; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu); /* * First, turn off all PMCs on this CPU. */ for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ evsel = rdmsr(AMD_PMC_EVSEL_0 + i); evsel &= ~AMD_PMC_ENABLE; wrmsr(AMD_PMC_EVSEL_0 + i, evsel); } /* * Next, free up allocated space. */ if ((pac = amd_pcpu[cpu]) == NULL) return (0); amd_pcpu[cpu] = NULL; #ifdef HWPMC_DEBUG for (i = 0; i < AMD_NPMCS; i++) { KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); } #endif pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; /* * Reset pointers in the MI 'per-cpu' state. */ for (i = 0; i < AMD_NPMCS; i++) { pc->pc_hwpmcs[i + first_ri] = NULL; } free(pac, M_PMC); return (0); } /* * Initialize ourselves. */ struct pmc_mdep * pmc_amd_initialize(void) { int classindex, error, i, ncpus; struct pmc_classdep *pcd; enum pmc_cputype cputype; struct pmc_mdep *pmc_mdep; enum pmc_class class; int family, model, stepping; char *name; /* * The presence of hardware performance counters on the AMD * Athlon, Duron or later processors, is _not_ indicated by * any of the processor feature flags set by the 'CPUID' * instruction, so we only check the 'instruction family' * field returned by CPUID for instruction family >= 6. */ name = NULL; family = CPUID_TO_FAMILY(cpu_id); model = CPUID_TO_MODEL(cpu_id); stepping = CPUID_TO_STEPPING(cpu_id); if (family == 0x18) snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", family, model, stepping); else snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", family, model, stepping); switch (cpu_id & 0xF00) { #if defined(__i386__) case 0x600: /* Athlon(tm) processor */ classindex = PMC_MDEP_CLASS_INDEX_K7; cputype = PMC_CPU_AMD_K7; class = PMC_CLASS_K7; name = "K7"; break; #endif case 0xF00: /* Athlon64/Opteron processor */ classindex = PMC_MDEP_CLASS_INDEX_K8; cputype = PMC_CPU_AMD_K8; class = PMC_CLASS_K8; name = "K8"; break; default: (void) printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, (cpu_id & 0xF00) >> 8, model); return NULL; } #ifdef HWPMC_DEBUG amd_pmc_class = class; #endif /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* * These processors have two classes of PMCs: the TSC and * programmable PMCs. */ pmc_mdep = pmc_mdep_alloc(2); pmc_mdep->pmd_cputype = cputype; ncpus = pmc_cpu_max(); /* Initialize the TSC. */ error = pmc_tsc_initialize(pmc_mdep, ncpus); if (error) goto error; /* Initialize AMD K7 and K8 PMC handling. */ pcd = &pmc_mdep->pmd_classdep[classindex]; pcd->pcd_caps = AMD_PMC_CAPS; pcd->pcd_class = class; pcd->pcd_num = AMD_NPMCS; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 48; /* fill in the correct pmc name and class */ for (i = 0; i < AMD_NPMCS; i++) { (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", name, i); amd_pmcdesc[i].pm_descr.pd_class = class; } pcd->pcd_allocate_pmc = amd_allocate_pmc; pcd->pcd_config_pmc = amd_config_pmc; pcd->pcd_describe = amd_describe; pcd->pcd_get_config = amd_get_config; pcd->pcd_get_msr = amd_get_msr; pcd->pcd_pcpu_fini = amd_pcpu_fini; pcd->pcd_pcpu_init = amd_pcpu_init; pcd->pcd_read_pmc = amd_read_pmc; pcd->pcd_release_pmc = amd_release_pmc; pcd->pcd_start_pmc = amd_start_pmc; pcd->pcd_stop_pmc = amd_stop_pmc; pcd->pcd_write_pmc = amd_write_pmc; pmc_mdep->pmd_pcpu_init = NULL; pmc_mdep->pmd_pcpu_fini = NULL; pmc_mdep->pmd_intr = amd_intr; pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; pmc_mdep->pmd_npmc += AMD_NPMCS; PMCDBG0(MDP,INI,0,"amd-initialize"); return (pmc_mdep); error: if (error) { free(pmc_mdep, M_PMC); pmc_mdep = NULL; } return (NULL); } /* * Finalization code for AMD CPUs. */ void pmc_amd_finalize(struct pmc_mdep *md) { #if defined(INVARIANTS) int classindex, i, ncpus, pmcclass; #endif pmc_tsc_finalize(md); KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer", __LINE__)); #if defined(INVARIANTS) switch (md->pmd_cputype) { #if defined(__i386__) case PMC_CPU_AMD_K7: classindex = PMC_MDEP_CLASS_INDEX_K7; pmcclass = PMC_CLASS_K7; break; #endif default: classindex = PMC_MDEP_CLASS_INDEX_K8; pmcclass = PMC_CLASS_K8; } KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass, ("[amd,%d] pmc class mismatch", __LINE__)); ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu", __LINE__)); #endif free(amd_pcpu, M_PMC); amd_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_arm64.c b/sys/dev/hwpmc/hwpmc_arm64.c index 80d97070b913..a89f26f90dd0 100644 --- a/sys/dev/hwpmc/hwpmc_arm64.c +++ b/sys/dev/hwpmc/hwpmc_arm64.c @@ -1,632 +1,629 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by the University of Cambridge Computer * Laboratory with support from ARM Ltd. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int arm64_npmcs; struct arm64_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; /* * Per-processor information. */ struct arm64_cpu { struct pmc_hw *pc_arm64pmcs; }; static struct arm64_cpu **arm64_pcpu; /* * Interrupt Enable Set Register */ static __inline void arm64_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenset_el1, reg); isb(); } /* * Interrupt Clear Set Register */ static __inline void arm64_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenclr_el1, reg); isb(); } /* * Counter Set Enable Register */ static __inline void arm64_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenset_el0, reg); isb(); } /* * Counter Clear Enable Register */ static __inline void arm64_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenclr_el0, reg); isb(); } /* * Performance Monitors Control Register */ static uint32_t arm64_pmcr_read(void) { uint32_t reg; reg = READ_SPECIALREG(pmcr_el0); return (reg); } static void arm64_pmcr_write(uint32_t reg) { WRITE_SPECIALREG(pmcr_el0, reg); isb(); } /* * Performance Count Register N */ static uint32_t arm64_pmcn_read(unsigned int pmc) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); isb(); return (READ_SPECIALREG(pmxevcntr_el0)); } static void arm64_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); WRITE_SPECIALREG(pmxevcntr_el0, reg); isb(); } static int arm64_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t config; struct arm64_cpu *pac; enum pmc_event pe; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pac = arm64_pcpu[cpu]; if (a->pm_class != PMC_CLASS_ARMV8) { return (EINVAL); } pe = a->pm_ev; /* Adjust the config value if needed. */ config = a->pm_md.pm_md_config; if ((a->pm_md.pm_md_flags & PM_MD_RAW_EVENT) == 0) { config = (uint32_t)pe - PMC_EV_ARMV8_FIRST; if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST)) return (EINVAL); } switch (a->pm_caps & (PMC_CAP_SYSTEM | PMC_CAP_USER)) { case PMC_CAP_SYSTEM: config |= PMEVTYPER_U; break; case PMC_CAP_USER: config |= PMEVTYPER_P; break; default: /* * Trace both USER and SYSTEM if none are specified * (default setting) or if both flags are specified * (user explicitly requested both qualifiers). */ break; } pm->pm_md.pm_arm64.pm_arm64_evsel = config; PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config); return (0); } static int arm64_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; /* * Ensure we don't get interrupted while updating the overflow count. */ s = intr_disable(); tmp = arm64_pmcn_read(ri); reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) { /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = arm64_pmcn_read(ri); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ull << 63)) == 0) tmp = 0; else tmp = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } *v = tmp; return (0); } static int arm64_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; arm64_pmcn_write(ri, v); return (0); } static int arm64_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int arm64_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_arm64.pm_arm64_evsel; /* * Configure the event selection. */ WRITE_SPECIALREG(pmselr_el0, ri); WRITE_SPECIALREG(pmxevtyper_el0, config); isb(); /* * Enable the PMC. */ arm64_interrupt_enable(ri); arm64_counter_enable(ri); return (0); } static int arm64_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. */ arm64_counter_disable(ri); arm64_interrupt_disable(ri); return (0); } static int arm64_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } static int arm64_intr(struct trapframe *tf) { struct arm64_cpu *pc; int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] CPU %d out of range", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf, TRAPF_USERMODE(tf)); retval = 0; pc = arm64_pcpu[cpu]; for (ri = 0; ri < arm64_npmcs; ri++) { pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter is overflowed */ reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0) continue; /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); isb(); retval = 1; /* Found an interrupting PMC. */ pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) arm64_stop_pmc(cpu, ri); /* Reload sampling count */ arm64_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - char arm64_name[PMC_NAME_MAX]; struct pmc_hw *phw; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] row-index %d out of range", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; - snprintf(arm64_name, sizeof(arm64_name), "ARMV8-%d", ri); - if ((error = copystr(arm64_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); + + snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV8-%d", ri); pi->pm_class = PMC_CLASS_ARMV8; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int arm64_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; return (0); } /* * XXX don't know what we should do here. */ static int arm64_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_pcpu_init(struct pmc_mdep *md, int cpu) { struct arm64_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint64_t pmcr; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "arm64-init cpu=%d", cpu); arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC, M_WAITOK | M_ZERO); pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs, M_PMC, M_WAITOK | M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri; KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* * Disable all counters and overflow interrupts. Upon reset they are in * an undefined state. * * Don't issue an isb here, just wait for the one in arm64_pmcr_write() * to make the writes visible. */ WRITE_SPECIALREG(pmcntenclr_el0, 0xffffffff); WRITE_SPECIALREG(pmintenclr_el1, 0xffffffff); /* Enable unit */ pmcr = arm64_pmcr_read(); pmcr |= PMCR_E; arm64_pmcr_write(pmcr); return (0); } static int arm64_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmcr; pmcr = arm64_pmcr_read(); pmcr &= ~PMCR_E; arm64_pmcr_write(pmcr); return (0); } struct pmc_mdep * pmc_arm64_initialize(void) { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode, impcode; int reg; uint64_t midr; reg = arm64_pmcr_read(); arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT; impcode = (reg & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT; idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs); /* * Write the CPU model to kern.hwpmc.cpuid. * * We zero the variant and revision fields. * * TODO: how to handle differences between cores due to big.LITTLE? * For now, just use MIDR from CPU 0. */ midr = (uint64_t)(pcpu_find(0)->pc_midr); midr &= ~(CPU_VAR_MASK | CPU_REV_MASK); snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch(impcode) { case PMCR_IMP_ARM: switch (idcode) { case PMCR_IDCODE_CORTEX_A76: case PMCR_IDCODE_NEOVERSE_N1: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76; break; case PMCR_IDCODE_CORTEX_A57: case PMCR_IDCODE_CORTEX_A72: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57; break; default: case PMCR_IDCODE_CORTEX_A53: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } break; default: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8]; pcd->pcd_caps = ARMV8_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV8; pcd->pcd_num = arm64_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = arm64_allocate_pmc; pcd->pcd_config_pmc = arm64_config_pmc; pcd->pcd_pcpu_fini = arm64_pcpu_fini; pcd->pcd_pcpu_init = arm64_pcpu_init; pcd->pcd_describe = arm64_describe; pcd->pcd_get_config = arm64_get_config; pcd->pcd_read_pmc = arm64_read_pmc; pcd->pcd_release_pmc = arm64_release_pmc; pcd->pcd_start_pmc = arm64_start_pmc; pcd->pcd_stop_pmc = arm64_stop_pmc; pcd->pcd_write_pmc = arm64_write_pmc; pmc_mdep->pmd_intr = arm64_intr; pmc_mdep->pmd_switch_in = arm64_switch_in; pmc_mdep->pmd_switch_out = arm64_switch_out; pmc_mdep->pmd_npmc += arm64_npmcs; return (pmc_mdep); } void pmc_arm64_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_armv7.c b/sys/dev/hwpmc/hwpmc_armv7.c index cd4761df46be..2cfac1e857bd 100644 --- a/sys/dev/hwpmc/hwpmc_armv7.c +++ b/sys/dev/hwpmc/hwpmc_armv7.c @@ -1,575 +1,572 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int armv7_npmcs; struct armv7_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; #define PMC_EV_CPU_CYCLES 0xFF /* * Per-processor information. */ struct armv7_cpu { struct pmc_hw *pc_armv7pmcs; }; static struct armv7_cpu **armv7_pcpu; /* * Interrupt Enable Set Register */ static __inline void armv7_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_set(reg); } /* * Interrupt Clear Set Register */ static __inline void armv7_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_clr(reg); } /* * Counter Set Enable Register */ static __inline void armv7_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_set(reg); } /* * Counter Clear Enable Register */ static __inline void armv7_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_clr(reg); } /* * Performance Count Register N */ static uint32_t armv7_pmcn_read(unsigned int pmc, uint32_t evsel) { if (evsel == PMC_EV_CPU_CYCLES) { return ((uint32_t)cp15_pmccntr_get()); } KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); return (cp15_pmxevcntr_get()); } static uint32_t armv7_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); cp15_pmxevcntr_set(reg); return (reg); } static int armv7_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { struct armv7_cpu *pac; enum pmc_event pe; uint32_t config; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); pac = armv7_pcpu[cpu]; if (a->pm_class != PMC_CLASS_ARMV7) return (EINVAL); pe = a->pm_ev; config = (pe & EVENT_ID_MASK); pm->pm_md.pm_armv7.pm_armv7_evsel = config; PMCDBG2(MDP, ALL, 2, "armv7-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int armv7_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; u_int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; s = intr_disable(); tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) != 0) { /* Clear Overflow Flag */ cp15_pmovsr_set(reg); pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "armv7-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ull << 63)) == 0) tmp = 0; else tmp = ARMV7_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } *v = tmp; return 0; } static int armv7_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV7_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "armv7-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) cp15_pmccntr_set(v); else armv7_pmcn_write(ri, v); return 0; } static int armv7_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[armv7,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int armv7_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; /* * Configure the event selection. */ if (config != PMC_EV_CPU_CYCLES) { cp15_pmselr_set(ri); cp15_pmxevtyper_set(config); } else ri = 31; /* * Enable the PMC. */ armv7_interrupt_enable(ri); armv7_counter_enable(ri); return 0; } static int armv7_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; uint32_t config; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; if (config == PMC_EV_CPU_CYCLES) ri = 31; /* * Disable the PMCs. */ armv7_counter_disable(ri); armv7_interrupt_disable(ri); return 0; } static int armv7_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[armv7,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int armv7_intr(struct trapframe *tf) { struct armv7_cpu *pc; int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; pc = armv7_pcpu[cpu]; for (ri = 0; ri < armv7_npmcs; ri++) { pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) == 0) { continue; } /* Clear Overflow Flag */ cp15_pmovsr_set(reg); retval = 1; /* Found an interrupting PMC. */ pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) armv7_stop_pmc(cpu, ri); /* Reload sampling count */ armv7_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int armv7_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - char armv7_name[PMC_NAME_MAX]; struct pmc_hw *phw; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] row-index %d out of range", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; - snprintf(armv7_name, sizeof(armv7_name), "ARMV7-%d", ri); - if ((error = copystr(armv7_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV7-%d", ri); pi->pm_class = PMC_CLASS_ARMV7; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int armv7_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; return 0; } /* * XXX don't know what we should do here. */ static int armv7_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_pcpu_init(struct pmc_mdep *md, int cpu) { struct armv7_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint32_t pmnc; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "armv7-init cpu=%d", cpu); armv7_pcpu[cpu] = pac = malloc(sizeof(struct armv7_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_armv7pmcs = malloc(sizeof(struct pmc_hw) * armv7_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7].pcd_ri; KASSERT(pc != NULL, ("[armv7,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_armv7pmcs; i < armv7_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); /* Enable unit */ pmnc = cp15_pmcr_get(); pmnc |= ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); return 0; } static int armv7_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmnc; pmnc = cp15_pmcr_get(); pmnc &= ~ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); return 0; } struct pmc_mdep * pmc_armv7_initialize(void) { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode; int reg; reg = cp15_pmcr_get(); armv7_npmcs = (reg >> ARMV7_PMNC_N_SHIFT) & \ ARMV7_PMNC_N_MASK; idcode = (reg & ARMV7_IDCODE_MASK) >> ARMV7_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "armv7-init npmcs=%d", armv7_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ armv7_pcpu = malloc(sizeof(struct armv7_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch (idcode) { case ARMV7_IDCODE_CORTEX_A9: pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A9; break; default: case ARMV7_IDCODE_CORTEX_A8: /* * On A8 we implemented common events only, * so use it for the rest of machines. */ pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A8; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7]; pcd->pcd_caps = ARMV7_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV7; pcd->pcd_num = armv7_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = armv7_allocate_pmc; pcd->pcd_config_pmc = armv7_config_pmc; pcd->pcd_pcpu_fini = armv7_pcpu_fini; pcd->pcd_pcpu_init = armv7_pcpu_init; pcd->pcd_describe = armv7_describe; pcd->pcd_get_config = armv7_get_config; pcd->pcd_read_pmc = armv7_read_pmc; pcd->pcd_release_pmc = armv7_release_pmc; pcd->pcd_start_pmc = armv7_start_pmc; pcd->pcd_stop_pmc = armv7_stop_pmc; pcd->pcd_write_pmc = armv7_write_pmc; pmc_mdep->pmd_intr = armv7_intr; pmc_mdep->pmd_switch_in = armv7_switch_in; pmc_mdep->pmd_switch_out = armv7_switch_out; pmc_mdep->pmd_npmc += armv7_npmcs; return (pmc_mdep); } void pmc_armv7_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_beri.c b/sys/dev/hwpmc/hwpmc_beri.c index 4fec1b950299..ce15bb57fda4 100644 --- a/sys/dev/hwpmc/hwpmc_beri.c +++ b/sys/dev/hwpmc/hwpmc_beri.c @@ -1,540 +1,536 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Ruslan Bukin * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" #include #include #include #define BERI_NCOUNTERS 56 #define BERI_PMC_CAPS (PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_READ | PMC_CAP_WRITE ) struct beri_event_code_map { uint32_t pe_ev; /* enum value */ uint64_t (*get_func)(void); }; const struct beri_event_code_map beri_event_codes[BERI_NCOUNTERS] = { { PMC_EV_BERI_CYCLE, statcounters_get_cycle_count }, { PMC_EV_BERI_INST, statcounters_get_inst_count }, { PMC_EV_BERI_INST_USER, statcounters_get_inst_user_count }, { PMC_EV_BERI_INST_KERNEL, statcounters_get_inst_kernel_count }, { PMC_EV_BERI_IMPRECISE_SETBOUNDS, statcounters_get_imprecise_setbounds_count }, { PMC_EV_BERI_UNREPRESENTABLE_CAPS, statcounters_get_unrepresentable_caps_count }, { PMC_EV_BERI_ITLB_MISS, statcounters_get_itlb_miss_count }, { PMC_EV_BERI_DTLB_MISS, statcounters_get_dtlb_miss_count }, { PMC_EV_BERI_ICACHE_WRITE_HIT, statcounters_get_icache_write_hit_count }, { PMC_EV_BERI_ICACHE_WRITE_MISS, statcounters_get_icache_write_miss_count }, { PMC_EV_BERI_ICACHE_READ_HIT, statcounters_get_icache_read_hit_count }, { PMC_EV_BERI_ICACHE_READ_MISS, statcounters_get_icache_read_miss_count }, { PMC_EV_BERI_ICACHE_EVICT, statcounters_get_icache_evict_count }, { PMC_EV_BERI_DCACHE_WRITE_HIT, statcounters_get_dcache_write_hit_count }, { PMC_EV_BERI_DCACHE_WRITE_MISS, statcounters_get_dcache_write_miss_count }, { PMC_EV_BERI_DCACHE_READ_HIT, statcounters_get_dcache_read_hit_count }, { PMC_EV_BERI_DCACHE_READ_MISS, statcounters_get_dcache_read_miss_count }, { PMC_EV_BERI_DCACHE_EVICT, statcounters_get_dcache_evict_count }, { PMC_EV_BERI_DCACHE_SET_TAG_WRITE, statcounters_get_dcache_set_tag_write_count }, { PMC_EV_BERI_DCACHE_SET_TAG_READ, statcounters_get_dcache_set_tag_read_count }, { PMC_EV_BERI_L2CACHE_WRITE_HIT, statcounters_get_l2cache_write_hit_count }, { PMC_EV_BERI_L2CACHE_WRITE_MISS, statcounters_get_l2cache_write_miss_count }, { PMC_EV_BERI_L2CACHE_READ_HIT, statcounters_get_l2cache_read_hit_count }, { PMC_EV_BERI_L2CACHE_READ_MISS, statcounters_get_l2cache_read_miss_count }, { PMC_EV_BERI_L2CACHE_EVICT, statcounters_get_l2cache_evict_count }, { PMC_EV_BERI_L2CACHE_SET_TAG_WRITE, statcounters_get_l2cache_set_tag_write_count }, { PMC_EV_BERI_L2CACHE_SET_TAG_READ, statcounters_get_l2cache_set_tag_read_count }, { PMC_EV_BERI_MEM_BYTE_READ, statcounters_get_mem_byte_read_count }, { PMC_EV_BERI_MEM_BYTE_WRITE, statcounters_get_mem_byte_write_count }, { PMC_EV_BERI_MEM_HWORD_READ, statcounters_get_mem_hword_read_count }, { PMC_EV_BERI_MEM_HWORD_WRITE, statcounters_get_mem_hword_write_count }, { PMC_EV_BERI_MEM_WORD_READ, statcounters_get_mem_word_read_count }, { PMC_EV_BERI_MEM_WORD_WRITE, statcounters_get_mem_word_write_count }, { PMC_EV_BERI_MEM_DWORD_READ, statcounters_get_mem_dword_read_count }, { PMC_EV_BERI_MEM_DWORD_WRITE, statcounters_get_mem_dword_write_count }, { PMC_EV_BERI_MEM_CAP_READ, statcounters_get_mem_cap_read_count }, { PMC_EV_BERI_MEM_CAP_WRITE, statcounters_get_mem_cap_write_count }, { PMC_EV_BERI_MEM_CAP_READ_TAG_SET, statcounters_get_mem_cap_read_tag_set_count }, { PMC_EV_BERI_MEM_CAP_WRITE_TAG_SET, statcounters_get_mem_cap_write_tag_set_count }, { PMC_EV_BERI_TAGCACHE_WRITE_HIT, statcounters_get_tagcache_write_hit_count }, { PMC_EV_BERI_TAGCACHE_WRITE_MISS, statcounters_get_tagcache_write_miss_count }, { PMC_EV_BERI_TAGCACHE_READ_HIT, statcounters_get_tagcache_read_hit_count }, { PMC_EV_BERI_TAGCACHE_READ_MISS, statcounters_get_tagcache_read_miss_count }, { PMC_EV_BERI_TAGCACHE_EVICT, statcounters_get_tagcache_evict_count }, { PMC_EV_BERI_L2CACHEMASTER_READ_REQ, statcounters_get_l2cachemaster_read_req_count }, { PMC_EV_BERI_L2CACHEMASTER_WRITE_REQ, statcounters_get_l2cachemaster_write_req_count }, { PMC_EV_BERI_L2CACHEMASTER_WRITE_REQ_FLIT, statcounters_get_l2cachemaster_write_req_flit_count }, { PMC_EV_BERI_L2CACHEMASTER_READ_RSP, statcounters_get_l2cachemaster_read_rsp_count }, { PMC_EV_BERI_L2CACHEMASTER_READ_RSP_FLIT, statcounters_get_l2cachemaster_read_rsp_flit_count }, { PMC_EV_BERI_L2CACHEMASTER_WRITE_RSP, statcounters_get_l2cachemaster_write_rsp_count }, { PMC_EV_BERI_TAGCACHEMASTER_READ_REQ, statcounters_get_tagcachemaster_read_req_count }, { PMC_EV_BERI_TAGCACHEMASTER_WRITE_REQ, statcounters_get_tagcachemaster_write_req_count }, { PMC_EV_BERI_TAGCACHEMASTER_WRITE_REQ_FLIT, statcounters_get_tagcachemaster_write_req_flit_count }, { PMC_EV_BERI_TAGCACHEMASTER_READ_RSP, statcounters_get_tagcachemaster_read_rsp_count }, { PMC_EV_BERI_TAGCACHEMASTER_READ_RSP_FLIT, statcounters_get_tagcachemaster_read_rsp_flit_count }, { PMC_EV_BERI_TAGCACHEMASTER_WRITE_RSP, statcounters_get_tagcachemaster_write_rsp_count }, }; struct mips_pmc_spec beri_pmc_spec = { .ps_cpuclass = PMC_CLASS_BERI, .ps_cputype = PMC_CPU_MIPS_BERI, .ps_capabilities = BERI_PMC_CAPS, .ps_counter_width = 64 }; /* * Per-processor information. */ struct beri_cpu { struct pmc_hw *pc_beripmcs; uint64_t start_values[BERI_NCOUNTERS]; uint64_t stop_values[BERI_NCOUNTERS]; uint64_t saved_values[BERI_NCOUNTERS]; }; int beri_npmcs; static struct beri_cpu **beri_pcpu; static int beri_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t config; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != beri_pmc_spec.ps_cpuclass) return (EINVAL); for (i = 0; i < BERI_NCOUNTERS; i++) { if (beri_event_codes[i].pe_ev == a->pm_ev) { config = i; break; } } if (i == BERI_NCOUNTERS) return (EINVAL); pm->pm_md.pm_mips_evsel = config; PMCDBG2(MDP,ALL,2,"beri-allocate ri=%d -> config=0x%x", ri, config); return (0); } static int beri_read_pmc(int cpu, int ri, pmc_value_t *v) { uint32_t config; struct pmc *pm; pmc_value_t new; pmc_value_t start_val; pmc_value_t stop_val; pmc_value_t saved_val; pmc_value_t result; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] illegal row index %d", __LINE__, ri)); pm = beri_pcpu[cpu]->pc_beripmcs[ri].phw_pmc; config = pm->pm_md.pm_mips_evsel; start_val = beri_pcpu[cpu]->start_values[config]; if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { stop_val = beri_event_codes[config].get_func(); } else stop_val = beri_pcpu[cpu]->stop_values[config]; if (start_val <= stop_val) result = stop_val - start_val; else { if (config == 0) /* CYCLE counter is 48 bit */ result = 0x00ffffffffffffffUL; else result = 0xffffffffffffffffUL; result -= start_val; result += stop_val; } saved_val = beri_pcpu[cpu]->saved_values[config]; *v = result + saved_val; return (0); } static int beri_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; uint32_t config; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] illegal row-index %d", __LINE__, ri)); pm = beri_pcpu[cpu]->pc_beripmcs[ri].phw_pmc; config = pm->pm_md.pm_mips_evsel; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = (1UL << (beri_pmc_spec.ps_counter_width - 1)) - v; PMCDBG3(MDP,WRI,1,"beri-write cpu=%d ri=%d v=%jx", cpu, ri, v); if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) beri_pcpu[cpu]->saved_values[config] = 0; else beri_pcpu[cpu]->saved_values[config] = v; return (0); } static int beri_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] illegal row-index %d", __LINE__, ri)); phw = &beri_pcpu[cpu]->pc_beripmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[beri,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int beri_start_pmc(int cpu, int ri) { uint32_t config; struct pmc *pm; struct pmc_hw *phw; pmc_value_t v; phw = &beri_pcpu[cpu]->pc_beripmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_mips_evsel; v = beri_event_codes[config].get_func(); beri_pcpu[cpu]->start_values[config] = v; return (0); } static int beri_stop_pmc(int cpu, int ri) { uint32_t config; struct pmc *pm; struct pmc_hw *phw; pmc_value_t v; phw = &beri_pcpu[cpu]->pc_beripmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_mips_evsel; v = beri_event_codes[config].get_func(); beri_pcpu[cpu]->stop_values[config] = v; return (0); } static int beri_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] illegal row-index %d", __LINE__, ri)); phw = &beri_pcpu[cpu]->pc_beripmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[beri,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } static int beri_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { struct pmc_hw *phw; - char beri_name[PMC_NAME_MAX]; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < beri_npmcs, ("[beri,%d] row-index %d out of range", __LINE__, ri)); phw = &beri_pcpu[cpu]->pc_beripmcs[ri]; - snprintf(beri_name, sizeof(beri_name), "MIPS-%d", ri); - if ((error = copystr(beri_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "MIPS-%d", ri); pi->pm_class = beri_pmc_spec.ps_cpuclass; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int beri_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = beri_pcpu[cpu]->pc_beripmcs[ri].phw_pmc; return (0); } static int beri_pmc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int beri_pmc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int beri_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, i; struct pmc_cpu *pc; struct beri_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[beri,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"beri-init cpu=%d", cpu); beri_pcpu[cpu] = pac = malloc(sizeof(struct beri_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_beripmcs = malloc(sizeof(struct pmc_hw) * beri_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS].pcd_ri; KASSERT(pc != NULL, ("[beri,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_beripmcs; i < beri_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } return (0); } static int beri_pcpu_fini(struct pmc_mdep *md, int cpu) { return (0); } struct pmc_mdep * pmc_beri_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; snprintf(pmc_cpuid, sizeof(pmc_cpuid), "beri"); beri_npmcs = 2; PMCDBG1(MDP,INI,1,"beri-init npmcs=%d", beri_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ beri_pcpu = malloc(sizeof(struct beri_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); pmc_mdep->pmd_cputype = beri_pmc_spec.ps_cputype; pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS]; pcd->pcd_caps = beri_pmc_spec.ps_capabilities; pcd->pcd_class = beri_pmc_spec.ps_cpuclass; pcd->pcd_num = beri_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = beri_pmc_spec.ps_counter_width; pcd->pcd_allocate_pmc = beri_allocate_pmc; pcd->pcd_config_pmc = beri_config_pmc; pcd->pcd_pcpu_fini = beri_pcpu_fini; pcd->pcd_pcpu_init = beri_pcpu_init; pcd->pcd_describe = beri_describe; pcd->pcd_get_config = beri_get_config; pcd->pcd_read_pmc = beri_read_pmc; pcd->pcd_release_pmc = beri_release_pmc; pcd->pcd_start_pmc = beri_start_pmc; pcd->pcd_stop_pmc = beri_stop_pmc; pcd->pcd_write_pmc = beri_write_pmc; pmc_mdep->pmd_intr = NULL; pmc_mdep->pmd_switch_in = beri_pmc_switch_in; pmc_mdep->pmd_switch_out = beri_pmc_switch_out; pmc_mdep->pmd_npmc += beri_npmcs; return (pmc_mdep); } void pmc_beri_finalize(struct pmc_mdep *md) { } struct pmc_mdep * pmc_md_initialize() { return (pmc_beri_initialize()); } void pmc_md_finalize(struct pmc_mdep *md) { return (pmc_beri_finalize(md)); } int pmc_save_kernel_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { return (0); } int pmc_save_user_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { return (0); } diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index 5d29931c90f0..ad9d323bdb9a 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -1,1322 +1,1310 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Core PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #define CORE_CPUID_REQUEST 0xA #define CORE_CPUID_REQUEST_SIZE 0x4 #define CORE_CPUID_EAX 0x0 #define CORE_CPUID_EBX 0x1 #define CORE_CPUID_ECX 0x2 #define CORE_CPUID_EDX 0x3 #define IAF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30)) #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define EV_IS_NOTARCH 0 #define EV_IS_ARCH_SUPP 1 #define EV_IS_ARCH_NOTSUPP -1 /* * "Architectural" events defined by Intel. The values of these * symbols correspond to positions in the bitmask returned by * the CPUID.0AH instruction. */ enum core_arch_events { CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5, CORE_AE_BRANCH_MISSES_RETIRED = 6, CORE_AE_INSTRUCTION_RETIRED = 1, CORE_AE_LLC_MISSES = 4, CORE_AE_LLC_REFERENCE = 3, CORE_AE_UNHALTED_REFERENCE_CYCLES = 2, CORE_AE_UNHALTED_CORE_CYCLES = 0 }; static enum pmc_cputype core_cputype; static int core_version; struct core_cpu { volatile uint32_t pc_iafctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_corepmcs[]; }; static struct core_cpu **core_pcpu; static uint32_t core_architectural_events; static uint64_t core_pmcmask; static int core_iaf_ri; /* relative index of fixed counters */ static int core_iaf_width; static int core_iaf_npmc; static int core_iap_width; static int core_iap_npmc; static int core_iap_wroffset; static u_int pmc_alloc_refs; static bool pmc_tsx_force_abort_set; static int core_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int core_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct core_cpu *cc; struct pmc_hw *phw; int core_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; if (core_version >= 2) npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); core_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + core_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + core_ri] = phw; } if (core_version >= 2 && vm_guest == VM_GUEST_NO) { /* Enable Freezing PMCs on PMI. */ wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000); } return (0); } static int core_pcpu_fini(struct pmc_mdep *md, int cpu) { int core_ri, n, npmc; struct pmc_cpu *pc; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); if ((cc = core_pcpu[cpu]) == NULL) return (0); core_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(IAP_EVSEL0 + n, 0); if (core_version >= 2) { wrmsr(IAF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; } for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + core_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t iaf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iaf_width - 1))) == 0) return (0); v &= (1ULL << core_iaf_width) - 1; return (1ULL << core_iaf_width) - v; } static pmc_value_t iaf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iaf_width) - rlc; } static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev, umask; uint32_t caps; uint64_t config, flags; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > core_iaf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_IAF) return (EINVAL); iap = &a->pm_md.pm_iap; config = iap->pm_iap_config; ev = IAP_EVSEL_GET(config); umask = IAP_UMASK_GET(config); if (ev == 0x0) { if (umask != ri + 1) return (EINVAL); } else { switch (ri) { case 0: /* INST_RETIRED.ANY */ if (ev != 0xC0 || umask != 0x00) return (EINVAL); break; case 1: /* CPU_CLK_UNHALTED.THREAD */ if (ev != 0x3C || umask != 0x00) return (EINVAL); break; case 2: /* CPU_CLK_UNHALTED.REF */ if (ev != 0x3C || umask != 0x01) return (EINVAL); break; case 3: /* TOPDOWN.SLOTS */ if (ev != 0xA4 || umask != 0x01) return (EINVAL); break; default: return (EINVAL); } } pmc_alloc_refs++; if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 && !pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = true; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 1, NULL); } flags = 0; if (config & IAP_OS) flags |= IAF_OS; if (config & IAP_USR) flags |= IAF_USR; if (config & IAP_ANY) flags |= IAF_ANY; if (config & IAP_INT) flags |= IAF_PMI; caps = a->pm_caps; if (caps & PMC_CAP_INTERRUPT) flags |= IAF_PMI; if (caps & PMC_CAP_SYSTEM) flags |= IAF_OS; if (caps & PMC_CAP_USER) flags |= IAF_USR; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) flags |= (IAF_OS | IAF_USR); pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); return (0); } static int iaf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; return (0); } static int iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char iaf_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri]; - (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri); - if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri); pi->pm_class = PMC_CLASS_IAF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iaf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; return (0); } static int iaf_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[iaf,%d] ri %d out of range", __LINE__, ri)); *msr = IAF_RI_TO_MSR(ri); return (0); } static int iaf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + core_iaf_ri)); tmp = rdpmc(IAF_RI_TO_MSR(ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iaf_width) - 1); PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); return (0); } static int iaf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); MPASS(pmc_alloc_refs > 0); if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = false; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 0, NULL); } return (0); } static int iaf_start_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; wrmsr(IAF_CTRL, cc->pc_iafctrl); cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_stop_pmc(int cpu, int ri) { struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4)); wrmsr(IAF_CTRL, cc->pc_iafctrl); /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */ PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_write_pmc(int cpu, int ri, pmc_value_t v) { struct core_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iaf_reload_count_to_perfctr_value(v); /* Turn off the fixed counter */ wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4))); wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); /* Turn on fixed counters */ wrmsr(IAF_CTRL, cc->pc_iafctrl); PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); return (0); } static void iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iaf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; pcd->pcd_caps = IAF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iaf_allocate_pmc; pcd->pcd_config_pmc = iaf_config_pmc; pcd->pcd_describe = iaf_describe; pcd->pcd_get_config = iaf_get_config; pcd->pcd_get_msr = iaf_get_msr; pcd->pcd_pcpu_fini = core_pcpu_noop; pcd->pcd_pcpu_init = core_pcpu_noop; pcd->pcd_read_pmc = iaf_read_pmc; pcd->pcd_release_pmc = iaf_release_pmc; pcd->pcd_start_pmc = iaf_start_pmc; pcd->pcd_stop_pmc = iaf_stop_pmc; pcd->pcd_write_pmc = iaf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* Sub fields of UMASK that this event supports. */ #define IAP_M_CORE (1 << 0) /* Core specificity */ #define IAP_M_AGENT (1 << 1) /* Agent specificity */ #define IAP_M_PREFETCH (1 << 2) /* Prefetch */ #define IAP_M_MESI (1 << 3) /* MESI */ #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */ #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */ #define IAP_M_TRANSITION (1 << 6) /* Transition */ #define IAP_F_CORE (0x3 << 14) /* Core specificity */ #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */ #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */ #define IAP_F_MESI (0xF << 8) /* MESI */ #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */ #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */ #define IAP_F_TRANSITION (0x1 << 12) /* Transition */ #define IAP_PREFETCH_RESERVED (0x2 << 12) #define IAP_CORE_THIS (0x1 << 14) #define IAP_CORE_ALL (0x3 << 14) #define IAP_F_CMASK 0xFF000000 static pmc_value_t iap_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iap_width - 1))) == 0) return (0); v &= (1ULL << core_iap_width) - 1; return (1ULL << core_iap_width) - v; } static pmc_value_t iap_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iap_width) - rlc; } static int iap_pmc_has_overflowed(int ri) { uint64_t v; /* * We treat a Core (i.e., Intel architecture v1) PMC as has * having overflowed if its MSB is zero. */ v = rdpmc(ri); return ((v & (1ULL << (core_iap_width - 1))) == 0); } static int iap_event_corei7_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0, 1. */ case 0x40: case 0x41: case 0x42: case 0x43: case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x53: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_westmere_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0x60: case 0xB3: mask = 0x1; break; /* Events valid only on counter 0, 1. */ case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0xB7: mask = 0x1; break; /* Events valid only on counter 1. */ case 0xC0: mask = 0x2; break; /* Events valid only on counter 2. */ case 0x48: case 0xA2: case 0xA3: mask = 0x4; break; /* Events valid only on counter 3. */ case 0xBB: case 0xCD: mask = 0x8; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_core_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0. */ case 0x10: case 0x14: case 0x18: case 0xB3: case 0xC1: case 0xCB: mask = (1 << 0); break; /* * Events valid only on counter 1. */ case 0x11: case 0x12: case 0x13: mask = (1 << 1); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_IAP) return (EINVAL); iap = &a->pm_md.pm_iap; ev = IAP_EVSEL_GET(iap->pm_iap_config); switch (core_cputype) { case PMC_CPU_INTEL_CORE: case PMC_CPU_INTEL_CORE2: case PMC_CPU_INTEL_CORE2EXTREME: if (iap_event_core_ok_on_counter(ev, ri) == 0) return (EINVAL); case PMC_CPU_INTEL_COREI7: case PMC_CPU_INTEL_NEHALEM_EX: if (iap_event_corei7_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_WESTMERE: case PMC_CPU_INTEL_WESTMERE_EX: if (iap_event_westmere_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_SANDYBRIDGE: case PMC_CPU_INTEL_SANDYBRIDGE_XEON: case PMC_CPU_INTEL_IVYBRIDGE: case PMC_CPU_INTEL_IVYBRIDGE_XEON: case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_HASWELL_XEON: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_ATOM: case PMC_CPU_INTEL_ATOM_SILVERMONT: case PMC_CPU_INTEL_ATOM_GOLDMONT: case PMC_CPU_INTEL_ATOM_GOLDMONT_P: case PMC_CPU_INTEL_ATOM_TREMONT: case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_ICELAKE: case PMC_CPU_INTEL_ICELAKE_XEON: case PMC_CPU_INTEL_ALDERLAKE: default: break; } pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config; return (0); } static int iap_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; return (0); } static int iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char iap_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri]; - (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri); - if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri); pi->pm_class = PMC_CLASS_IAP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iap_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; return (0); } static int iap_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iap_npmc, ("[iap,%d] ri %d out of range", __LINE__, ri)); *msr = ri; return (0); } static int iap_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdpmc(ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iap_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iap_width) - 1); PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAP_PMC0 + ri, *v); return (0); } static int iap_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iap_start_pmc(int cpu, int ri) { struct pmc *pm; uint64_t evsel; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_iap.pm_iap_evsel; PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, IAP_EVSEL0 + ri, evsel); /* Event specific configuration. */ switch (IAP_EVSEL_GET(evsel)) { case 0xB7: wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); break; case 0xBB: wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp); break; default: break; } wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN); if (core_version >= 2) { cc->pc_globalctrl |= (1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } return (0); } static int iap_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); wrmsr(IAP_EVSEL0 + ri, 0); /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */ return (0); } static int iap_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iap_reload_count_to_perfctr_value(v); v &= (1ULL << core_iap_width) - 1; PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, IAP_PMC0 + ri, v); /* * Write the new value to the counter (or it's alias). The * counter will be in a stopped state when the pcd_write() * entry point is called. */ wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); return (0); } static void iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, int flags) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iap-initialize"); /* Remember the set of architectural events supported. */ core_architectural_events = ~flags; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; pcd->pcd_caps = IAP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iap_allocate_pmc; pcd->pcd_config_pmc = iap_config_pmc; pcd->pcd_describe = iap_describe; pcd->pcd_get_config = iap_get_config; pcd->pcd_get_msr = iap_get_msr; pcd->pcd_pcpu_fini = core_pcpu_fini; pcd->pcd_pcpu_init = core_pcpu_init; pcd->pcd_read_pmc = iap_read_pmc; pcd->pcd_release_pmc = iap_release_pmc; pcd->pcd_start_pmc = iap_start_pmc; pcd->pcd_stop_pmc = iap_stop_pmc; pcd->pcd_write_pmc = iap_write_pmc; md->pmd_npmc += npmc; } static int core_intr(struct trapframe *tf) { pmc_value_t v; struct pmc *pm; struct core_cpu *cc; int error, found_interrupt, ri; PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf, TRAPF_USERMODE(tf)); found_interrupt = 0; cc = core_pcpu[curcpu]; for (ri = 0; ri < core_iap_npmc; ri++) { if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (!iap_pmc_has_overflowed(ri)) continue; found_interrupt = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); v = pm->pm_sc.pm_reloadcount; v = iap_reload_count_to_perfctr_value(v); /* * Stop the counter, reload it but only restart it if * the PMC is not stalled. */ wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel); wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); if (__predict_false(error)) continue; wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); if (found_interrupt) lapic_reenable_pmc(); return (found_interrupt); } static int core2_intr(struct trapframe *tf) { int error, found_interrupt = 0, n, cpu; uint64_t flag, intrstatus, intrdisable = 0; struct pmc *pm; struct core_cpu *cc; pmc_value_t v; cpu = curcpu; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); /* * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which * PMCs have a pending PMI interrupt. We take a 'snapshot' of * the current set of interrupting PMCs and process these * after stopping them. */ intrstatus = rdmsr(IA_GLOBAL_STATUS); PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, (uintmax_t) intrstatus); /* * Stop PMCs unless hardware already done it. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) wrmsr(IA_GLOBAL_CTRL, 0); cc = core_pcpu[cpu]; KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); /* * Look for interrupts from fixed function PMCs. */ for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); /* Reload sampling count. */ wrmsr(IAF_CTR0 + n, v); PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu, error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); } /* * Process interrupts from the programmable counters. */ for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, (uintmax_t) v); /* Reload sampling count. */ wrmsr(core_iap_wroffset + IAP_PMC0 + n, v); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); if (found_interrupt) lapic_reenable_pmc(); /* * Reenable all non-stalled PMCs. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) { wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } else { if (__predict_false(intrdisable)) { cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); } PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_STATUS)); return (found_interrupt); } int pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) { int cpuid[CORE_CPUID_REQUEST_SIZE]; int flags, nflags; do_cpuid(CORE_CPUID_REQUEST, cpuid); core_cputype = md->pmd_cputype; core_version = (version_override > 0) ? version_override : cpuid[CORE_CPUID_EAX] & 0xFF; PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d", core_cputype, maxcpu, core_version); if (core_version < 1 || core_version > 5 || (core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) { /* Unknown PMC architecture. */ printf("hwpmc_core: unknown PMC architecture: %d\n", core_version); return (EPROGMISMATCH); } core_iap_wroffset = 0; if (cpu_feature2 & CPUID2_PDCM) { if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) { PMCDBG0(MDP, INI, 1, "core-init full-width write supported"); core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0; } else PMCDBG0(MDP, INI, 1, "core-init full-width write NOT supported"); } else PMCDBG0(MDP, INI, 1, "core-init pdcm not supported"); core_pmcmask = 0; /* * Initialize programmable counters. */ core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF; core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF; core_pmcmask |= ((1ULL << core_iap_npmc) - 1); nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF; flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1); iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags); /* * Initialize fixed function counters, if present. */ if (core_version >= 2) { core_iaf_ri = core_iap_npmc; core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; } PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, core_iaf_ri); core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); /* * Choose the appropriate interrupt handler. */ if (core_version >= 2) md->pmd_intr = core2_intr; else md->pmd_intr = core_intr; md->pmd_pcpu_fini = NULL; md->pmd_pcpu_init = NULL; return (0); } void pmc_core_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "core-finalize"); free(core_pcpu, M_PMC); core_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_mips.c b/sys/dev/hwpmc/hwpmc_mips.c index edf83154dc57..1ed3fdea8915 100644 --- a/sys/dev/hwpmc/hwpmc_mips.c +++ b/sys/dev/hwpmc/hwpmc_mips.c @@ -1,809 +1,805 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010, George V. Neville-Neil * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" #include #include #include #include #include #include #include #include int mips_npmcs; /* * Per-processor information. */ struct mips_cpu { struct pmc_hw *pc_mipspmcs; }; static struct mips_cpu **mips_pcpu; #if defined(__mips_n64) # define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \ ((vm_offset_t)(reg) >= MIPS_XKPHYS_START)) #else # define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \ ((vm_offset_t)(reg) >= MIPS_KSEG0_START)) #endif /* * We need some reasonable default to prevent backtrace code * from wandering too far */ #define MAX_FUNCTION_SIZE 0x10000 #define MAX_PROLOGUE_SIZE 0x100 static int mips_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t caps, config, counter; uint32_t event; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] illegal row index %d", __LINE__, ri)); caps = a->pm_caps; if (a->pm_class != mips_pmc_spec.ps_cpuclass) return (EINVAL); pe = a->pm_ev; counter = MIPS_CTR_ALL; event = 0; for (i = 0; i < mips_event_codes_size; i++) { if (mips_event_codes[i].pe_ev == pe) { event = mips_event_codes[i].pe_code; counter = mips_event_codes[i].pe_counter; break; } } if (i == mips_event_codes_size) return (EINVAL); if ((counter != MIPS_CTR_ALL) && (counter != ri)) return (EINVAL); config = mips_get_perfctl(cpu, ri, event, caps); pm->pm_md.pm_mips_evsel = config; PMCDBG2(MDP,ALL,2,"mips-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int mips_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] illegal row index %d", __LINE__, ri)); pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc; tmp = mips_pmcn_read(ri); PMCDBG2(MDP,REA,2,"mips-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = tmp - (1UL << (mips_pmc_spec.ps_counter_width - 1)); else *v = tmp; return 0; } static int mips_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] illegal row-index %d", __LINE__, ri)); pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = (1UL << (mips_pmc_spec.ps_counter_width - 1)) - v; PMCDBG3(MDP,WRI,1,"mips-write cpu=%d ri=%d v=%jx", cpu, ri, v); mips_pmcn_write(ri, v); return 0; } static int mips_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] illegal row-index %d", __LINE__, ri)); phw = &mips_pcpu[cpu]->pc_mipspmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[mips,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int mips_start_pmc(int cpu, int ri) { uint32_t config; struct pmc *pm; struct pmc_hw *phw; phw = &mips_pcpu[cpu]->pc_mipspmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_mips_evsel; /* Enable the PMC. */ switch (ri) { case 0: mips_wr_perfcnt0(config); break; case 1: mips_wr_perfcnt2(config); break; default: break; } return 0; } static int mips_stop_pmc(int cpu, int ri) { struct pmc *pm; struct pmc_hw *phw; phw = &mips_pcpu[cpu]->pc_mipspmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. * * Clearing the entire register turns the counter off as well * as removes the previously sampled event. */ switch (ri) { case 0: mips_wr_perfcnt0(0); break; case 1: mips_wr_perfcnt2(0); break; default: break; } return 0; } static int mips_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] illegal row-index %d", __LINE__, ri)); phw = &mips_pcpu[cpu]->pc_mipspmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[mips,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int mips_pmc_intr(struct trapframe *tf) { int error; int retval, ri, cpu; struct pmc *pm; struct mips_cpu *pc; uint32_t r0, r2; pmc_value_t r; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; pc = mips_pcpu[cpu]; /* Stop PMCs without clearing the counter */ r0 = mips_rd_perfcnt0(); mips_wr_perfcnt0(r0 & ~(0x1f)); r2 = mips_rd_perfcnt2(); mips_wr_perfcnt2(r2 & ~(0x1f)); for (ri = 0; ri < mips_npmcs; ri++) { pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc; if (pm == NULL) continue; if (! PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; r = mips_pmcn_read(ri); /* If bit 31 is set, the counter has overflowed */ if ((r & (1UL << (mips_pmc_spec.ps_counter_width - 1))) == 0) continue; retval = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) { /* Clear/disable the relevant counter */ if (ri == 0) r0 = 0; else if (ri == 1) r2 = 0; mips_stop_pmc(cpu, ri); } /* Reload sampling count */ mips_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } /* * Re-enable the PMC counters where they left off. * * Any counter which overflowed will have its sample count * reloaded in the loop above. */ mips_wr_perfcnt0(r0); mips_wr_perfcnt2(r2); return retval; } static int mips_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char mips_name[PMC_NAME_MAX]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < mips_npmcs, ("[mips,%d] row-index %d out of range", __LINE__, ri)); phw = &mips_pcpu[cpu]->pc_mipspmcs[ri]; - snprintf(mips_name, sizeof(mips_name), "MIPS-%d", ri); - if ((error = copystr(mips_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "MIPS-%d", ri); pi->pm_class = mips_pmc_spec.ps_cpuclass; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int mips_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc; return 0; } /* * XXX don't know what we should do here. */ static int mips_pmc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int mips_pmc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int mips_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, i; struct pmc_cpu *pc; struct mips_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[mips,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"mips-init cpu=%d", cpu); mips_pcpu[cpu] = pac = malloc(sizeof(struct mips_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_mipspmcs = malloc(sizeof(struct pmc_hw) * mips_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS].pcd_ri; KASSERT(pc != NULL, ("[mips,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_mipspmcs; i < mips_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* * Clear the counter control register which has the effect * of disabling counting. */ for (i = 0; i < mips_npmcs; i++) mips_pmcn_write(i, 0); return 0; } static int mips_pcpu_fini(struct pmc_mdep *md, int cpu) { return 0; } struct pmc_mdep * pmc_mips_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; /* * TODO: Use More bit of PerfCntlX register to detect actual * number of counters */ mips_npmcs = 2; PMCDBG1(MDP,INI,1,"mips-init npmcs=%d", mips_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ mips_pcpu = malloc(sizeof(struct mips_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); pmc_mdep->pmd_cputype = mips_pmc_spec.ps_cputype; pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS]; pcd->pcd_caps = mips_pmc_spec.ps_capabilities; pcd->pcd_class = mips_pmc_spec.ps_cpuclass; pcd->pcd_num = mips_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = mips_pmc_spec.ps_counter_width; pcd->pcd_allocate_pmc = mips_allocate_pmc; pcd->pcd_config_pmc = mips_config_pmc; pcd->pcd_pcpu_fini = mips_pcpu_fini; pcd->pcd_pcpu_init = mips_pcpu_init; pcd->pcd_describe = mips_describe; pcd->pcd_get_config = mips_get_config; pcd->pcd_read_pmc = mips_read_pmc; pcd->pcd_release_pmc = mips_release_pmc; pcd->pcd_start_pmc = mips_start_pmc; pcd->pcd_stop_pmc = mips_stop_pmc; pcd->pcd_write_pmc = mips_write_pmc; pmc_mdep->pmd_intr = mips_pmc_intr; pmc_mdep->pmd_switch_in = mips_pmc_switch_in; pmc_mdep->pmd_switch_out = mips_pmc_switch_out; pmc_mdep->pmd_npmc += mips_npmcs; return (pmc_mdep); } void pmc_mips_finalize(struct pmc_mdep *md) { (void) md; } #ifdef HWPMC_MIPS_BACKTRACE static int pmc_next_frame(register_t *pc, register_t *sp) { InstFmt i; uintptr_t va; uint32_t instr, mask; int more, stksize; register_t ra = 0; /* Jump here after a nonstandard (interrupt handler) frame */ stksize = 0; /* check for bad SP: could foul up next frame */ if (!MIPS_IS_VALID_KERNELADDR(*sp)) { goto error; } /* check for bad PC */ if (!MIPS_IS_VALID_KERNELADDR(*pc)) { goto error; } /* * Find the beginning of the current subroutine by scanning * backwards from the current PC for the end of the previous * subroutine. */ va = *pc - sizeof(int); while (1) { instr = *((uint32_t *)va); /* [d]addiu sp,sp,-X */ if (((instr & 0xffff8000) == 0x27bd8000) || ((instr & 0xffff8000) == 0x67bd8000)) break; /* jr ra */ if (instr == 0x03e00008) { /* skip over branch-delay slot instruction */ va += 2 * sizeof(int); break; } va -= sizeof(int); } /* skip over nulls which might separate .o files */ while ((instr = *((uint32_t *)va)) == 0) va += sizeof(int); /* scan forwards to find stack size and any saved registers */ stksize = 0; more = 3; mask = 0; for (; more; va += sizeof(int), more = (more == 3) ? 3 : more - 1) { /* stop if hit our current position */ if (va >= *pc) break; instr = *((uint32_t *)va); i.word = instr; switch (i.JType.op) { case OP_SPECIAL: switch (i.RType.func) { case OP_JR: case OP_JALR: more = 2; /* stop after next instruction */ break; case OP_SYSCALL: case OP_BREAK: more = 1; /* stop now */ } break; case OP_BCOND: case OP_J: case OP_JAL: case OP_BEQ: case OP_BNE: case OP_BLEZ: case OP_BGTZ: more = 2; /* stop after next instruction */ break; case OP_COP0: case OP_COP1: case OP_COP2: case OP_COP3: switch (i.RType.rs) { case OP_BCx: case OP_BCy: more = 2; /* stop after next instruction */ } break; case OP_SW: case OP_SD: /* * SP is being saved using S8(FP). Most likely it indicates * that SP is modified in the function and we can't get * its value safely without emulating code backward * So just bail out on functions like this */ if ((i.IType.rs == 30) && (i.IType.rt = 29)) return (-1); /* look for saved registers on the stack */ if (i.IType.rs != 29) break; /* only restore the first one */ if (mask & (1 << i.IType.rt)) break; mask |= (1 << i.IType.rt); if (i.IType.rt == 31) ra = *((register_t *)(*sp + (short)i.IType.imm)); break; case OP_ADDI: case OP_ADDIU: case OP_DADDI: case OP_DADDIU: /* look for stack pointer adjustment */ if (i.IType.rs != 29 || i.IType.rt != 29) break; stksize = -((short)i.IType.imm); } } if (!MIPS_IS_VALID_KERNELADDR(ra)) return (-1); *pc = ra; *sp += stksize; return (0); error: return (-1); } static int pmc_next_uframe(register_t *pc, register_t *sp, register_t *ra) { int offset, registers_on_stack; uint32_t opcode, mask; register_t function_start; int stksize; InstFmt i; registers_on_stack = 0; mask = 0; function_start = 0; offset = 0; stksize = 0; while (offset < MAX_FUNCTION_SIZE) { opcode = fuword32((void *)(*pc - offset)); /* [d]addiu sp, sp, -X*/ if (((opcode & 0xffff8000) == 0x27bd8000) || ((opcode & 0xffff8000) == 0x67bd8000)) { function_start = *pc - offset; registers_on_stack = 1; break; } /* lui gp, X */ if ((opcode & 0xffff8000) == 0x3c1c0000) { /* * Function might start with this instruction * Keep an eye on "jr ra" and sp correction * with positive value further on */ function_start = *pc - offset; } if (function_start) { /* * Stop looking further. Possible end of * function instruction: it means there is no * stack modifications, sp is unchanged */ /* [d]addiu sp,sp,X */ if (((opcode & 0xffff8000) == 0x27bd0000) || ((opcode & 0xffff8000) == 0x67bd0000)) break; if (opcode == 0x03e00008) break; } offset += sizeof(int); } if (!function_start) return (-1); if (registers_on_stack) { offset = 0; while ((offset < MAX_PROLOGUE_SIZE) && ((function_start + offset) < *pc)) { i.word = fuword32((void *)(function_start + offset)); switch (i.JType.op) { case OP_SW: /* look for saved registers on the stack */ if (i.IType.rs != 29) break; /* only restore the first one */ if (mask & (1 << i.IType.rt)) break; mask |= (1 << i.IType.rt); if (i.IType.rt == 31) *ra = fuword32((void *)(*sp + (short)i.IType.imm)); break; #if defined(__mips_n64) case OP_SD: /* look for saved registers on the stack */ if (i.IType.rs != 29) break; /* only restore the first one */ if (mask & (1 << i.IType.rt)) break; mask |= (1 << i.IType.rt); /* ra */ if (i.IType.rt == 31) *ra = fuword64((void *)(*sp + (short)i.IType.imm)); break; #endif case OP_ADDI: case OP_ADDIU: case OP_DADDI: case OP_DADDIU: /* look for stack pointer adjustment */ if (i.IType.rs != 29 || i.IType.rt != 29) break; stksize = -((short)i.IType.imm); } offset += sizeof(int); } } /* * We reached the end of backtrace */ if (*pc == *ra) return (-1); *pc = *ra; *sp += stksize; return (0); } #endif /* HWPMC_MIPS_BACKTRACE */ struct pmc_mdep * pmc_md_initialize() { return pmc_mips_initialize(); } void pmc_md_finalize(struct pmc_mdep *md) { return pmc_mips_finalize(md); } int pmc_save_kernel_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { register_t pc, ra, sp; int frames = 0; pc = tf->pc; sp = tf->sp; ra = tf->ra; cc[frames++] = pc; #ifdef HWPMC_MIPS_BACKTRACE /* * Unwind, and unwind, and unwind */ while (1) { if (frames >= nframes) break; if (pmc_next_frame(&pc, &sp) < 0) break; cc[frames++] = pc; } #endif return (frames); } int pmc_save_user_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) { register_t pc, ra, sp; int frames = 0; pc = tf->pc; sp = tf->sp; ra = tf->ra; cc[frames++] = pc; #ifdef HWPMC_MIPS_BACKTRACE /* * Unwind, and unwind, and unwind */ while (1) { if (frames >= nframes) break; if (pmc_next_uframe(&pc, &sp, &ra) < 0) break; cc[frames++] = pc; } #endif return (frames); } diff --git a/sys/dev/hwpmc/hwpmc_powerpc.c b/sys/dev/hwpmc/hwpmc_powerpc.c index 7c6df5825399..4c2a9cf84ef4 100644 --- a/sys/dev/hwpmc/hwpmc_powerpc.c +++ b/sys/dev/hwpmc/hwpmc_powerpc.c @@ -1,650 +1,647 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011,2013 Justin Hibbits * Copyright (c) 2005, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "hwpmc_powerpc.h" #ifdef __powerpc64__ #define OFFSET 4 /* Account for the TOC reload slot */ #else #define OFFSET 0 #endif struct powerpc_cpu **powerpc_pcpu; struct pmc_ppc_event *ppc_event_codes; size_t ppc_event_codes_size; int ppc_event_first; int ppc_event_last; int ppc_max_pmcs; enum pmc_class ppc_class; void (*powerpc_set_pmc)(int cpu, int ri, int config); pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); void (*powerpc_resume_pmc)(bool ie); int pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t *osp, *sp; uintptr_t pc; int frames = 0; cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); osp = (uintptr_t *)PAGE_SIZE; for (; frames < maxsamples; frames++) { if (sp <= osp) break; #ifdef __powerpc64__ pc = sp[2]; #else pc = sp[1]; #endif if ((pc & 3) || (pc < 0x100)) break; /* * trapexit() and asttrapexit() are sentinels * for kernel stack tracing. * */ if (pc + OFFSET == (uintptr_t) &trapexit || pc + OFFSET == (uintptr_t) &asttrapexit) break; cc[frames] = pc; osp = sp; sp = (uintptr_t *)*sp; } return (frames); } static int powerpc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int powerpc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } int powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char powerpc_name[PMC_NAME_MAX]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; - snprintf(powerpc_name, sizeof(powerpc_name), "POWERPC-%d", ri); - if ((error = copystr(powerpc_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "POWERPC-%d", ri); pi->pm_class = powerpc_pcpu[cpu]->pc_class; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } int powerpc_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; return (0); } int powerpc_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct powerpc_cpu *pac; struct pmc_hw *phw; int first_ri, i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu) + ppc_max_pmcs * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); pac->pc_class = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } return (0); } int powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) { PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); free(powerpc_pcpu[cpu], M_PMC); powerpc_pcpu[cpu] = NULL; return (0); } int powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t caps, config = 0, counter = 0; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != ppc_class) return (EINVAL); caps = a->pm_caps; pe = a->pm_ev; if (pe < ppc_event_first || pe > ppc_event_last) return (EINVAL); for (i = 0; i < ppc_event_codes_size; i++) { if (ppc_event_codes[i].pe_event == pe) { config = ppc_event_codes[i].pe_code; counter = ppc_event_codes[i].pe_flags; break; } } if (i == ppc_event_codes_size) return (EINVAL); if ((counter & (1 << ri)) == 0) return (EINVAL); if (caps & PMC_CAP_SYSTEM) config |= POWERPC_PMC_KERNEL_ENABLE; if (caps & PMC_CAP_USER) config |= POWERPC_PMC_USER_ENABLE; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) config |= POWERPC_PMC_ENABLE; pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", cpu, ri, config); return (0); } int powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } int powerpc_start_pmc(int cpu, int ri) { struct pmc *pm; PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); return (0); } int powerpc_stop_pmc(int cpu, int ri) { PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); powerpc_set_pmc(cpu, ri, PMCN_NONE); return (0); } int powerpc_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } pmc_value_t powerpc_pmcn_read_default(unsigned int pmc) { pmc_value_t val; if (pmc > ppc_max_pmcs) panic("Invalid PMC number: %d\n", pmc); switch (pmc) { case 0: val = mfspr(SPR_PMC1); break; case 1: val = mfspr(SPR_PMC2); break; case 2: val = mfspr(SPR_PMC3); break; case 3: val = mfspr(SPR_PMC4); break; case 4: val = mfspr(SPR_PMC5); break; case 5: val = mfspr(SPR_PMC6); break; case 6: val = mfspr(SPR_PMC7); break; case 7: val = mfspr(SPR_PMC8); break; } return (val); } void powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) { if (pmc > ppc_max_pmcs) panic("Invalid PMC number: %d\n", pmc); switch (pmc) { case 0: mtspr(SPR_PMC1, val); break; case 1: mtspr(SPR_PMC2, val); break; case 2: mtspr(SPR_PMC3, val); break; case 3: mtspr(SPR_PMC4, val); break; case 4: mtspr(SPR_PMC5, val); break; case 5: mtspr(SPR_PMC6, val); break; case 6: mtspr(SPR_PMC7, val); break; case 7: mtspr(SPR_PMC8, val); break; } } int powerpc_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t p, r, tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); /* * After an interrupt occurs because of a PMC overflow, the PMC value * is not always MAX_PMC_VALUE + 1, but may be a little above it. * This may mess up calculations and frustrate machine independent * layer expectations, such as that no value read should be greater * than reload count in sampling mode. * To avoid these issues, use MAX_PMC_VALUE as an upper limit. */ p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); r = pm->pm_sc.pm_reloadcount; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Special case 1: r is too big * This usually happens when a PMC write fails, the PMC is * stopped and then it is read. * * Special case 2: PMC was reseted or has a value * that should not be possible with current r. * * In the above cases, just return 0 instead of an arbitrary * value. */ if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) tmp = 0; else tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); } else tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), (uintmax_t)p); *v = tmp; return (0); } int powerpc_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; pmc_value_t vlo; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); vlo = v % (POWERPC_MAX_PMC_VALUE + 1); } else if (v > POWERPC_MAX_PMC_VALUE) { PMCDBG3(MDP,WRI,2, "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", cpu, ri, (uintmax_t)v); return (EINVAL); } else vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), (uintmax_t)vlo); powerpc_pmcn_write(ri, vlo); return (0); } int powerpc_pmc_intr(struct trapframe *tf) { struct pmc *pm; struct powerpc_cpu *pc; int cpu, error, i, retval; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pc = powerpc_pcpu[cpu]; /* * Look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association. */ for (i = 0; i < ppc_max_pmcs; i++) { if (!POWERPC_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ /* * Always clear the PMC, to make it stop interrupting. * If pm is available and in sampling mode, use reload * count, to make PMC read after stop correct. * Otherwise, just reset the PMC. */ if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { if (pm->pm_state != PMC_STATE_RUNNING) { powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); continue; } } else { if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 1) % PPC_OVERFLOWCNT_MAX; PMCDBG3(MDP,INT,2, "cpu=%d ri=%d: overflowcnt=%d", cpu, i, PPC_OVERFLOWCNT(pm)); } powerpc_pmcn_write(i, 0); continue; } error = pmc_process_interrupt(PMC_HR, pm, tf); if (error != 0) { PMCDBG3(MDP,INT,3, "cpu=%d ri=%d: error %d processing interrupt", cpu, i, error); powerpc_stop_pmc(cpu, i); } /* Reload sampling count */ powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); } if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); /* * Re-enable PERF exceptions if we were able to find the interrupt * source and handle it. Otherwise, it's better to disable PERF * interrupts, to avoid the risk of processing the same interrupt * forever. */ powerpc_resume_pmc(retval != 0); if (retval == 0) log(LOG_WARNING, "pmc_intr: couldn't find interrupting PMC on cpu %d - " "disabling PERF interrupts\n", cpu); return (retval); } struct pmc_mdep * pmc_md_initialize(void) { struct pmc_mdep *pmc_mdep; int error; uint16_t vers; /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); vers = mfpvr() >> 16; pmc_mdep->pmd_switch_in = powerpc_switch_in; pmc_mdep->pmd_switch_out = powerpc_switch_out; switch (vers) { case MPC7447A: case MPC7448: case MPC7450: case MPC7455: case MPC7457: error = pmc_mpc7xxx_initialize(pmc_mdep); break; case IBM970: case IBM970FX: case IBM970MP: error = pmc_ppc970_initialize(pmc_mdep); break; case IBMPOWER8E: case IBMPOWER8NVL: case IBMPOWER8: case IBMPOWER9: error = pmc_power8_initialize(pmc_mdep); break; case FSL_E500v1: case FSL_E500v2: case FSL_E500mc: case FSL_E5500: error = pmc_e500_initialize(pmc_mdep); break; default: error = -1; break; } if (error != 0) { pmc_mdep_free(pmc_mdep); pmc_mdep = NULL; } return (pmc_mdep); } void pmc_md_finalize(struct pmc_mdep *md) { free(powerpc_pcpu, M_PMC); powerpc_pcpu = NULL; } int pmc_save_user_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t *osp, *sp; int frames = 0; cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); osp = NULL; for (; frames < maxsamples; frames++) { if (sp <= osp) break; osp = sp; #ifdef __powerpc64__ /* Check if 32-bit mode. */ if (!(tf->srr1 & PSL_SF)) { cc[frames] = fuword32((uint32_t *)sp + 1); sp = (uintptr_t *)(uintptr_t)fuword32(sp); } else { cc[frames] = fuword(sp + 2); sp = (uintptr_t *)fuword(sp); } #else cc[frames] = fuword32((uint32_t *)sp + 1); sp = (uintptr_t *)fuword32(sp); #endif } return (frames); } diff --git a/sys/dev/hwpmc/hwpmc_soft.c b/sys/dev/hwpmc/hwpmc_soft.c index cf2401e9159e..f3a6ffc70677 100644 --- a/sys/dev/hwpmc/hwpmc_soft.c +++ b/sys/dev/hwpmc/hwpmc_soft.c @@ -1,498 +1,493 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "hwpmc_soft.h" /* * Software PMC support. */ #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) struct soft_descr { struct pmc_descr pm_descr; /* "base class" */ }; static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = { #define SOFT_PMCDESCR(N) \ { \ .pm_descr = \ { \ .pd_name = #N, \ .pd_class = PMC_CLASS_SOFT, \ .pd_caps = SOFT_CAPS, \ .pd_width = 64 \ }, \ } SOFT_PMCDESCR(SOFT0), SOFT_PMCDESCR(SOFT1), SOFT_PMCDESCR(SOFT2), SOFT_PMCDESCR(SOFT3), SOFT_PMCDESCR(SOFT4), SOFT_PMCDESCR(SOFT5), SOFT_PMCDESCR(SOFT6), SOFT_PMCDESCR(SOFT7), SOFT_PMCDESCR(SOFT8), SOFT_PMCDESCR(SOFT9), SOFT_PMCDESCR(SOFT10), SOFT_PMCDESCR(SOFT11), SOFT_PMCDESCR(SOFT12), SOFT_PMCDESCR(SOFT13), SOFT_PMCDESCR(SOFT14), SOFT_PMCDESCR(SOFT15) }; /* * Per-CPU data structure. */ struct soft_cpu { struct pmc_hw soft_hw[SOFT_NPMCS]; pmc_value_t soft_values[SOFT_NPMCS]; }; static struct soft_cpu **soft_pcpu; static int soft_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event ev; struct pmc_soft *ps; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_SOFT) return (EINVAL); if ((pm->pm_caps & SOFT_CAPS) == 0) return (EINVAL); if ((pm->pm_caps & ~SOFT_CAPS) != 0) return (EPERM); ev = pm->pm_event; if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) return (EINVAL); /* Check if event is registered. */ ps = pmc_soft_ev_acquire(ev); if (ps == NULL) return (EINVAL); pmc_soft_ev_release(ps); /* Module unload is protected by pmc SX lock. */ if (ps->ps_alloc != NULL) ps->ps_alloc(); return (0); } static int soft_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct soft_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; pd = &soft_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return (error); - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int soft_get_config(int cpu, int ri, struct pmc **ppm) { (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; return (0); } static int soft_pcpu_fini(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); free(soft_pcpu[cpu], M_PMC); soft_pcpu[cpu] = NULL; ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] ri=%d", __LINE__, ri)); pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; return (0); } static int soft_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, n; struct pmc_cpu *pc; struct soft_cpu *soft_pc; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", __LINE__)); soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); soft_pcpu[cpu] = soft_pc; phw = soft_pc->soft_hw; first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; for (n = 0; n < SOFT_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } static int soft_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm __diagused; const struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); PMCDBG1(MDP,REA,1,"soft-read id=%d", ri); *v = soft_pcpu[cpu]->soft_values[ri]; return (0); } static int soft_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); soft_pcpu[cpu]->soft_values[ri] = v; return (0); } static int soft_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; enum pmc_event ev; struct pmc_soft *ps; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; KASSERT(phw->phw_pmc == NULL, ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); ev = pmc->pm_event; /* Check if event is registered. */ ps = pmc_soft_ev_acquire(ev); KASSERT(ps != NULL, ("[soft,%d] unregistered event %d", __LINE__, ev)); pmc_soft_ev_release(ps); /* Module unload is protected by pmc SX lock. */ if (ps->ps_release != NULL) ps->ps_release(); return (0); } static int soft_start_pmc(int cpu, int ri) { struct pmc *pm; struct soft_cpu *pc; struct pmc_soft *ps; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pc = soft_pcpu[cpu]; pm = pc->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); ps = pmc_soft_ev_acquire(pm->pm_event); if (ps == NULL) return (EINVAL); atomic_add_int(&ps->ps_running, 1); pmc_soft_ev_release(ps); return (0); } static int soft_stop_pmc(int cpu, int ri) { struct pmc *pm; struct soft_cpu *pc; struct pmc_soft *ps; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pc = soft_pcpu[cpu]; pm = pc->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); ps = pmc_soft_ev_acquire(pm->pm_event); /* event unregistered ? */ if (ps != NULL) { atomic_subtract_int(&ps->ps_running, 1); pmc_soft_ev_release(ps); } return (0); } int pmc_soft_intr(struct pmckern_soft *ks) { struct pmc *pm; struct soft_cpu *pc; int ri, processed, error, user_mode; KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); processed = 0; pc = soft_pcpu[ks->pm_cpu]; for (ri = 0; ri < SOFT_NPMCS; ri++) { pm = pc->soft_hw[ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || pm->pm_event != ks->pm_ev) { continue; } processed = 1; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { if ((pc->soft_values[ri]--) <= 0) pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; else continue; user_mode = TRAPF_USERMODE(ks->pm_tf); error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf); if (error) { soft_stop_pmc(ks->pm_cpu, ri); continue; } if (user_mode) { /* If in user mode setup AST to process * callchain out of interrupt context. */ curthread->td_flags |= TDF_ASTPENDING; } } else pc->soft_values[ri]++; } if (processed) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); return (processed); } void pmc_soft_initialize(struct pmc_mdep *md) { struct pmc_classdep *pcd; /* Add SOFT PMCs. */ soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, M_ZERO|M_WAITOK); pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; pcd->pcd_caps = SOFT_CAPS; pcd->pcd_class = PMC_CLASS_SOFT; pcd->pcd_num = SOFT_NPMCS; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 64; pcd->pcd_allocate_pmc = soft_allocate_pmc; pcd->pcd_config_pmc = soft_config_pmc; pcd->pcd_describe = soft_describe; pcd->pcd_get_config = soft_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_init = soft_pcpu_init; pcd->pcd_pcpu_fini = soft_pcpu_fini; pcd->pcd_read_pmc = soft_read_pmc; pcd->pcd_write_pmc = soft_write_pmc; pcd->pcd_release_pmc = soft_release_pmc; pcd->pcd_start_pmc = soft_start_pmc; pcd->pcd_stop_pmc = soft_stop_pmc; md->pmd_npmc += SOFT_NPMCS; } void pmc_soft_finalize(struct pmc_mdep *md) { #ifdef INVARIANTS int i, ncpus; ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", __LINE__, i)); KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); #endif free(soft_pcpu, M_PMC); soft_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_tsc.c b/sys/dev/hwpmc/hwpmc_tsc.c index d59c8908f4ca..25b2e9f8fd57 100644 --- a/sys/dev/hwpmc/hwpmc_tsc.c +++ b/sys/dev/hwpmc/hwpmc_tsc.c @@ -1,381 +1,376 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * TSC support. */ #define TSC_CAPS PMC_CAP_READ struct tsc_descr { struct pmc_descr pm_descr; /* "base class" */ }; static struct tsc_descr tsc_pmcdesc[TSC_NPMCS] = { { .pm_descr = { .pd_name = "TSC", .pd_class = PMC_CLASS_TSC, .pd_caps = TSC_CAPS, .pd_width = 64 } } }; /* * Per-CPU data structure for TSCs. */ struct tsc_cpu { struct pmc_hw tc_hw; }; static struct tsc_cpu **tsc_pcpu; static int tsc_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_TSC) return (EINVAL); if (a->pm_ev != PMC_EV_TSC_TSC || a->pm_mode != PMC_MODE_SC) return (EINVAL); return (0); } static int tsc_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[tsc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int tsc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct tsc_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pd = &tsc_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return (error); - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int tsc_get_config(int cpu, int ri, struct pmc **ppm) { (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); *ppm = tsc_pcpu[cpu]->tc_hw.phw_pmc; return (0); } static int tsc_get_msr(int ri, uint32_t *msr) { (void) ri; KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] ri %d out of range", __LINE__, ri)); *msr = MSR_TSC; return (0); } static int tsc_pcpu_fini(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu[cpu] != NULL, ("[tsc,%d] null pcpu", __LINE__)); free(tsc_pcpu[cpu], M_PMC); tsc_pcpu[cpu] = NULL; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; return (0); } static int tsc_pcpu_init(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; struct tsc_cpu *tsc_pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu, ("[tsc,%d] null pcpu", __LINE__)); KASSERT(tsc_pcpu[cpu] == NULL, ("[tsc,%d] non-null per-cpu", __LINE__)); tsc_pc = malloc(sizeof(struct tsc_cpu), M_PMC, M_WAITOK|M_ZERO); tsc_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | PMC_PHW_FLAG_IS_SHAREABLE; tsc_pcpu[cpu] = tsc_pc; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; KASSERT(pmc_pcpu, ("[tsc,%d] null generic pcpu", __LINE__)); pc = pmc_pcpu[cpu]; KASSERT(pc, ("[tsc,%d] null generic per-cpu", __LINE__)); pc->pc_hwpmcs[ri] = &tsc_pc->tc_hw; return (0); } static int tsc_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; enum pmc_mode mode __diagused; const struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal ri %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[tsc,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); KASSERT(mode == PMC_MODE_SC, ("[tsc,%d] illegal pmc mode %d", __LINE__, mode)); PMCDBG1(MDP,REA,1,"tsc-read id=%d", ri); *v = rdtsc(); return (0); } static int tsc_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(phw->phw_pmc == NULL, ("[tsc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); /* * Nothing to do. */ return (0); } static int tsc_start_pmc(int cpu, int ri) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* TSCs are always running. */ } static int tsc_stop_pmc(int cpu, int ri) { (void) cpu; (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* Cannot actually stop a TSC. */ } static int tsc_write_pmc(int cpu, int ri, pmc_value_t v) { (void) cpu; (void) ri; (void) v; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); /* * The TSCs are used as timecounters by the kernel, so even * though some i386 CPUs support writeable TSCs, we don't * support writing changing TSC values through the HWPMC API. */ return (0); } int pmc_tsc_initialize(struct pmc_mdep *md, int maxcpu) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[tsc,%d] md is NULL", __LINE__)); KASSERT(md->pmd_nclass >= 1, ("[tsc,%d] dubious md->nclass %d", __LINE__, md->pmd_nclass)); tsc_pcpu = malloc(sizeof(struct tsc_cpu *) * maxcpu, M_PMC, M_ZERO|M_WAITOK); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC]; pcd->pcd_caps = PMC_CAP_READ; pcd->pcd_class = PMC_CLASS_TSC; pcd->pcd_num = TSC_NPMCS; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 64; pcd->pcd_allocate_pmc = tsc_allocate_pmc; pcd->pcd_config_pmc = tsc_config_pmc; pcd->pcd_describe = tsc_describe; pcd->pcd_get_config = tsc_get_config; pcd->pcd_get_msr = tsc_get_msr; pcd->pcd_pcpu_init = tsc_pcpu_init; pcd->pcd_pcpu_fini = tsc_pcpu_fini; pcd->pcd_read_pmc = tsc_read_pmc; pcd->pcd_release_pmc = tsc_release_pmc; pcd->pcd_start_pmc = tsc_start_pmc; pcd->pcd_stop_pmc = tsc_stop_pmc; pcd->pcd_write_pmc = tsc_write_pmc; md->pmd_npmc += TSC_NPMCS; return (0); } void pmc_tsc_finalize(struct pmc_mdep *md) { #ifdef INVARIANTS int i, ncpus; ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(tsc_pcpu[i] == NULL, ("[tsc,%d] non-null pcpu cpu %d", __LINE__, i)); KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_class == PMC_CLASS_TSC, ("[tsc,%d] class mismatch", __LINE__)); #else (void) md; #endif free(tsc_pcpu, M_PMC); tsc_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_uncore.c b/sys/dev/hwpmc/hwpmc_uncore.c index c9ae128b55cf..47d1e62e2698 100644 --- a/sys/dev/hwpmc/hwpmc_uncore.c +++ b/sys/dev/hwpmc/hwpmc_uncore.c @@ -1,821 +1,809 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Uncore PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if (__FreeBSD_version >= 1100000) #include #else #include #endif #include #include #include #define UCF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE) #define UCP_PMC_CAPS \ (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define SELECTSEL(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCP_CB0_EVSEL0 : UCP_EVSEL0) #define SELECTOFF(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCF_OFFSET_SB : UCF_OFFSET) static enum pmc_cputype uncore_cputype; struct uncore_cpu { volatile uint32_t pc_ucfctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_uncorepmcs[]; }; static struct uncore_cpu **uncore_pcpu; static uint64_t uncore_pmcmask; static int uncore_ucf_ri; /* relative index of fixed counters */ static int uncore_ucf_width; static int uncore_ucf_npmc; static int uncore_ucp_width; static int uncore_ucp_npmc; static int uncore_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int uncore_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct uncore_cpu *cc; struct pmc_hw *phw; int uncore_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); uncore_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + uncore_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + uncore_ri] = phw; } return (0); } static int uncore_pcpu_fini(struct pmc_mdep *md, int cpu) { int uncore_ri, n, npmc; struct pmc_cpu *pc; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); if ((cc = uncore_pcpu[cpu]) == NULL) return (0); uncore_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(SELECTSEL(uncore_cputype) + n, 0); wrmsr(UCF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + uncore_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t ucf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (uncore_ucf_width - 1))) == 0) return (0); v &= (1ULL << uncore_ucf_width) - 1; return (1ULL << uncore_ucf_width) - v; } static pmc_value_t ucf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucf_width) - rlc; } static int ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t flags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > uncore_ucf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_UCF) return (EINVAL); flags = UCF_EN; pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); return (0); } static int ucf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; return (0); } static int ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char ucf_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; - (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); - if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "UCF-%d", ri); pi->pm_class = PMC_CLASS_UCF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; return (0); } static int ucf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + uncore_ucf_ri)); tmp = rdmsr(UCF_CTR0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); return (0); } static int ucf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); } static void ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; pcd->pcd_caps = UCF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucf_allocate_pmc; pcd->pcd_config_pmc = ucf_config_pmc; pcd->pcd_describe = ucf_describe; pcd->pcd_get_config = ucf_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_noop; pcd->pcd_pcpu_init = uncore_pcpu_noop; pcd->pcd_read_pmc = ucf_read_pmc; pcd->pcd_release_pmc = ucf_release_pmc; pcd->pcd_start_pmc = ucf_start_pmc; pcd->pcd_stop_pmc = ucf_stop_pmc; pcd->pcd_write_pmc = ucf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct ucp_event_descr { enum pmc_event ucp_ev; unsigned char ucp_evcode; unsigned char ucp_umask; unsigned char ucp_flags; }; #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ #define UCP_F_WM (1 << 1) /* CPU: Westmere */ #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ #define UCP_F_HW (1 << 3) /* CPU: Haswell */ #define UCP_F_FM (1 << 4) /* Fixed mask */ #define UCP_F_ALLCPUS \ (UCP_F_I7 | UCP_F_WM) #define UCP_F_CMASK 0xFF000000 static pmc_value_t ucp_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucp_width) - 1; return (1ULL << uncore_ucp_width) - v; } static pmc_value_t ucp_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucp_width) - rlc; } /* * Counter specific event information for Sandybridge and Haswell */ static int ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) { uint32_t mask; switch (ev) { /* * Events valid only on counter 0. */ case 0x80: case 0x83: mask = (1 << 0); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_ucp_op_pmcallocate *ucp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_UCP) return (EINVAL); ucp = &a->pm_md.pm_ucp; ev = UCP_EVSEL(ucp->pm_ucp_config); switch (uncore_cputype) { case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_SANDYBRIDGE: if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: break; } pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; return (0); } static int ucp_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; return (0); } static int ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char ucp_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; - (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); - if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "UCP-%d", ri); pi->pm_class = PMC_CLASS_UCP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucp_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; return (0); } static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); } static int ucp_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; uint64_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); return (0); } static int ucp_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); /* stop hw. */ wrmsr(SELECTSEL(uncore_cputype) + ri, 0); /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ return (0); } static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); } static void ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucp-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; pcd->pcd_caps = UCP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucp_allocate_pmc; pcd->pcd_config_pmc = ucp_config_pmc; pcd->pcd_describe = ucp_describe; pcd->pcd_get_config = ucp_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_fini; pcd->pcd_pcpu_init = uncore_pcpu_init; pcd->pcd_read_pmc = ucp_read_pmc; pcd->pcd_release_pmc = ucp_release_pmc; pcd->pcd_start_pmc = ucp_start_pmc; pcd->pcd_stop_pmc = ucp_stop_pmc; pcd->pcd_write_pmc = ucp_write_pmc; md->pmd_npmc += npmc; } int pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) { uncore_cputype = md->pmd_cputype; uncore_pmcmask = 0; /* * Initialize programmable counters. */ uncore_ucp_npmc = 8; uncore_ucp_width = 48; uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); /* * Initialize fixed function counters, if present. */ uncore_ucf_ri = uncore_ucp_npmc; uncore_ucf_npmc = 1; uncore_ucf_width = 48; ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, uncore_ucf_ri); uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); return (0); } void pmc_uncore_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "uncore-finalize"); free(uncore_pcpu, M_PMC); uncore_pcpu = NULL; }