diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 936974080fed..c35f5fd4f095 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -1,1245 +1,1240 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* Support for the AMD K7 and later processors */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_DEBUG enum pmc_class amd_pmc_class; #endif #define OVERFLOW_WAIT_COUNT 50 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter); /* AMD K7 & K8 PMCs */ struct amd_descr { struct pmc_descr pm_descr; /* "base class" */ uint32_t pm_evsel; /* address of EVSEL register */ uint32_t pm_perfctr; /* address of PERFCTR register */ }; static struct amd_descr amd_pmcdesc[AMD_NPMCS] = { { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_0, .pm_perfctr = AMD_PMC_PERFCTR_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_1, .pm_perfctr = AMD_PMC_PERFCTR_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_2, .pm_perfctr = AMD_PMC_PERFCTR_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_3, .pm_perfctr = AMD_PMC_PERFCTR_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_4, .pm_perfctr = AMD_PMC_PERFCTR_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_5, .pm_perfctr = AMD_PMC_PERFCTR_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_3 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_4, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_4 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_L3_5, .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_5 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_0, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_0 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_1, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_1 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_2, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_2 }, { .pm_descr = { .pd_name = "", .pd_class = -1, .pd_caps = AMD_PMC_CAPS, .pd_width = 48 }, .pm_evsel = AMD_PMC_EVSEL_EP_DF_3, .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_3 } }; struct amd_event_code_map { enum pmc_event pe_ev; /* enum value */ uint16_t pe_code; /* encoded event mask */ uint8_t pe_mask; /* bits allowed in unit mask */ }; const struct amd_event_code_map amd_event_codes[] = { #if defined(__i386__) /* 32 bit Athlon (K7) only */ { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, { PMC_EV_K7_DC_MISSES, 0x41, 0 }, { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, { PMC_EV_K7_IC_MISSES, 0x81, 0 }, { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, #endif { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 0x21, 0x00 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 0xCA, 0x00 }, { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 0xCC, 0x07 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 0xD2, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 0xD5, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 0xD6, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 0xD9, 0x00 }, { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 0xDA, 0x00 }, { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 0xE2, 0x00 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } }; const int amd_event_codes_size = nitems(amd_event_codes); /* * Per-processor information */ struct amd_cpu { struct pmc_hw pc_amdpmcs[AMD_NPMCS]; }; static struct amd_cpu **amd_pcpu; /* * read a pmc register */ static int amd_read_pmc(int cpu, int ri, pmc_value_t *v) { enum pmc_mode mode; const struct amd_descr *pd; struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); KASSERT(amd_pcpu[cpu], ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(mode)) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ULL << 47)) == 0) tmp = 0; else { /* Sign extend 48 bit value to 64 bits. */ tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16); tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } } *v = tmp; PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v); return 0; } /* * Write a PMC MSR. */ static int amd_write_pmc(int cpu, int ri, pmc_value_t v) { const struct amd_descr *pd; enum pmc_mode mode; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); #ifdef HWPMC_DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, pd->pm_descr.pd_class)); #endif /* use 2's complement of the count for sampling mode PMCs */ if (PMC_IS_SAMPLING_MODE(mode)) v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); /* write the PMC value */ wrmsr(pd->pm_perfctr, v); return 0; } /* * configure hardware pmc according to the configuration recorded in * pmc 'pm'. */ static int amd_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } /* * Retrieve a configured PMC pointer from hardware state. */ static int amd_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; return 0; } /* * Machine dependent actions taken during the context switch in of a * thread. */ static int amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); /* enable the RDPMC instruction if needed */ if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) load_cr4(rcr4() | CR4_PCE); return 0; } /* * Machine dependent actions taken during the context switch out of a * thread. */ static int amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; (void) pp; /* can be NULL */ PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); /* always turn off the RDPMC instruction */ load_cr4(rcr4() & ~CR4_PCE); return 0; } /* * Check if a given allocation is feasible. */ static int amd_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int i; uint64_t allowed_unitmask, caps, config, unitmask; enum pmc_event pe; const struct pmc_descr *pd; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row index %d", __LINE__, ri)); pd = &amd_pmcdesc[ri].pm_descr; /* check class match */ if (pd->pd_class != a->pm_class) return EINVAL; caps = pm->pm_caps; PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); if((ri >= 0 && ri < 6) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_CORE)) return EINVAL; if((ri >= 6 && ri < 12) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_L3_CACHE)) return EINVAL; if((ri >= 12 && ri < 16) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_DATA_FABRIC)) return EINVAL; if (strlen(pmc_cpuid) != 0) { pm->pm_md.pm_amd.pm_amd_evsel = a->pm_md.pm_amd.pm_amd_config; PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, a->pm_md.pm_amd.pm_amd_config); return (0); } pe = a->pm_ev; /* map ev to the correct event mask code */ config = allowed_unitmask = 0; for (i = 0; i < amd_event_codes_size; i++) if (amd_event_codes[i].pe_ev == pe) { config = AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); allowed_unitmask = AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); break; } if (i == amd_event_codes_size) return EINVAL; unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ return EINVAL; if (unitmask && (caps & PMC_CAP_QUALIFIER)) config |= unitmask; if (caps & PMC_CAP_THRESHOLD) config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; /* set at least one of the 'usr' or 'os' caps */ if (caps & PMC_CAP_USER) config |= AMD_PMC_USR; if (caps & PMC_CAP_SYSTEM) config |= AMD_PMC_OS; if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) config |= (AMD_PMC_USR|AMD_PMC_OS); if (caps & PMC_CAP_EDGE) config |= AMD_PMC_EDGE; if (caps & PMC_CAP_INVERT) config |= AMD_PMC_INVERT; if (caps & PMC_CAP_INTERRUPT) config |= AMD_PMC_INT; pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); return 0; } /* * Release machine dependent state associated with a PMC. This is a * no-op on this architecture. * */ /* ARGSUSED0 */ static int amd_release_pmc(int cpu, int ri, struct pmc *pmc) { #ifdef HWPMC_DEBUG const struct amd_descr *pd; #endif struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); #ifdef HWPMC_DEBUG pd = &amd_pmcdesc[ri]; if (pd->pm_descr.pd_class == amd_pmc_class) KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC %d released while active", __LINE__, ri)); #endif return 0; } /* * start a PMC. */ static int amd_start_pmc(int cpu, int ri) { uint64_t config; struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, ri, cpu, pd->pm_descr.pd_name)); /* turn on the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config); wrmsr(pd->pm_evsel, config); return 0; } /* * Stop a PMC. */ static int amd_stop_pmc(int cpu, int ri) { struct pmc *pm; struct pmc_hw *phw; const struct amd_descr *pd; uint64_t config; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", __LINE__, ri, cpu, pd->pm_descr.pd_name)); PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri); /* turn off the PMC ENABLE bit */ config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; wrmsr(pd->pm_evsel, config); /* * Due to NMI latency on newer AMD processors * NMI interrupts are ignored, which leads to * panic or messages based on kernel configuration */ /* Wait for the count to be reset */ for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1))) break; DELAY(1); } return 0; } /* * Interrupt handler. This function needs to return '1' if the * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed * to sleep or do anything a 'fast' interrupt handler is not allowed * to do. */ static int amd_intr(struct trapframe *tf) { int i, error, retval, cpu; uint64_t config, evsel, perfctr; struct pmc *pm; struct amd_cpu *pac; pmc_value_t v; uint32_t active = 0, count = 0; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pac = amd_pcpu[cpu]; /* * look for all PMCs that have interrupted: * - look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association * * If found, we call a helper to process the interrupt. * * PMCs interrupting at the same time are collapsed into * a single interrupt. Check all the valid pmcs for * overflow. */ for (i = 0; i < AMD_CORE_NPMCS; i++) { if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; } /* Consider pmc with valid handle as active */ active++; if (!AMD_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* Stop the PMC, reload count. */ evsel = amd_pmcdesc[i].pm_evsel; perfctr = amd_pmcdesc[i].pm_perfctr; v = pm->pm_sc.pm_reloadcount; config = rdmsr(evsel); KASSERT((config & ~AMD_PMC_ENABLE) == (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__, (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel)); wrmsr(evsel, config & ~AMD_PMC_ENABLE); wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); /* Restart the counter if logging succeeded. */ error = pmc_process_interrupt(PMC_HR, pm, tf); if (error == 0) wrmsr(evsel, config); } /* * Due to NMI latency, there can be a scenario in which * multiple pmcs gets serviced in an earlier NMI and we * do not find an overflow in the subsequent NMI. * * For such cases we keep a per-cpu count of active NMIs * and compare it with min(active pmcs, 2) to determine * if this NMI was for a pmc overflow which was serviced * in an earlier request or should be ignored. */ if (retval) { DPCPU_SET(nmi_counter, min(2, active)); } else { if ((count = DPCPU_GET(nmi_counter))) { retval = 1; DPCPU_SET(nmi_counter, --count); } } if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); PMCDBG1(MDP,INT,2, "retval=%d", retval); return (retval); } /* * describe a PMC */ static int amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct amd_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] row-index %d out of range", __LINE__, ri)); phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pd = &amd_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return error; - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return 0; } /* * i386 specific entry points */ /* * return the MSR address of the given PMC. */ static int amd_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] ri %d out of range", __LINE__, ri)); *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; return (0); } /* * processor dependent initialization. */ static int amd_pcpu_init(struct pmc_mdep *md, int cpu) { int classindex, first_ri, n; struct pmc_cpu *pc; struct amd_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu); amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, M_WAITOK|M_ZERO); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * processor dependent cleanup prior to the KLD * being unloaded */ static int amd_pcpu_fini(struct pmc_mdep *md, int cpu) { int classindex, first_ri, i; uint32_t evsel; struct pmc_cpu *pc; struct amd_cpu *pac; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu); /* * First, turn off all PMCs on this CPU. */ for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ evsel = rdmsr(AMD_PMC_EVSEL_0 + i); evsel &= ~AMD_PMC_ENABLE; wrmsr(AMD_PMC_EVSEL_0 + i, evsel); } /* * Next, free up allocated space. */ if ((pac = amd_pcpu[cpu]) == NULL) return (0); amd_pcpu[cpu] = NULL; #ifdef HWPMC_DEBUG for (i = 0; i < AMD_NPMCS; i++) { KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i), ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); } #endif pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); #if defined(__amd64__) classindex = PMC_MDEP_CLASS_INDEX_K8; #elif defined(__i386__) classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; #endif first_ri = md->pmd_classdep[classindex].pcd_ri; /* * Reset pointers in the MI 'per-cpu' state. */ for (i = 0; i < AMD_NPMCS; i++) { pc->pc_hwpmcs[i + first_ri] = NULL; } free(pac, M_PMC); return (0); } /* * Initialize ourselves. */ struct pmc_mdep * pmc_amd_initialize(void) { int classindex, error, i, ncpus; struct pmc_classdep *pcd; enum pmc_cputype cputype; struct pmc_mdep *pmc_mdep; enum pmc_class class; int family, model, stepping; char *name; /* * The presence of hardware performance counters on the AMD * Athlon, Duron or later processors, is _not_ indicated by * any of the processor feature flags set by the 'CPUID' * instruction, so we only check the 'instruction family' * field returned by CPUID for instruction family >= 6. */ name = NULL; family = CPUID_TO_FAMILY(cpu_id); model = CPUID_TO_MODEL(cpu_id); stepping = CPUID_TO_STEPPING(cpu_id); if (family == 0x18) snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X", family, model, stepping); else snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X", family, model, stepping); switch (cpu_id & 0xF00) { #if defined(__i386__) case 0x600: /* Athlon(tm) processor */ classindex = PMC_MDEP_CLASS_INDEX_K7; cputype = PMC_CPU_AMD_K7; class = PMC_CLASS_K7; name = "K7"; break; #endif case 0xF00: /* Athlon64/Opteron processor */ classindex = PMC_MDEP_CLASS_INDEX_K8; cputype = PMC_CPU_AMD_K8; class = PMC_CLASS_K8; name = "K8"; break; default: (void) printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, (cpu_id & 0xF00) >> 8, model); return NULL; } #ifdef HWPMC_DEBUG amd_pmc_class = class; #endif /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* * These processors have two classes of PMCs: the TSC and * programmable PMCs. */ pmc_mdep = pmc_mdep_alloc(2); pmc_mdep->pmd_cputype = cputype; ncpus = pmc_cpu_max(); /* Initialize the TSC. */ error = pmc_tsc_initialize(pmc_mdep, ncpus); if (error) goto error; /* Initialize AMD K7 and K8 PMC handling. */ pcd = &pmc_mdep->pmd_classdep[classindex]; pcd->pcd_caps = AMD_PMC_CAPS; pcd->pcd_class = class; pcd->pcd_num = AMD_NPMCS; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 48; /* fill in the correct pmc name and class */ for (i = 0; i < AMD_NPMCS; i++) { (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", name, i); amd_pmcdesc[i].pm_descr.pd_class = class; } pcd->pcd_allocate_pmc = amd_allocate_pmc; pcd->pcd_config_pmc = amd_config_pmc; pcd->pcd_describe = amd_describe; pcd->pcd_get_config = amd_get_config; pcd->pcd_get_msr = amd_get_msr; pcd->pcd_pcpu_fini = amd_pcpu_fini; pcd->pcd_pcpu_init = amd_pcpu_init; pcd->pcd_read_pmc = amd_read_pmc; pcd->pcd_release_pmc = amd_release_pmc; pcd->pcd_start_pmc = amd_start_pmc; pcd->pcd_stop_pmc = amd_stop_pmc; pcd->pcd_write_pmc = amd_write_pmc; pmc_mdep->pmd_pcpu_init = NULL; pmc_mdep->pmd_pcpu_fini = NULL; pmc_mdep->pmd_intr = amd_intr; pmc_mdep->pmd_switch_in = amd_switch_in; pmc_mdep->pmd_switch_out = amd_switch_out; pmc_mdep->pmd_npmc += AMD_NPMCS; PMCDBG0(MDP,INI,0,"amd-initialize"); return (pmc_mdep); error: if (error) { free(pmc_mdep, M_PMC); pmc_mdep = NULL; } return (NULL); } /* * Finalization code for AMD CPUs. */ void pmc_amd_finalize(struct pmc_mdep *md) { #if defined(INVARIANTS) int classindex, i, ncpus, pmcclass; #endif pmc_tsc_finalize(md); KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer", __LINE__)); #if defined(INVARIANTS) switch (md->pmd_cputype) { #if defined(__i386__) case PMC_CPU_AMD_K7: classindex = PMC_MDEP_CLASS_INDEX_K7; pmcclass = PMC_CLASS_K7; break; #endif default: classindex = PMC_MDEP_CLASS_INDEX_K8; pmcclass = PMC_CLASS_K8; } KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass, ("[amd,%d] pmc class mismatch", __LINE__)); ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu", __LINE__)); #endif free(amd_pcpu, M_PMC); amd_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_arm64.c b/sys/dev/hwpmc/hwpmc_arm64.c index ec796bcc8ff5..baf05e52b42a 100644 --- a/sys/dev/hwpmc/hwpmc_arm64.c +++ b/sys/dev/hwpmc/hwpmc_arm64.c @@ -1,642 +1,639 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by the University of Cambridge Computer * Laboratory with support from ARM Ltd. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include "opt_acpi.h" static int arm64_npmcs; struct arm64_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; /* * Per-processor information. */ struct arm64_cpu { struct pmc_hw *pc_arm64pmcs; }; static struct arm64_cpu **arm64_pcpu; /* * Interrupt Enable Set Register */ static __inline void arm64_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenset_el1, reg); isb(); } /* * Interrupt Clear Set Register */ static __inline void arm64_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmintenclr_el1, reg); isb(); } /* * Counter Set Enable Register */ static __inline void arm64_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenset_el0, reg); isb(); } /* * Counter Clear Enable Register */ static __inline void arm64_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); WRITE_SPECIALREG(pmcntenclr_el0, reg); isb(); } /* * Performance Monitors Control Register */ static uint32_t arm64_pmcr_read(void) { uint32_t reg; reg = READ_SPECIALREG(pmcr_el0); return (reg); } static void arm64_pmcr_write(uint32_t reg) { WRITE_SPECIALREG(pmcr_el0, reg); isb(); } /* * Performance Count Register N */ static uint32_t arm64_pmcn_read(unsigned int pmc) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); isb(); return (READ_SPECIALREG(pmxevcntr_el0)); } static void arm64_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); WRITE_SPECIALREG(pmselr_el0, pmc); WRITE_SPECIALREG(pmxevcntr_el0, reg); isb(); } static int arm64_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t config; enum pmc_event pe; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_ARMV8) { return (EINVAL); } pe = a->pm_ev; /* Adjust the config value if needed. */ config = a->pm_md.pm_md_config; if ((a->pm_md.pm_md_flags & PM_MD_RAW_EVENT) == 0) { config = (uint32_t)pe - PMC_EV_ARMV8_FIRST; if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST)) return (EINVAL); } switch (a->pm_caps & (PMC_CAP_SYSTEM | PMC_CAP_USER)) { case PMC_CAP_SYSTEM: config |= PMEVTYPER_U; break; case PMC_CAP_USER: config |= PMEVTYPER_P; break; default: /* * Trace both USER and SYSTEM if none are specified * (default setting) or if both flags are specified * (user explicitly requested both qualifiers). */ break; } pm->pm_md.pm_arm64.pm_arm64_evsel = config; PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config); return (0); } static int arm64_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; /* * Ensure we don't get interrupted while updating the overflow count. */ s = intr_disable(); tmp = arm64_pmcn_read(ri); reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) { /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = arm64_pmcn_read(ri); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ull << 63)) == 0) tmp = 0; else tmp = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } *v = tmp; return (0); } static int arm64_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; arm64_pmcn_write(ri, v); return (0); } static int arm64_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int arm64_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_arm64.pm_arm64_evsel; /* * Configure the event selection. */ WRITE_SPECIALREG(pmselr_el0, ri); WRITE_SPECIALREG(pmxevtyper_el0, config); isb(); /* * Enable the PMC. */ arm64_interrupt_enable(ri); arm64_counter_enable(ri); return (0); } static int arm64_stop_pmc(int cpu, int ri) { /* * Disable the PMCs. */ arm64_counter_disable(ri); arm64_interrupt_disable(ri); return (0); } static int arm64_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] illegal row-index %d", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } static int arm64_intr(struct trapframe *tf) { int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] CPU %d out of range", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf, TRAPF_USERMODE(tf)); retval = 0; for (ri = 0; ri < arm64_npmcs; ri++) { pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter is overflowed */ reg = (1 << ri); if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0) continue; /* Clear Overflow Flag */ WRITE_SPECIALREG(pmovsclr_el0, reg); isb(); retval = 1; /* Found an interrupting PMC. */ pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) arm64_stop_pmc(cpu, ri); /* Reload sampling count */ arm64_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - char arm64_name[PMC_NAME_MAX]; struct pmc_hw *phw; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < arm64_npmcs, ("[arm64,%d] row-index %d out of range", __LINE__, ri)); phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri]; - snprintf(arm64_name, sizeof(arm64_name), "ARMV8-%d", ri); - if ((error = copystr(arm64_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); + + snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV8-%d", ri); pi->pm_class = PMC_CLASS_ARMV8; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int arm64_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc; return (0); } /* * XXX don't know what we should do here. */ static int arm64_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int arm64_pcpu_init(struct pmc_mdep *md, int cpu) { struct arm64_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint64_t pmcr; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[arm64,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "arm64-init cpu=%d", cpu); arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC, M_WAITOK | M_ZERO); pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs, M_PMC, M_WAITOK | M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri; KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* * Disable all counters and overflow interrupts. Upon reset they are in * an undefined state. * * Don't issue an isb here, just wait for the one in arm64_pmcr_write() * to make the writes visible. */ WRITE_SPECIALREG(pmcntenclr_el0, 0xffffffff); WRITE_SPECIALREG(pmintenclr_el1, 0xffffffff); /* Enable unit */ pmcr = arm64_pmcr_read(); pmcr |= PMCR_E; arm64_pmcr_write(pmcr); return (0); } static int arm64_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmcr; pmcr = arm64_pmcr_read(); pmcr &= ~PMCR_E; arm64_pmcr_write(pmcr); return (0); } struct pmc_mdep * pmc_arm64_initialize(void) { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int classes, idcode, impcode; int reg; uint64_t midr; reg = arm64_pmcr_read(); arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT; impcode = (reg & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT; idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs); /* * Write the CPU model to kern.hwpmc.cpuid. * * We zero the variant and revision fields. * * TODO: how to handle differences between cores due to big.LITTLE? * For now, just use MIDR from CPU 0. */ midr = (uint64_t)(pcpu_find(0)->pc_midr); midr &= ~(CPU_VAR_MASK | CPU_REV_MASK); snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* One AArch64 CPU class */ classes = 1; #ifdef DEV_ACPI /* Query presence of optional classes and set max class. */ if (pmc_cmn600_nclasses() > 0) classes = MAX(classes, PMC_MDEP_CLASS_INDEX_CMN600); if (pmc_dmc620_nclasses() > 0) classes = MAX(classes, PMC_MDEP_CLASS_INDEX_DMC620_C); #endif pmc_mdep = pmc_mdep_alloc(classes); switch(impcode) { case PMCR_IMP_ARM: switch (idcode) { case PMCR_IDCODE_CORTEX_A76: case PMCR_IDCODE_NEOVERSE_N1: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76; break; case PMCR_IDCODE_CORTEX_A57: case PMCR_IDCODE_CORTEX_A72: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57; break; default: case PMCR_IDCODE_CORTEX_A53: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } break; default: pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8]; pcd->pcd_caps = ARMV8_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV8; pcd->pcd_num = arm64_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = arm64_allocate_pmc; pcd->pcd_config_pmc = arm64_config_pmc; pcd->pcd_pcpu_fini = arm64_pcpu_fini; pcd->pcd_pcpu_init = arm64_pcpu_init; pcd->pcd_describe = arm64_describe; pcd->pcd_get_config = arm64_get_config; pcd->pcd_read_pmc = arm64_read_pmc; pcd->pcd_release_pmc = arm64_release_pmc; pcd->pcd_start_pmc = arm64_start_pmc; pcd->pcd_stop_pmc = arm64_stop_pmc; pcd->pcd_write_pmc = arm64_write_pmc; pmc_mdep->pmd_intr = arm64_intr; pmc_mdep->pmd_switch_in = arm64_switch_in; pmc_mdep->pmd_switch_out = arm64_switch_out; pmc_mdep->pmd_npmc += arm64_npmcs; #ifdef DEV_ACPI if (pmc_cmn600_nclasses() > 0) pmc_cmn600_initialize(pmc_mdep); if (pmc_dmc620_nclasses() > 0) { pmc_dmc620_initialize_cd2(pmc_mdep); pmc_dmc620_initialize_c(pmc_mdep); } #endif return (pmc_mdep); } void pmc_arm64_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_armv7.c b/sys/dev/hwpmc/hwpmc_armv7.c index 1ba1dbb6e695..34f3fd009c9d 100644 --- a/sys/dev/hwpmc/hwpmc_armv7.c +++ b/sys/dev/hwpmc/hwpmc_armv7.c @@ -1,570 +1,567 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static int armv7_npmcs; struct armv7_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; #define PMC_EV_CPU_CYCLES 0xFF /* * Per-processor information. */ struct armv7_cpu { struct pmc_hw *pc_armv7pmcs; }; static struct armv7_cpu **armv7_pcpu; /* * Interrupt Enable Set Register */ static __inline void armv7_interrupt_enable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_set(reg); } /* * Interrupt Clear Set Register */ static __inline void armv7_interrupt_disable(uint32_t pmc) { uint32_t reg; reg = (1 << pmc); cp15_pminten_clr(reg); } /* * Counter Set Enable Register */ static __inline void armv7_counter_enable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_set(reg); } /* * Counter Clear Enable Register */ static __inline void armv7_counter_disable(unsigned int pmc) { uint32_t reg; reg = (1 << pmc); cp15_pmcnten_clr(reg); } /* * Performance Count Register N */ static uint32_t armv7_pmcn_read(unsigned int pmc, uint32_t evsel) { if (evsel == PMC_EV_CPU_CYCLES) { return ((uint32_t)cp15_pmccntr_get()); } KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); return (cp15_pmxevcntr_get()); } static uint32_t armv7_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc)); cp15_pmselr_set(pmc); cp15_pmxevcntr_set(reg); return (reg); } static int armv7_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t config; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_ARMV7) return (EINVAL); pe = a->pm_ev; config = (pe & EVENT_ID_MASK); pm->pm_md.pm_armv7.pm_armv7_evsel = config; PMCDBG2(MDP, ALL, 2, "armv7-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int armv7_read_pmc(int cpu, int ri, pmc_value_t *v) { pmc_value_t tmp; struct pmc *pm; register_t s; u_int reg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; s = intr_disable(); tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) != 0) { /* Clear Overflow Flag */ cp15_pmovsr_set(reg); pm->pm_pcpu_state[cpu].pps_overflowcnt++; /* Reread counter in case we raced. */ tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel); } tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt; intr_restore(s); PMCDBG2(MDP, REA, 2, "armv7-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Clamp value to 0 if the counter just overflowed, * otherwise the returned reload count would wrap to a * huge value. */ if ((tmp & (1ull << 63)) == 0) tmp = 0; else tmp = ARMV7_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); } *v = tmp; return 0; } static int armv7_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ARMV7_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP, WRI, 1, "armv7-write cpu=%d ri=%d v=%jx", cpu, ri, v); pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32; if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) cp15_pmccntr_set(v); else armv7_pmcn_write(ri, v); return 0; } static int armv7_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[armv7,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int armv7_start_pmc(int cpu, int ri) { struct pmc_hw *phw; uint32_t config; struct pmc *pm; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; /* * Configure the event selection. */ if (config != PMC_EV_CPU_CYCLES) { cp15_pmselr_set(ri); cp15_pmxevtyper_set(config); } else ri = 31; /* * Enable the PMC. */ armv7_interrupt_enable(ri); armv7_counter_enable(ri); return 0; } static int armv7_stop_pmc(int cpu, int ri) { struct pmc_hw *phw; struct pmc *pm; uint32_t config; phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_armv7.pm_armv7_evsel; if (config == PMC_EV_CPU_CYCLES) ri = 31; /* * Disable the PMCs. */ armv7_counter_disable(ri); armv7_interrupt_disable(ri); return 0; } static int armv7_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] illegal row-index %d", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[armv7,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int armv7_intr(struct trapframe *tf) { int retval, ri; struct pmc *pm; int error; int reg, cpu; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; for (ri = 0; ri < armv7_npmcs; ri++) { pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; if (pm == NULL) continue; /* Check if counter has overflowed */ if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES) reg = (1u << 31); else reg = (1u << ri); if ((cp15_pmovsr_get() & reg) == 0) { continue; } /* Clear Overflow Flag */ cp15_pmovsr_set(reg); retval = 1; /* Found an interrupting PMC. */ pm->pm_pcpu_state[cpu].pps_overflowcnt += 1; if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) armv7_stop_pmc(cpu, ri); /* Reload sampling count */ armv7_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); } return (retval); } static int armv7_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - char armv7_name[PMC_NAME_MAX]; struct pmc_hw *phw; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < armv7_npmcs, ("[armv7,%d] row-index %d out of range", __LINE__, ri)); phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri]; - snprintf(armv7_name, sizeof(armv7_name), "ARMV7-%d", ri); - if ((error = copystr(armv7_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV7-%d", ri); pi->pm_class = PMC_CLASS_ARMV7; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int armv7_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc; return 0; } /* * XXX don't know what we should do here. */ static int armv7_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int armv7_pcpu_init(struct pmc_mdep *md, int cpu) { struct armv7_cpu *pac; struct pmc_hw *phw; struct pmc_cpu *pc; uint32_t pmnc; int first_ri; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[armv7,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "armv7-init cpu=%d", cpu); armv7_pcpu[cpu] = pac = malloc(sizeof(struct armv7_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_armv7pmcs = malloc(sizeof(struct pmc_hw) * armv7_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7].pcd_ri; KASSERT(pc != NULL, ("[armv7,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_armv7pmcs; i < armv7_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); /* Enable unit */ pmnc = cp15_pmcr_get(); pmnc |= ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); return 0; } static int armv7_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t pmnc; pmnc = cp15_pmcr_get(); pmnc &= ~ARMV7_PMNC_ENABLE; cp15_pmcr_set(pmnc); pmnc = 0xffffffff; cp15_pmcnten_clr(pmnc); cp15_pminten_clr(pmnc); cp15_pmovsr_set(pmnc); return 0; } struct pmc_mdep * pmc_armv7_initialize(void) { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode; int reg; reg = cp15_pmcr_get(); armv7_npmcs = (reg >> ARMV7_PMNC_N_SHIFT) & \ ARMV7_PMNC_N_MASK; idcode = (reg & ARMV7_IDCODE_MASK) >> ARMV7_IDCODE_SHIFT; PMCDBG1(MDP, INI, 1, "armv7-init npmcs=%d", armv7_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ armv7_pcpu = malloc(sizeof(struct armv7_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); switch (idcode) { case ARMV7_IDCODE_CORTEX_A9: pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A9; break; default: case ARMV7_IDCODE_CORTEX_A8: /* * On A8 we implemented common events only, * so use it for the rest of machines. */ pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A8; break; } pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7]; pcd->pcd_caps = ARMV7_PMC_CAPS; pcd->pcd_class = PMC_CLASS_ARMV7; pcd->pcd_num = armv7_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = armv7_allocate_pmc; pcd->pcd_config_pmc = armv7_config_pmc; pcd->pcd_pcpu_fini = armv7_pcpu_fini; pcd->pcd_pcpu_init = armv7_pcpu_init; pcd->pcd_describe = armv7_describe; pcd->pcd_get_config = armv7_get_config; pcd->pcd_read_pmc = armv7_read_pmc; pcd->pcd_release_pmc = armv7_release_pmc; pcd->pcd_start_pmc = armv7_start_pmc; pcd->pcd_stop_pmc = armv7_stop_pmc; pcd->pcd_write_pmc = armv7_write_pmc; pmc_mdep->pmd_intr = armv7_intr; pmc_mdep->pmd_switch_in = armv7_switch_in; pmc_mdep->pmd_switch_out = armv7_switch_out; pmc_mdep->pmd_npmc += armv7_npmcs; return (pmc_mdep); } void pmc_armv7_finalize(struct pmc_mdep *md) { } diff --git a/sys/dev/hwpmc/hwpmc_cmn600.c b/sys/dev/hwpmc/hwpmc_cmn600.c index fe98fdf969fc..82b6e553801a 100644 --- a/sys/dev/hwpmc/hwpmc_cmn600.c +++ b/sys/dev/hwpmc/hwpmc_cmn600.c @@ -1,826 +1,823 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * Copyright (c) 2021 ARM Ltd * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Arm CoreLink CMN-600 Coherent Mesh Network PMU Driver */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include struct cmn600_descr { struct pmc_descr pd_descr; /* "base class" */ void *pd_rw_arg; /* Argument to use with read/write */ struct pmc *pd_pmc; struct pmc_hw *pd_phw; uint32_t pd_nodeid; int32_t pd_node_type; int pd_local_counter; }; static struct cmn600_descr **cmn600_pmcdesc; static struct cmn600_pmc cmn600_pmcs[CMN600_UNIT_MAX]; static int cmn600_units = 0; static inline struct cmn600_descr * cmn600desc(int ri) { return (cmn600_pmcdesc[ri]); } static inline int class_ri2unit(int ri) { return (ri / CMN600_COUNTERS_N); } #define EVENCNTR(x) (((x) >> POR_DT_PMEVCNT_EVENCNT_SHIFT) << \ POR_DTM_PMEVCNT_CNTR_WIDTH) #define ODDCNTR(x) (((x) >> POR_DT_PMEVCNT_ODDCNT_SHIFT) << \ POR_DTM_PMEVCNT_CNTR_WIDTH) static uint64_t cmn600_pmu_readcntr(void *arg, u_int nodeid, u_int xpcntr, u_int dtccntr, u_int width) { uint64_t dtcval, xpval; KASSERT(xpcntr < 4, ("[cmn600,%d] XP counter number %d is too big." " Max: 3", __LINE__, xpcntr)); KASSERT(dtccntr < 8, ("[cmn600,%d] Global counter number %d is too" " big. Max: 7", __LINE__, dtccntr)); dtcval = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMEVCNT(dtccntr >> 1)); if (width == 4) { dtcval = (dtccntr & 1) ? ODDCNTR(dtcval) : EVENCNTR(dtcval); dtcval &= 0xffffffff0000UL; } else dtcval <<= POR_DTM_PMEVCNT_CNTR_WIDTH; xpval = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMEVCNT); xpval >>= xpcntr * POR_DTM_PMEVCNT_CNTR_WIDTH; xpval &= 0xffffUL; return (dtcval | xpval); } static void cmn600_pmu_writecntr(void *arg, u_int nodeid, u_int xpcntr, u_int dtccntr, u_int width, uint64_t val) { int shift; KASSERT(xpcntr < 4, ("[cmn600,%d] XP counter number %d is too big." " Max: 3", __LINE__, xpcntr)); KASSERT(dtccntr < 8, ("[cmn600,%d] Global counter number %d is too" " big. Max: 7", __LINE__, dtccntr)); if (width == 4) { shift = (dtccntr & 1) ? POR_DT_PMEVCNT_ODDCNT_SHIFT : POR_DT_PMEVCNT_EVENCNT_SHIFT; pmu_cmn600_md8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMEVCNT(dtccntr >> 1), 0xffffffffUL << shift, ((val >> POR_DTM_PMEVCNT_CNTR_WIDTH) & 0xffffffff) << shift); } else pmu_cmn600_wr8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMEVCNT(dtccntr & ~0x1), val >> POR_DTM_PMEVCNT_CNTR_WIDTH); shift = xpcntr * POR_DTM_PMEVCNT_CNTR_WIDTH; val &= 0xffffUL; pmu_cmn600_md8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMEVCNT, 0xffffUL << shift, val << shift); } #undef EVENCNTR #undef ODDCNTR /* * read a pmc register */ static int cmn600_read_pmc(int cpu, int ri, pmc_value_t *v) { int counter, local_counter, nodeid; struct cmn600_descr *desc; struct pmc *pm; void *arg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); counter = ri % CMN600_COUNTERS_N; desc = cmn600desc(ri); pm = desc->pd_phw->phw_pmc; arg = desc->pd_rw_arg; nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; KASSERT(pm != NULL, ("[cmn600,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); *v = cmn600_pmu_readcntr(arg, nodeid, local_counter, counter, 4); PMCDBG3(MDP, REA, 2, "%s id=%d -> %jd", __func__, ri, *v); return (0); } /* * Write a pmc register. */ static int cmn600_write_pmc(int cpu, int ri, pmc_value_t v) { int counter, local_counter, nodeid; struct cmn600_descr *desc; struct pmc *pm; void *arg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); counter = ri % CMN600_COUNTERS_N; desc = cmn600desc(ri); pm = desc->pd_phw->phw_pmc; arg = desc->pd_rw_arg; nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; KASSERT(pm != NULL, ("[cmn600,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, cpu, ri)); PMCDBG4(MDP, WRI, 1, "%s cpu=%d ri=%d v=%jx", __func__, cpu, ri, v); cmn600_pmu_writecntr(arg, nodeid, local_counter, counter, 4, v); return (0); } /* * configure hardware pmc according to the configuration recorded in * pmc 'pm'. */ static int cmn600_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG4(MDP, CFG, 1, "%s cpu=%d ri=%d pm=%p", __func__, cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); phw = cmn600desc(ri)->pd_phw; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[cmn600,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } /* * Retrieve a configured PMC pointer from hardware state. */ static int cmn600_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = cmn600desc(ri)->pd_phw->phw_pmc; return (0); } #define CASE_DN_VER_EVT(n, id) case PMC_EV_CMN600_PMU_ ## n: { *event = id; \ return (0); } static int cmn600_map_ev2event(int ev, int rev, int *node_type, uint8_t *event) { if (ev < PMC_EV_CMN600_PMU_dn_rxreq_dvmop || ev > PMC_EV_CMN600_PMU_rni_rdb_ord) return (EINVAL); if (ev <= PMC_EV_CMN600_PMU_dn_rxreq_trk_full) { *node_type = NODE_TYPE_DVM; if (rev < 0x200) { switch (ev) { CASE_DN_VER_EVT(dn_rxreq_dvmop, 1); CASE_DN_VER_EVT(dn_rxreq_dvmsync, 2); CASE_DN_VER_EVT(dn_rxreq_dvmop_vmid_filtered, 3); CASE_DN_VER_EVT(dn_rxreq_retried, 4); CASE_DN_VER_EVT(dn_rxreq_trk_occupancy, 5); } } else { switch (ev) { CASE_DN_VER_EVT(dn_rxreq_tlbi_dvmop, 0x01); CASE_DN_VER_EVT(dn_rxreq_bpi_dvmop, 0x02); CASE_DN_VER_EVT(dn_rxreq_pici_dvmop, 0x03); CASE_DN_VER_EVT(dn_rxreq_vivi_dvmop, 0x04); CASE_DN_VER_EVT(dn_rxreq_dvmsync, 0x05); CASE_DN_VER_EVT(dn_rxreq_dvmop_vmid_filtered, 0x06); CASE_DN_VER_EVT(dn_rxreq_dvmop_other_filtered, 0x07); CASE_DN_VER_EVT(dn_rxreq_retried, 0x08); CASE_DN_VER_EVT(dn_rxreq_snp_sent, 0x09); CASE_DN_VER_EVT(dn_rxreq_snp_stalled, 0x0a); CASE_DN_VER_EVT(dn_rxreq_trk_full, 0x0b); CASE_DN_VER_EVT(dn_rxreq_trk_occupancy, 0x0c); } } return (EINVAL); } else if (ev <= PMC_EV_CMN600_PMU_hnf_snp_fwded) { *node_type = NODE_TYPE_HN_F; *event = ev - PMC_EV_CMN600_PMU_hnf_cache_miss; return (0); } else if (ev <= PMC_EV_CMN600_PMU_hni_pcie_serialization) { *node_type = NODE_TYPE_HN_I; *event = ev - PMC_EV_CMN600_PMU_hni_rrt_rd_occ_cnt_ovfl; return (0); } else if (ev <= PMC_EV_CMN600_PMU_xp_partial_dat_flit) { *node_type = NODE_TYPE_XP; *event = ev - PMC_EV_CMN600_PMU_xp_txflit_valid; return (0); } else if (ev <= PMC_EV_CMN600_PMU_sbsx_txrsp_stall) { *node_type = NODE_TYPE_SBSX; *event = ev - PMC_EV_CMN600_PMU_sbsx_rd_req; return (0); } else if (ev <= PMC_EV_CMN600_PMU_rnd_rdb_ord) { *node_type = NODE_TYPE_RN_D; *event = ev - PMC_EV_CMN600_PMU_rnd_s0_rdata_beats; return (0); } else if (ev <= PMC_EV_CMN600_PMU_rni_rdb_ord) { *node_type = NODE_TYPE_RN_I; *event = ev - PMC_EV_CMN600_PMU_rni_s0_rdata_beats; return (0); } else if (ev <= PMC_EV_CMN600_PMU_cxha_snphaz_occ) { *node_type = NODE_TYPE_CXHA; *event = ev - PMC_EV_CMN600_PMU_cxha_rddatbyp; return (0); } else if (ev <= PMC_EV_CMN600_PMU_cxra_ext_dat_stall) { *node_type = NODE_TYPE_CXRA; *event = ev - PMC_EV_CMN600_PMU_cxra_req_trk_occ; return (0); } else if (ev <= PMC_EV_CMN600_PMU_cxla_avg_latency_form_tx_tlp) { *node_type = NODE_TYPE_CXLA; *event = ev - PMC_EV_CMN600_PMU_cxla_rx_tlp_link0; return (0); } return (EINVAL); } /* * Check if a given allocation is feasible. */ static int cmn600_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { struct cmn600_descr *desc; const struct pmc_descr *pd; uint64_t caps __unused; int local_counter, node_type; enum pmc_event pe; void *arg; uint8_t e; int err; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); desc = cmn600desc(ri); arg = desc->pd_rw_arg; pd = &desc->pd_descr; if (cmn600_pmcs[class_ri2unit(ri)].domain != pcpu_find(cpu)->pc_domain) return (EINVAL); /* check class match */ if (pd->pd_class != a->pm_class) return (EINVAL); caps = pm->pm_caps; PMCDBG3(MDP, ALL, 1, "%s ri=%d caps=0x%x", __func__, ri, caps); pe = a->pm_ev; err = cmn600_map_ev2event(pe, pmu_cmn600_rev(arg), &node_type, &e); if (err != 0) return (err); err = pmu_cmn600_alloc_localpmc(arg, a->pm_md.pm_cmn600.pma_cmn600_nodeid, node_type, &local_counter); if (err != 0) return (err); pm->pm_md.pm_cmn600.pm_cmn600_config = a->pm_md.pm_cmn600.pma_cmn600_config; pm->pm_md.pm_cmn600.pm_cmn600_occupancy = a->pm_md.pm_cmn600.pma_cmn600_occupancy; desc->pd_nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid = a->pm_md.pm_cmn600.pma_cmn600_nodeid; desc->pd_node_type = pm->pm_md.pm_cmn600.pm_cmn600_node_type = node_type; pm->pm_md.pm_cmn600.pm_cmn600_event = e; desc->pd_local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter = local_counter; return (0); } /* Release machine dependent state associated with a PMC. */ static int cmn600_release_pmc(int cpu, int ri, struct pmc *pmc) { struct cmn600_descr *desc; struct pmc_hw *phw; struct pmc *pm __diagused; int err; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); desc = cmn600desc(ri); phw = desc->pd_phw; pm = phw->phw_pmc; err = pmu_cmn600_free_localpmc(desc->pd_rw_arg, desc->pd_nodeid, desc->pd_node_type, desc->pd_local_counter); if (err != 0) return (err); KASSERT(pm == NULL, ("[cmn600,%d] PHW pmc %p non-NULL", __LINE__, pm)); return (0); } static inline uint64_t cmn600_encode_source(int node_type, int counter, int port, int sub) { /* Calculate pmevcnt0_input_sel based on list in Table 3-794. */ if (node_type == NODE_TYPE_XP) return (0x4 | counter); return (((port + 1) << 4) | (sub << 2) | counter); } /* * start a PMC. */ static int cmn600_start_pmc(int cpu, int ri) { int counter, local_counter, node_type, shift; uint64_t config, occupancy, source, xp_pmucfg; struct cmn600_descr *desc; struct pmc_hw *phw; struct pmc *pm; uint8_t event, port, sub; uint16_t nodeid; void *arg; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); counter = ri % CMN600_COUNTERS_N; desc = cmn600desc(ri); phw = desc->pd_phw; pm = phw->phw_pmc; arg = desc->pd_rw_arg; KASSERT(pm != NULL, ("[cmn600,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, cpu, ri)); PMCDBG3(MDP, STA, 1, "%s cpu=%d ri=%d", __func__, cpu, ri); config = pm->pm_md.pm_cmn600.pm_cmn600_config; occupancy = pm->pm_md.pm_cmn600.pm_cmn600_occupancy; node_type = pm->pm_md.pm_cmn600.pm_cmn600_node_type; event = pm->pm_md.pm_cmn600.pm_cmn600_event; nodeid = pm->pm_md.pm_cmn600.pm_cmn600_nodeid; local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; port = (nodeid >> 2) & 1; sub = nodeid & 3; switch (node_type) { case NODE_TYPE_DVM: case NODE_TYPE_HN_F: case NODE_TYPE_CXHA: case NODE_TYPE_CXRA: pmu_cmn600_md8(arg, nodeid, node_type, CMN600_COMMON_PMU_EVENT_SEL, CMN600_COMMON_PMU_EVENT_SEL_OCC_MASK, occupancy << CMN600_COMMON_PMU_EVENT_SEL_OCC_SHIFT); break; case NODE_TYPE_XP: /* Set PC and Interface.*/ event |= config; } /* * 5.5.1 Set up PMU counters * 1. Ensure that the NIDEN input is asserted. HW side. */ /* 2. Select event of target node for one of four outputs. */ pmu_cmn600_md8(arg, nodeid, node_type, CMN600_COMMON_PMU_EVENT_SEL, 0xff << (local_counter * 8), event << (local_counter * 8)); xp_pmucfg = pmu_cmn600_rd8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG); /* * 3. configure XP to connect one of four target node outputs to local * counter. */ source = cmn600_encode_source(node_type, local_counter, port, sub); shift = (local_counter * POR_DTM_PMU_CONFIG_VCNT_INPUT_SEL_WIDTH) + POR_DTM_PMU_CONFIG_VCNT_INPUT_SEL_SHIFT; xp_pmucfg &= ~(0xffUL << shift); xp_pmucfg |= source << shift; /* 4. Pair with global counters A, B, C, ..., H. */ shift = (local_counter * 4) + 16; xp_pmucfg &= ~(0xfUL << shift); xp_pmucfg |= counter << shift; /* Enable pairing.*/ xp_pmucfg |= 1 << (local_counter + 4); /* 5. Combine local counters 0 with 1, 2 with 3 or all four. */ xp_pmucfg &= ~0xeUL; /* 6. Enable XP's PMU function. */ xp_pmucfg |= POR_DTM_PMU_CONFIG_PMU_EN; pmu_cmn600_wr8(arg, nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG, xp_pmucfg); if (node_type == NODE_TYPE_CXLA) pmu_cmn600_set8(arg, nodeid, NODE_TYPE_CXLA, POR_CXG_RA_CFG_CTL, EN_CXLA_PMUCMD_PROP); /* 7. Enable DTM. */ pmu_cmn600_set8(arg, nodeid, NODE_TYPE_XP, POR_DTM_CONTROL, POR_DTM_CONTROL_DTM_ENABLE); /* 8. Reset grouping of global counters. Use 32 bits. */ pmu_cmn600_clr8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, POR_DT_PMCR_CNTCFG_MASK); /* 9. Enable DTC. */ pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_DTC_CTL, POR_DT_DTC_CTL_DT_EN); /* 10. Enable Overflow Interrupt. */ pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, POR_DT_PMCR_OVFL_INTR_EN); /* 11. Run PMC. */ pmu_cmn600_set8(arg, nodeid, NODE_TYPE_DTC, POR_DT_PMCR, POR_DT_PMCR_PMU_EN); return (0); } /* * Stop a PMC. */ static int cmn600_stop_pmc(int cpu, int ri) { struct cmn600_descr *desc; struct pmc_hw *phw; struct pmc *pm; int local_counter; uint64_t val; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); desc = cmn600desc(ri); phw = desc->pd_phw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[cmn600,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP, STO, 1, "%s ri=%d", __func__, ri); /* Disable pairing. */ local_counter = pm->pm_md.pm_cmn600.pm_cmn600_local_counter; pmu_cmn600_clr8(desc->pd_rw_arg, pm->pm_md.pm_cmn600.pm_cmn600_nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG, (1 << (local_counter + 4))); /* Shutdown XP's DTM function if no paired counters. */ val = pmu_cmn600_rd8(desc->pd_rw_arg, pm->pm_md.pm_cmn600.pm_cmn600_nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG); if ((val & 0xf0) == 0) pmu_cmn600_clr8(desc->pd_rw_arg, pm->pm_md.pm_cmn600.pm_cmn600_nodeid, NODE_TYPE_XP, POR_DTM_PMU_CONFIG, POR_DTM_CONTROL_DTM_ENABLE); return (0); } /* * describe a PMC */ static int cmn600_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { + struct pmc_descr *pd; struct pmc_hw *phw; - size_t copied; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[cmn600,%d] row-index %d out of range", __LINE__, ri)); phw = cmn600desc(ri)->pd_phw; + pd = &cmn600desc(ri)->pd_descr; - if ((error = copystr(cmn600desc(ri)->pd_descr.pd_name, - pi->pm_name, PMC_NAME_MAX, &copied)) != 0) - return (error); - - pi->pm_class = cmn600desc(ri)->pd_descr.pd_class; + strlcpy(pi->pm_name, pd->pd_name, sizeof(pi->pm_name)); + pi->pm_class = pd->pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } /* * processor dependent initialization. */ static int cmn600_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, n, npmc; struct pmc_hw *phw; struct pmc_cpu *pc; int mdep_class; mdep_class = PMC_MDEP_CLASS_INDEX_CMN600; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "cmn600-init cpu=%d", cpu); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[mdep_class].pcd_ri; npmc = md->pmd_classdep[mdep_class].pcd_num; for (n = 0; n < npmc; n++, phw++) { phw = cmn600desc(n)->pd_phw; phw->phw_state = PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); /* Set enabled only if unit present. */ if (cmn600_pmcs[class_ri2unit(n)].arg != NULL) phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * processor dependent cleanup prior to the KLD * being unloaded */ static int cmn600_pcpu_fini(struct pmc_mdep *md, int cpu) { return (0); } static int cmn600_pmu_intr(struct trapframe *tf, int unit, int i) { struct pmc_cpu *pc __diagused; struct pmc_hw *phw; struct pmc *pm; int error, cpu, ri; ri = i + unit * CMN600_COUNTERS_N; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[cmn600,%d] CPU %d out of range", __LINE__, cpu)); pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("pc != NULL")); phw = cmn600desc(ri)->pd_phw; KASSERT(phw != NULL, ("phw != NULL")); pm = phw->phw_pmc; if (pm == NULL) return (0); if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* Always CPU0. */ pm->pm_pcpu_state[0].pps_overflowcnt += 1; return (0); } if (pm->pm_state != PMC_STATE_RUNNING) return (0); error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) cmn600_stop_pmc(cpu, ri); /* Reload sampling count */ cmn600_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount); return (0); } /* * Initialize ourselves. */ static int cmn600_init_pmc_units(void) { int i; if (cmn600_units > 0) { /* Already initialized. */ return (0); } cmn600_units = cmn600_pmc_nunits(); if (cmn600_units == 0) return (ENOENT); for (i = 0; i < cmn600_units; i++) { if (cmn600_pmc_getunit(i, &cmn600_pmcs[i].arg, &cmn600_pmcs[i].domain) != 0) cmn600_pmcs[i].arg = NULL; } return (0); } int pmc_cmn600_nclasses(void) { if (cmn600_pmc_nunits() > 0) return (1); return (0); } int pmc_cmn600_initialize(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc, unit; cmn600_init_pmc_units(); KASSERT(md != NULL, ("[cmn600,%d] md is NULL", __LINE__)); KASSERT(cmn600_units < CMN600_UNIT_MAX, ("[cmn600,%d] cmn600_units too big", __LINE__)); PMCDBG0(MDP,INI,1, "cmn600-initialize"); npmc = CMN600_COUNTERS_N * cmn600_units; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600]; pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_QUALIFIER | PMC_CAP_INTERRUPT | PMC_CAP_DOMWIDE; pcd->pcd_class = PMC_CLASS_CMN600_PMU; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 48; pcd->pcd_allocate_pmc = cmn600_allocate_pmc; pcd->pcd_config_pmc = cmn600_config_pmc; pcd->pcd_describe = cmn600_describe; pcd->pcd_get_config = cmn600_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = cmn600_pcpu_fini; pcd->pcd_pcpu_init = cmn600_pcpu_init; pcd->pcd_read_pmc = cmn600_read_pmc; pcd->pcd_release_pmc = cmn600_release_pmc; pcd->pcd_start_pmc = cmn600_start_pmc; pcd->pcd_stop_pmc = cmn600_stop_pmc; pcd->pcd_write_pmc = cmn600_write_pmc; md->pmd_npmc += npmc; cmn600_pmcdesc = malloc(sizeof(struct cmn600_descr *) * npmc * CMN600_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); for (i = 0; i < npmc; i++) { cmn600_pmcdesc[i] = malloc(sizeof(struct cmn600_descr), M_PMC, M_WAITOK|M_ZERO); unit = i / CMN600_COUNTERS_N; KASSERT(unit >= 0, ("unit >= 0")); KASSERT(cmn600_pmcs[unit].arg != NULL, ("arg != NULL")); cmn600_pmcdesc[i]->pd_rw_arg = cmn600_pmcs[unit].arg; cmn600_pmcdesc[i]->pd_descr.pd_class = PMC_CLASS_CMN600_PMU; cmn600_pmcdesc[i]->pd_descr.pd_caps = pcd->pcd_caps; cmn600_pmcdesc[i]->pd_phw = (struct pmc_hw *)malloc( sizeof(struct pmc_hw), M_PMC, M_WAITOK|M_ZERO); snprintf(cmn600_pmcdesc[i]->pd_descr.pd_name, 63, "CMN600_%d", i); cmn600_pmu_intr_cb(cmn600_pmcs[unit].arg, cmn600_pmu_intr); } return (0); } void pmc_cmn600_finalize(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc; KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600].pcd_class == PMC_CLASS_CMN600_PMU, ("[cmn600,%d] pmc class mismatch", __LINE__)); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CMN600]; npmc = pcd->pcd_num; for (i = 0; i < npmc; i++) { free(cmn600_pmcdesc[i]->pd_phw, M_PMC); free(cmn600_pmcdesc[i], M_PMC); } free(cmn600_pmcdesc, M_PMC); cmn600_pmcdesc = NULL; } MODULE_DEPEND(pmc, cmn600, 1, 1, 1); diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index 5d29931c90f0..ad9d323bdb9a 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -1,1322 +1,1310 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Core PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #define CORE_CPUID_REQUEST 0xA #define CORE_CPUID_REQUEST_SIZE 0x4 #define CORE_CPUID_EAX 0x0 #define CORE_CPUID_EBX 0x1 #define CORE_CPUID_ECX 0x2 #define CORE_CPUID_EDX 0x3 #define IAF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30)) #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define EV_IS_NOTARCH 0 #define EV_IS_ARCH_SUPP 1 #define EV_IS_ARCH_NOTSUPP -1 /* * "Architectural" events defined by Intel. The values of these * symbols correspond to positions in the bitmask returned by * the CPUID.0AH instruction. */ enum core_arch_events { CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5, CORE_AE_BRANCH_MISSES_RETIRED = 6, CORE_AE_INSTRUCTION_RETIRED = 1, CORE_AE_LLC_MISSES = 4, CORE_AE_LLC_REFERENCE = 3, CORE_AE_UNHALTED_REFERENCE_CYCLES = 2, CORE_AE_UNHALTED_CORE_CYCLES = 0 }; static enum pmc_cputype core_cputype; static int core_version; struct core_cpu { volatile uint32_t pc_iafctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_corepmcs[]; }; static struct core_cpu **core_pcpu; static uint32_t core_architectural_events; static uint64_t core_pmcmask; static int core_iaf_ri; /* relative index of fixed counters */ static int core_iaf_width; static int core_iaf_npmc; static int core_iap_width; static int core_iap_npmc; static int core_iap_wroffset; static u_int pmc_alloc_refs; static bool pmc_tsx_force_abort_set; static int core_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int core_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct core_cpu *cc; struct pmc_hw *phw; int core_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; if (core_version >= 2) npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); core_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + core_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + core_ri] = phw; } if (core_version >= 2 && vm_guest == VM_GUEST_NO) { /* Enable Freezing PMCs on PMI. */ wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000); } return (0); } static int core_pcpu_fini(struct pmc_mdep *md, int cpu) { int core_ri, n, npmc; struct pmc_cpu *pc; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); if ((cc = core_pcpu[cpu]) == NULL) return (0); core_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(IAP_EVSEL0 + n, 0); if (core_version >= 2) { wrmsr(IAF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; } for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + core_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t iaf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iaf_width - 1))) == 0) return (0); v &= (1ULL << core_iaf_width) - 1; return (1ULL << core_iaf_width) - v; } static pmc_value_t iaf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iaf_width) - rlc; } static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev, umask; uint32_t caps; uint64_t config, flags; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > core_iaf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_IAF) return (EINVAL); iap = &a->pm_md.pm_iap; config = iap->pm_iap_config; ev = IAP_EVSEL_GET(config); umask = IAP_UMASK_GET(config); if (ev == 0x0) { if (umask != ri + 1) return (EINVAL); } else { switch (ri) { case 0: /* INST_RETIRED.ANY */ if (ev != 0xC0 || umask != 0x00) return (EINVAL); break; case 1: /* CPU_CLK_UNHALTED.THREAD */ if (ev != 0x3C || umask != 0x00) return (EINVAL); break; case 2: /* CPU_CLK_UNHALTED.REF */ if (ev != 0x3C || umask != 0x01) return (EINVAL); break; case 3: /* TOPDOWN.SLOTS */ if (ev != 0xA4 || umask != 0x01) return (EINVAL); break; default: return (EINVAL); } } pmc_alloc_refs++; if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 && !pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = true; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 1, NULL); } flags = 0; if (config & IAP_OS) flags |= IAF_OS; if (config & IAP_USR) flags |= IAF_USR; if (config & IAP_ANY) flags |= IAF_ANY; if (config & IAP_INT) flags |= IAF_PMI; caps = a->pm_caps; if (caps & PMC_CAP_INTERRUPT) flags |= IAF_PMI; if (caps & PMC_CAP_SYSTEM) flags |= IAF_OS; if (caps & PMC_CAP_USER) flags |= IAF_USR; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) flags |= (IAF_OS | IAF_USR); pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); return (0); } static int iaf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; return (0); } static int iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char iaf_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri]; - (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri); - if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri); pi->pm_class = PMC_CLASS_IAF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iaf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; return (0); } static int iaf_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[iaf,%d] ri %d out of range", __LINE__, ri)); *msr = IAF_RI_TO_MSR(ri); return (0); } static int iaf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + core_iaf_ri)); tmp = rdpmc(IAF_RI_TO_MSR(ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iaf_width) - 1); PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); return (0); } static int iaf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); MPASS(pmc_alloc_refs > 0); if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) { pmc_tsx_force_abort_set = false; x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL | MSR_OP_WRITE, 0, NULL); } return (0); } static int iaf_start_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; wrmsr(IAF_CTRL, cc->pc_iafctrl); cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_stop_pmc(int cpu, int ri) { struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri); cc = core_pcpu[cpu]; cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4)); wrmsr(IAF_CTRL, cc->pc_iafctrl); /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */ PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_write_pmc(int cpu, int ri, pmc_value_t v) { struct core_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iaf_reload_count_to_perfctr_value(v); /* Turn off the fixed counter */ wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4))); wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); /* Turn on fixed counters */ wrmsr(IAF_CTRL, cc->pc_iafctrl); PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); return (0); } static void iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iaf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; pcd->pcd_caps = IAF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iaf_allocate_pmc; pcd->pcd_config_pmc = iaf_config_pmc; pcd->pcd_describe = iaf_describe; pcd->pcd_get_config = iaf_get_config; pcd->pcd_get_msr = iaf_get_msr; pcd->pcd_pcpu_fini = core_pcpu_noop; pcd->pcd_pcpu_init = core_pcpu_noop; pcd->pcd_read_pmc = iaf_read_pmc; pcd->pcd_release_pmc = iaf_release_pmc; pcd->pcd_start_pmc = iaf_start_pmc; pcd->pcd_stop_pmc = iaf_stop_pmc; pcd->pcd_write_pmc = iaf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* Sub fields of UMASK that this event supports. */ #define IAP_M_CORE (1 << 0) /* Core specificity */ #define IAP_M_AGENT (1 << 1) /* Agent specificity */ #define IAP_M_PREFETCH (1 << 2) /* Prefetch */ #define IAP_M_MESI (1 << 3) /* MESI */ #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */ #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */ #define IAP_M_TRANSITION (1 << 6) /* Transition */ #define IAP_F_CORE (0x3 << 14) /* Core specificity */ #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */ #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */ #define IAP_F_MESI (0xF << 8) /* MESI */ #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */ #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */ #define IAP_F_TRANSITION (0x1 << 12) /* Transition */ #define IAP_PREFETCH_RESERVED (0x2 << 12) #define IAP_CORE_THIS (0x1 << 14) #define IAP_CORE_ALL (0x3 << 14) #define IAP_F_CMASK 0xFF000000 static pmc_value_t iap_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iap_width - 1))) == 0) return (0); v &= (1ULL << core_iap_width) - 1; return (1ULL << core_iap_width) - v; } static pmc_value_t iap_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iap_width) - rlc; } static int iap_pmc_has_overflowed(int ri) { uint64_t v; /* * We treat a Core (i.e., Intel architecture v1) PMC as has * having overflowed if its MSB is zero. */ v = rdpmc(ri); return ((v & (1ULL << (core_iap_width - 1))) == 0); } static int iap_event_corei7_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0, 1. */ case 0x40: case 0x41: case 0x42: case 0x43: case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x53: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_westmere_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0x60: case 0xB3: mask = 0x1; break; /* Events valid only on counter 0, 1. */ case 0x4C: case 0x4E: case 0x51: case 0x52: case 0x63: mask = 0x3; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* Events valid only on counter 0. */ case 0xB7: mask = 0x1; break; /* Events valid only on counter 1. */ case 0xC0: mask = 0x2; break; /* Events valid only on counter 2. */ case 0x48: case 0xA2: case 0xA3: mask = 0x4; break; /* Events valid only on counter 3. */ case 0xBB: case 0xCD: mask = 0x8; break; /* Any row index is ok. */ default: mask = ~0; } return (mask & (1 << ri)); } static int iap_event_core_ok_on_counter(uint8_t evsel, int ri) { uint32_t mask; switch (evsel) { /* * Events valid only on counter 0. */ case 0x10: case 0x14: case 0x18: case 0xB3: case 0xC1: case 0xCB: mask = (1 << 0); break; /* * Events valid only on counter 1. */ case 0x11: case 0x12: case 0x13: mask = (1 << 1); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_iap_op_pmcallocate *iap; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_IAP) return (EINVAL); iap = &a->pm_md.pm_iap; ev = IAP_EVSEL_GET(iap->pm_iap_config); switch (core_cputype) { case PMC_CPU_INTEL_CORE: case PMC_CPU_INTEL_CORE2: case PMC_CPU_INTEL_CORE2EXTREME: if (iap_event_core_ok_on_counter(ev, ri) == 0) return (EINVAL); case PMC_CPU_INTEL_COREI7: case PMC_CPU_INTEL_NEHALEM_EX: if (iap_event_corei7_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_WESTMERE: case PMC_CPU_INTEL_WESTMERE_EX: if (iap_event_westmere_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_SANDYBRIDGE: case PMC_CPU_INTEL_SANDYBRIDGE_XEON: case PMC_CPU_INTEL_IVYBRIDGE: case PMC_CPU_INTEL_IVYBRIDGE_XEON: case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_HASWELL_XEON: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_ATOM: case PMC_CPU_INTEL_ATOM_SILVERMONT: case PMC_CPU_INTEL_ATOM_GOLDMONT: case PMC_CPU_INTEL_ATOM_GOLDMONT_P: case PMC_CPU_INTEL_ATOM_TREMONT: case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_ICELAKE: case PMC_CPU_INTEL_ICELAKE_XEON: case PMC_CPU_INTEL_ALDERLAKE: default: break; } pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config; return (0); } static int iap_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; return (0); } static int iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char iap_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri]; - (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri); - if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri); pi->pm_class = PMC_CLASS_IAP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iap_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; return (0); } static int iap_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iap_npmc, ("[iap,%d] ri %d out of range", __LINE__, ri)); *msr = ri; return (0); } static int iap_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdpmc(ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iap_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iap_width) - 1); PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAP_PMC0 + ri, *v); return (0); } static int iap_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iap_start_pmc(int cpu, int ri) { struct pmc *pm; uint64_t evsel; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_iap.pm_iap_evsel; PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, IAP_EVSEL0 + ri, evsel); /* Event specific configuration. */ switch (IAP_EVSEL_GET(evsel)) { case 0xB7: wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); break; case 0xBB: wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp); break; default: break; } wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN); if (core_version >= 2) { cc->pc_globalctrl |= (1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } return (0); } static int iap_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); wrmsr(IAP_EVSEL0 + ri, 0); /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */ return (0); } static int iap_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iap_reload_count_to_perfctr_value(v); v &= (1ULL << core_iap_width) - 1; PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, IAP_PMC0 + ri, v); /* * Write the new value to the counter (or it's alias). The * counter will be in a stopped state when the pcd_write() * entry point is called. */ wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); return (0); } static void iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, int flags) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iap-initialize"); /* Remember the set of architectural events supported. */ core_architectural_events = ~flags; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; pcd->pcd_caps = IAP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iap_allocate_pmc; pcd->pcd_config_pmc = iap_config_pmc; pcd->pcd_describe = iap_describe; pcd->pcd_get_config = iap_get_config; pcd->pcd_get_msr = iap_get_msr; pcd->pcd_pcpu_fini = core_pcpu_fini; pcd->pcd_pcpu_init = core_pcpu_init; pcd->pcd_read_pmc = iap_read_pmc; pcd->pcd_release_pmc = iap_release_pmc; pcd->pcd_start_pmc = iap_start_pmc; pcd->pcd_stop_pmc = iap_stop_pmc; pcd->pcd_write_pmc = iap_write_pmc; md->pmd_npmc += npmc; } static int core_intr(struct trapframe *tf) { pmc_value_t v; struct pmc *pm; struct core_cpu *cc; int error, found_interrupt, ri; PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf, TRAPF_USERMODE(tf)); found_interrupt = 0; cc = core_pcpu[curcpu]; for (ri = 0; ri < core_iap_npmc; ri++) { if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (!iap_pmc_has_overflowed(ri)) continue; found_interrupt = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); v = pm->pm_sc.pm_reloadcount; v = iap_reload_count_to_perfctr_value(v); /* * Stop the counter, reload it but only restart it if * the PMC is not stalled. */ wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel); wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); if (__predict_false(error)) continue; wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); if (found_interrupt) lapic_reenable_pmc(); return (found_interrupt); } static int core2_intr(struct trapframe *tf) { int error, found_interrupt = 0, n, cpu; uint64_t flag, intrstatus, intrdisable = 0; struct pmc *pm; struct core_cpu *cc; pmc_value_t v; cpu = curcpu; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); /* * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which * PMCs have a pending PMI interrupt. We take a 'snapshot' of * the current set of interrupting PMCs and process these * after stopping them. */ intrstatus = rdmsr(IA_GLOBAL_STATUS); PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, (uintmax_t) intrstatus); /* * Stop PMCs unless hardware already done it. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) wrmsr(IA_GLOBAL_CTRL, 0); cc = core_pcpu[cpu]; KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); /* * Look for interrupts from fixed function PMCs. */ for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); /* Reload sampling count. */ wrmsr(IAF_CTR0 + n, v); PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu, error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); } /* * Process interrupts from the programmable counters. */ for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(PMC_HR, pm, tf); if (__predict_false(error)) intrdisable |= flag; v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, (uintmax_t) v); /* Reload sampling count. */ wrmsr(core_iap_wroffset + IAP_PMC0 + n, v); } if (found_interrupt) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); if (found_interrupt) lapic_reenable_pmc(); /* * Reenable all non-stalled PMCs. */ if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) { wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } else { if (__predict_false(intrdisable)) { cc->pc_globalctrl &= ~intrdisable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus); } PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_STATUS)); return (found_interrupt); } int pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) { int cpuid[CORE_CPUID_REQUEST_SIZE]; int flags, nflags; do_cpuid(CORE_CPUID_REQUEST, cpuid); core_cputype = md->pmd_cputype; core_version = (version_override > 0) ? version_override : cpuid[CORE_CPUID_EAX] & 0xFF; PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d", core_cputype, maxcpu, core_version); if (core_version < 1 || core_version > 5 || (core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) { /* Unknown PMC architecture. */ printf("hwpmc_core: unknown PMC architecture: %d\n", core_version); return (EPROGMISMATCH); } core_iap_wroffset = 0; if (cpu_feature2 & CPUID2_PDCM) { if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) { PMCDBG0(MDP, INI, 1, "core-init full-width write supported"); core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0; } else PMCDBG0(MDP, INI, 1, "core-init full-width write NOT supported"); } else PMCDBG0(MDP, INI, 1, "core-init pdcm not supported"); core_pmcmask = 0; /* * Initialize programmable counters. */ core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF; core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF; core_pmcmask |= ((1ULL << core_iap_npmc) - 1); nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF; flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1); iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags); /* * Initialize fixed function counters, if present. */ if (core_version >= 2) { core_iaf_ri = core_iap_npmc; core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; } PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, core_iaf_ri); core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); /* * Choose the appropriate interrupt handler. */ if (core_version >= 2) md->pmd_intr = core2_intr; else md->pmd_intr = core_intr; md->pmd_pcpu_fini = NULL; md->pmd_pcpu_init = NULL; return (0); } void pmc_core_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "core-finalize"); free(core_pcpu, M_PMC); core_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_dmc620.c b/sys/dev/hwpmc/hwpmc_dmc620.c index a6a1dfb44ca4..6152e264a169 100644 --- a/sys/dev/hwpmc/hwpmc_dmc620.c +++ b/sys/dev/hwpmc/hwpmc_dmc620.c @@ -1,741 +1,738 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * Copyright (c) 2021 Ampere Computing LLC * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Support for ARM DMC-620 Memory Controller PMU */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #define DMC620_TYPE_CLKDIV2 0 #define DMC620_TYPE_CLK 1 #define CLASS2TYPE(c) ((c) - PMC_CLASS_DMC620_PMU_CD2) /* Create wrapper for each class. */ #define CLASSDEP_FN2(fn, t1, a1, t2, a2) \ static int fn(int class, t1 a1, t2 a2); \ static int fn ## _cd2(t1 a1, t2 a2) \ { \ return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2)); \ } \ static int fn ## _c(t1 a1, t2 a2) \ { \ return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2)); \ } \ static int fn(int class, t1 a1, t2 a2) #define CLASSDEP_FN3(fn, t1, a1, t2, a2, t3, a3) \ static int fn(int class, t1 a1, t2 a2, t3 a3); \ static int fn ## _cd2(t1 a1, t2 a2, t3 a3) \ { \ return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2, a3)); \ } \ static int fn ## _c(t1 a1, t2 a2, t3 a3) \ { \ return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2, a3)); \ } \ static int fn(int class, t1 a1, t2 a2, t3 a3) #define CLASSDEP_FN4(fn, t1, a1, t2, a2, t3, a3, t4, a4) \ static int fn(int class, t1 a1, t2 a2, t3 a3, t4 a4); \ static int fn ## _cd2(t1 a1, t2 a2, t3 a3, t4 a4) \ { \ return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2, a3, a4)); \ } \ static int fn ## _c(t1 a1, t2 a2, t3 a3, t4 a4) \ { \ return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2, a3, a4)); \ } \ static int fn(int class, t1 a1, t2 a2, t3 a3, t4 a4) struct dmc620_pmc { void *arg; int domain; }; struct dmc620_descr { struct pmc_descr pd_descr; /* "base class" */ void *pd_rw_arg; /* Argument to use with read/write */ struct pmc *pd_pmc; struct pmc_hw *pd_phw; uint32_t pd_config; uint32_t pd_match; uint32_t pd_mask; uint32_t pd_evsel; /* address of EVSEL register */ uint32_t pd_perfctr; /* address of PERFCTR register */ }; static struct dmc620_descr **dmc620_pmcdesc[2]; static struct dmc620_pmc dmc620_pmcs[DMC620_UNIT_MAX]; static int dmc620_npmcs = 0; void dmc620_pmc_register(int unit, void *arg, int domain) { if (unit >= DMC620_UNIT_MAX) { /* TODO */ return; } dmc620_pmcs[unit].arg = arg; dmc620_pmcs[unit].domain = domain; dmc620_npmcs++; } void dmc620_pmc_unregister(int unit) { dmc620_pmcs[unit].arg = NULL; dmc620_npmcs--; } int pmc_dmc620_nclasses(void) { if (dmc620_npmcs > 0) return (2); return (0); } static inline struct dmc620_descr * dmc620desc(int class, int cpu, int ri) { int c; c = CLASS2TYPE(class); KASSERT((c & 0xfffffffe) == 0, ("[dmc620,%d] 'c' can only be 0 or 1. " "now %d", __LINE__, c)); return (dmc620_pmcdesc[c][ri]); } static inline int cntr(int class, int ri) { int c; c = CLASS2TYPE(class); KASSERT((c & 0xfffffffe) == 0, ("[dmc620,%d] 'c' can only be 0 or 1. " "now %d", __LINE__, c)); if (c == DMC620_TYPE_CLKDIV2) return (ri % DMC620_CLKDIV2_COUNTERS_N); return ((ri % DMC620_CLK_COUNTERS_N) + DMC620_CLKDIV2_COUNTERS_N); } static inline int class2mdep(int class) { switch (class) { case PMC_CLASS_DMC620_PMU_CD2: return (PMC_MDEP_CLASS_INDEX_DMC620_CD2); case PMC_CLASS_DMC620_PMU_C: return (PMC_MDEP_CLASS_INDEX_DMC620_C); } return (-1); } static inline int class_ri2unit(int class, int ri) { if (class == PMC_CLASS_DMC620_PMU_CD2) return (ri / DMC620_CLKDIV2_COUNTERS_N); else return (ri / DMC620_CLK_COUNTERS_N); } /* * read a pmc register */ CLASSDEP_FN3(dmc620_read_pmc, int, cpu, int, ri, pmc_value_t *, v) { struct dmc620_descr *desc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); desc = dmc620desc(class, cpu, ri); pm = desc->pd_phw->phw_pmc; KASSERT(pm != NULL, ("[dmc620,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); PMCDBG3(MDP,REA,1,"%s id=%d class=%d", __func__, ri, class); /* * Should emulate 64bits, because 32 bits counter overflows faster than * pmcstat default period. */ /* Always CPU0. Single controller for all CPUs. */ *v = ((uint64_t)pm->pm_pcpu_state[0].pps_overflowcnt << 32) | pmu_dmc620_rd4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_VALUE_LO); PMCDBG3(MDP, REA, 2, "%s id=%d -> %jd", __func__, ri, *v); return (0); } /* * Write a pmc register. */ CLASSDEP_FN3(dmc620_write_pmc, int, cpu, int, ri, pmc_value_t, v) { struct dmc620_descr *desc; struct pmc *pm __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); desc = dmc620desc(class, cpu, ri); pm = desc->pd_phw->phw_pmc; KASSERT(pm != NULL, ("[dmc620,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, cpu, ri)); PMCDBG4(MDP, WRI, 1, "%s cpu=%d ri=%d v=%jx", __func__, cpu, ri, v); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_VALUE_LO, v); return (0); } /* * configure hardware pmc according to the configuration recorded in * pmc 'pm'. */ CLASSDEP_FN3(dmc620_config_pmc, int, cpu, int, ri, struct pmc *, pm) { struct pmc_hw *phw; PMCDBG4(MDP, CFG, 1, "%s cpu=%d ri=%d pm=%p", __func__, cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); phw = dmc620desc(class, cpu, ri)->pd_phw; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[dmc620,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } /* * Retrieve a configured PMC pointer from hardware state. */ CLASSDEP_FN3(dmc620_get_config, int, cpu, int, ri, struct pmc **, ppm) { *ppm = dmc620desc(class, cpu, ri)->pd_phw->phw_pmc; return (0); } /* * Check if a given allocation is feasible. */ CLASSDEP_FN4(dmc620_allocate_pmc, int, cpu, int, ri, struct pmc *,pm, const struct pmc_op_pmcallocate *, a) { const struct pmc_descr *pd; uint64_t caps, control; enum pmc_event pe; uint8_t e; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); pd = &dmc620desc(class, cpu, ri)->pd_descr; if (dmc620_pmcs[class_ri2unit(class, ri)].domain != pcpu_find(cpu)->pc_domain) return (EINVAL); /* check class match */ if (pd->pd_class != a->pm_class) return (EINVAL); caps = pm->pm_caps; PMCDBG3(MDP, ALL, 1, "%s ri=%d caps=0x%x", __func__, ri, caps); pe = a->pm_ev; if (class == PMC_CLASS_DMC620_PMU_CD2) e = pe - PMC_EV_DMC620_PMU_CD2_FIRST; else e = pe - PMC_EV_DMC620_PMU_C_FIRST; control = (e << DMC620_COUNTER_CONTROL_EVENT_SHIFT) & DMC620_COUNTER_CONTROL_EVENT_MASK; if (caps & PMC_CAP_INVERT) control |= DMC620_COUNTER_CONTROL_INVERT; pm->pm_md.pm_dmc620.pm_control = control; pm->pm_md.pm_dmc620.pm_match = a->pm_md.pm_dmc620.pm_dmc620_match; pm->pm_md.pm_dmc620.pm_mask = a->pm_md.pm_dmc620.pm_dmc620_mask; PMCDBG3(MDP, ALL, 2, "%s ri=%d -> control=0x%x", __func__, ri, control); return (0); } /* * Release machine dependent state associated with a PMC. This is a * no-op on this architecture. * */ /* ARGSUSED0 */ CLASSDEP_FN3(dmc620_release_pmc, int, cpu, int, ri, struct pmc *, pmc) { struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); phw = dmc620desc(class, cpu, ri)->pd_phw; KASSERT(phw->phw_pmc == NULL, ("[dmc620,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } /* * start a PMC. */ CLASSDEP_FN2(dmc620_start_pmc, int, cpu, int, ri) { struct dmc620_descr *desc; struct pmc_hw *phw; uint64_t control; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); desc = dmc620desc(class, cpu, ri); phw = desc->pd_phw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[dmc620,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, cpu, ri)); PMCDBG3(MDP, STA, 1, "%s cpu=%d ri=%d", __func__, cpu, ri); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_MASK_LO, pm->pm_md.pm_dmc620.pm_mask & 0xffffffff); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_MASK_HI, pm->pm_md.pm_dmc620.pm_mask >> 32); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_MATCH_LO, pm->pm_md.pm_dmc620.pm_match & 0xffffffff); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_MATCH_HI, pm->pm_md.pm_dmc620.pm_match >> 32); /* turn on the PMC ENABLE bit */ control = pm->pm_md.pm_dmc620.pm_control | DMC620_COUNTER_CONTROL_ENABLE; PMCDBG2(MDP, STA, 2, "%s control=0x%x", __func__, control); pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_CONTROL, control); return (0); } /* * Stop a PMC. */ CLASSDEP_FN2(dmc620_stop_pmc, int, cpu, int, ri) { struct dmc620_descr *desc; struct pmc_hw *phw; struct pmc *pm; uint64_t control; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); desc = dmc620desc(class, cpu, ri); phw = desc->pd_phw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[dmc620,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP, STO, 1, "%s ri=%d", __func__, ri); /* turn off the PMC ENABLE bit */ control = pm->pm_md.pm_dmc620.pm_control & ~DMC620_COUNTER_CONTROL_ENABLE; pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), DMC620_COUNTER_CONTROL, control); return (0); } /* * describe a PMC */ CLASSDEP_FN4(dmc620_describe, int, cpu, int, ri, struct pmc_info *, pi, struct pmc **, ppmc) { + struct pmc_descr *pd; struct pmc_hw *phw; - size_t copied; - int error; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, ri)); phw = dmc620desc(class, cpu, ri)->pd_phw; + pd = &dmc620desc(class, cpu, ri)->pd_descr; - if ((error = copystr(dmc620desc(class, cpu, ri)->pd_descr.pd_name, - pi->pm_name, PMC_NAME_MAX, &copied)) != 0) - return (error); - - pi->pm_class = dmc620desc(class, cpu, ri)->pd_descr.pd_class; + strlcpy(pi->pm_name, pd->pd_name, sizeof(pi->pm_name)); + pi->pm_class = pd->pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } /* * processor dependent initialization. */ CLASSDEP_FN2(dmc620_pcpu_init, struct pmc_mdep *, md, int, cpu) { int first_ri, n, npmc; struct pmc_hw *phw; struct pmc_cpu *pc; int mdep_class; mdep_class = class2mdep(class); KASSERT(mdep_class != -1, ("[dmc620,%d] wrong class %d", __LINE__, class)); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP, INI, 1, "dmc620-init cpu=%d", cpu); /* * Set the content of the hardware descriptors to a known * state and initialize pointers in the MI per-cpu descriptor. */ pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[mdep_class].pcd_ri; npmc = md->pmd_classdep[mdep_class].pcd_num; for (n = 0; n < npmc; n++, phw++) { phw = dmc620desc(class, cpu, n)->pd_phw; phw->phw_state = PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); /* Set enabled only if unit present. */ if (dmc620_pmcs[class_ri2unit(class, n)].arg != NULL) phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } /* * processor dependent cleanup prior to the KLD * being unloaded */ CLASSDEP_FN2(dmc620_pcpu_fini, struct pmc_mdep *, md, int, cpu) { return (0); } int dmc620_intr(struct trapframe *tf, int class, int unit, int i) { struct pmc_cpu *pc __diagused; struct pmc_hw *phw; struct pmc *pm; int error, cpu, ri; ri = i + unit * ((class == PMC_CLASS_DMC620_PMU_CD2) ? DMC620_CLKDIV2_COUNTERS_N : DMC620_CLK_COUNTERS_N); cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[dmc620,%d] CPU %d out of range", __LINE__, cpu)); pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("pc != NULL")); phw = dmc620desc(class, cpu, ri)->pd_phw; KASSERT(phw != NULL, ("phw != NULL")); pm = phw->phw_pmc; if (pm == NULL) return (0); if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* Always CPU0. */ pm->pm_pcpu_state[0].pps_overflowcnt += 1; return (0); } if (pm->pm_state != PMC_STATE_RUNNING) return (0); error = pmc_process_interrupt(PMC_HR, pm, tf); if (error) dmc620_stop_pmc(class, cpu, ri); /* Reload sampling count */ dmc620_write_pmc(class, cpu, ri, pm->pm_sc.pm_reloadcount); return (0); } /* * Initialize ourselves. */ int pmc_dmc620_initialize_cd2(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc, unit; KASSERT(md != NULL, ("[dmc620,%d] md is NULL", __LINE__)); KASSERT(dmc620_npmcs <= DMC620_UNIT_MAX, ("[dmc620,%d] dmc620_npmcs too big", __LINE__)); PMCDBG0(MDP,INI,1, "dmc620-initialize"); npmc = DMC620_CLKDIV2_COUNTERS_N * dmc620_npmcs; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2]; pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_INTERRUPT | PMC_CAP_DOMWIDE; pcd->pcd_class = PMC_CLASS_DMC620_PMU_CD2; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = dmc620_allocate_pmc_cd2; pcd->pcd_config_pmc = dmc620_config_pmc_cd2; pcd->pcd_describe = dmc620_describe_cd2; pcd->pcd_get_config = dmc620_get_config_cd2; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = dmc620_pcpu_fini_cd2; pcd->pcd_pcpu_init = dmc620_pcpu_init_cd2; pcd->pcd_read_pmc = dmc620_read_pmc_cd2; pcd->pcd_release_pmc = dmc620_release_pmc_cd2; pcd->pcd_start_pmc = dmc620_start_pmc_cd2; pcd->pcd_stop_pmc = dmc620_stop_pmc_cd2; pcd->pcd_write_pmc = dmc620_write_pmc_cd2; md->pmd_npmc += npmc; dmc620_pmcdesc[0] = malloc(sizeof(struct dmc620_descr *) * npmc * DMC620_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); for (i = 0; i < npmc; i++) { dmc620_pmcdesc[0][i] = malloc(sizeof(struct dmc620_descr), M_PMC, M_WAITOK|M_ZERO); unit = i / DMC620_CLKDIV2_COUNTERS_N; KASSERT(unit >= 0, ("unit >= 0")); KASSERT(dmc620_pmcs[unit].arg != NULL, ("arg != NULL")); dmc620_pmcdesc[0][i]->pd_rw_arg = dmc620_pmcs[unit].arg; dmc620_pmcdesc[0][i]->pd_descr.pd_class = PMC_CLASS_DMC620_PMU_CD2; dmc620_pmcdesc[0][i]->pd_descr.pd_caps = pcd->pcd_caps; dmc620_pmcdesc[0][i]->pd_phw = malloc(sizeof(struct pmc_hw), M_PMC, M_WAITOK|M_ZERO); snprintf(dmc620_pmcdesc[0][i]->pd_descr.pd_name, 63, "DMC620_CD2_%d", i); } return (0); } int pmc_dmc620_initialize_c(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc, unit; KASSERT(md != NULL, ("[dmc620,%d] md is NULL", __LINE__)); KASSERT(dmc620_npmcs <= DMC620_UNIT_MAX, ("[dmc620,%d] dmc620_npmcs too big", __LINE__)); PMCDBG0(MDP,INI,1, "dmc620-initialize"); npmc = DMC620_CLK_COUNTERS_N * dmc620_npmcs; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C]; pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_INTERRUPT | PMC_CAP_DOMWIDE; pcd->pcd_class = PMC_CLASS_DMC620_PMU_C; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = dmc620_allocate_pmc_c; pcd->pcd_config_pmc = dmc620_config_pmc_c; pcd->pcd_describe = dmc620_describe_c; pcd->pcd_get_config = dmc620_get_config_c; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = dmc620_pcpu_fini_c; pcd->pcd_pcpu_init = dmc620_pcpu_init_c; pcd->pcd_read_pmc = dmc620_read_pmc_c; pcd->pcd_release_pmc = dmc620_release_pmc_c; pcd->pcd_start_pmc = dmc620_start_pmc_c; pcd->pcd_stop_pmc = dmc620_stop_pmc_c; pcd->pcd_write_pmc = dmc620_write_pmc_c; md->pmd_npmc += npmc; dmc620_pmcdesc[1] = malloc(sizeof(struct dmc620_descr *) * npmc * DMC620_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); for (i = 0; i < npmc; i++) { dmc620_pmcdesc[1][i] = malloc(sizeof(struct dmc620_descr), M_PMC, M_WAITOK|M_ZERO); unit = i / DMC620_CLK_COUNTERS_N; KASSERT(unit >= 0, ("unit >= 0")); KASSERT(dmc620_pmcs[unit].arg != NULL, ("arg != NULL")); dmc620_pmcdesc[1][i]->pd_rw_arg = dmc620_pmcs[unit].arg; dmc620_pmcdesc[1][i]->pd_descr.pd_class = PMC_CLASS_DMC620_PMU_C; dmc620_pmcdesc[1][i]->pd_descr.pd_caps = pcd->pcd_caps; dmc620_pmcdesc[1][i]->pd_phw = malloc(sizeof(struct pmc_hw), M_PMC, M_WAITOK|M_ZERO); snprintf(dmc620_pmcdesc[1][i]->pd_descr.pd_name, 63, "DMC620_C_%d", i); } return (0); } void pmc_dmc620_finalize_cd2(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc; KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2].pcd_class == PMC_CLASS_DMC620_PMU_CD2, ("[dmc620,%d] pmc class mismatch", __LINE__)); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2]; npmc = pcd->pcd_num; for (i = 0; i < npmc; i++) { free(dmc620_pmcdesc[0][i]->pd_phw, M_PMC); free(dmc620_pmcdesc[0][i], M_PMC); } free(dmc620_pmcdesc[0], M_PMC); dmc620_pmcdesc[0] = NULL; } void pmc_dmc620_finalize_c(struct pmc_mdep *md) { struct pmc_classdep *pcd; int i, npmc; KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C].pcd_class == PMC_CLASS_DMC620_PMU_C, ("[dmc620,%d] pmc class mismatch", __LINE__)); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C]; npmc = pcd->pcd_num; for (i = 0; i < npmc; i++) { free(dmc620_pmcdesc[1][i]->pd_phw, M_PMC); free(dmc620_pmcdesc[1][i], M_PMC); } free(dmc620_pmcdesc[1], M_PMC); dmc620_pmcdesc[1] = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_powerpc.c b/sys/dev/hwpmc/hwpmc_powerpc.c index aeb7285ead2c..3d65d8753fcb 100644 --- a/sys/dev/hwpmc/hwpmc_powerpc.c +++ b/sys/dev/hwpmc/hwpmc_powerpc.c @@ -1,653 +1,650 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011,2013 Justin Hibbits * Copyright (c) 2005, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "hwpmc_powerpc.h" #ifdef __powerpc64__ #define OFFSET 4 /* Account for the TOC reload slot */ #else #define OFFSET 0 #endif struct powerpc_cpu **powerpc_pcpu; struct pmc_ppc_event *ppc_event_codes; size_t ppc_event_codes_size; int ppc_event_first; int ppc_event_last; int ppc_max_pmcs; enum pmc_class ppc_class; void (*powerpc_set_pmc)(int cpu, int ri, int config); pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc); void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val); void (*powerpc_resume_pmc)(bool ie); int pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t *osp, *sp; uintptr_t pc; int frames = 0; cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); osp = (uintptr_t *)PAGE_SIZE; for (; frames < maxsamples; frames++) { if (sp <= osp) break; #ifdef __powerpc64__ pc = sp[2]; #else pc = sp[1]; #endif if ((pc & 3) || (pc < 0x100)) break; /* * trapexit() and asttrapexit() are sentinels * for kernel stack tracing. * */ if (pc + OFFSET == (uintptr_t) &trapexit || pc + OFFSET == (uintptr_t) &asttrapexit) break; cc[frames] = pc; osp = sp; sp = (uintptr_t *)*sp; } return (frames); } static int powerpc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } static int powerpc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return (0); } int powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char powerpc_name[PMC_NAME_MAX]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d], illegal CPU %d", __LINE__, cpu)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; - snprintf(powerpc_name, sizeof(powerpc_name), "POWERPC-%d", ri); - if ((error = copystr(powerpc_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return error; + + snprintf(pi->pm_name, sizeof(pi->pm_name), "POWERPC-%d", ri); pi->pm_class = powerpc_pcpu[cpu]->pc_class; + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } int powerpc_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; return (0); } int powerpc_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct powerpc_cpu *pac; struct pmc_hw *phw; int first_ri, i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu) + ppc_max_pmcs * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); pac->pc_class = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class; pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } return (0); } int powerpc_pcpu_fini(struct pmc_mdep *md, int cpu) { PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu); free(powerpc_pcpu[cpu], M_PMC); powerpc_pcpu[cpu] = NULL; return (0); } int powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t caps, config = 0, counter = 0; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != ppc_class) return (EINVAL); caps = a->pm_caps; pe = a->pm_ev; if (pe < ppc_event_first || pe > ppc_event_last) return (EINVAL); for (i = 0; i < ppc_event_codes_size; i++) { if (ppc_event_codes[i].pe_event == pe) { config = ppc_event_codes[i].pe_code; counter = ppc_event_codes[i].pe_flags; break; } } if (i == ppc_event_codes_size) return (EINVAL); if ((counter & (1 << ri)) == 0) return (EINVAL); if (caps & PMC_CAP_SYSTEM) config |= POWERPC_PMC_KERNEL_ENABLE; if (caps & PMC_CAP_USER) config |= POWERPC_PMC_USER_ENABLE; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) config |= POWERPC_PMC_ENABLE; pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x", cpu, ri, config); return (0); } int powerpc_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return (0); } int powerpc_start_pmc(int cpu, int ri) { struct pmc *pm; PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel); return (0); } int powerpc_stop_pmc(int cpu, int ri) { PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri); powerpc_set_pmc(cpu, ri, PMCN_NONE); return (0); } int powerpc_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } pmc_value_t powerpc_pmcn_read_default(unsigned int pmc) { pmc_value_t val; if (pmc > ppc_max_pmcs) panic("Invalid PMC number: %d\n", pmc); switch (pmc) { case 0: val = mfspr(SPR_PMC1); break; case 1: val = mfspr(SPR_PMC2); break; case 2: val = mfspr(SPR_PMC3); break; case 3: val = mfspr(SPR_PMC4); break; case 4: val = mfspr(SPR_PMC5); break; case 5: val = mfspr(SPR_PMC6); break; case 6: val = mfspr(SPR_PMC7); break; case 7: val = mfspr(SPR_PMC8); break; } return (val); } void powerpc_pmcn_write_default(unsigned int pmc, uint32_t val) { if (pmc > ppc_max_pmcs) panic("Invalid PMC number: %d\n", pmc); switch (pmc) { case 0: mtspr(SPR_PMC1, val); break; case 1: mtspr(SPR_PMC2, val); break; case 2: mtspr(SPR_PMC3, val); break; case 3: mtspr(SPR_PMC4, val); break; case 4: mtspr(SPR_PMC5, val); break; case 5: mtspr(SPR_PMC6, val); break; case 6: mtspr(SPR_PMC7, val); break; case 7: mtspr(SPR_PMC8, val); break; } } int powerpc_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t p, r, tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); /* * After an interrupt occurs because of a PMC overflow, the PMC value * is not always MAX_PMC_VALUE + 1, but may be a little above it. * This may mess up calculations and frustrate machine independent * layer expectations, such as that no value read should be greater * than reload count in sampling mode. * To avoid these issues, use MAX_PMC_VALUE as an upper limit. */ p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE); r = pm->pm_sc.pm_reloadcount; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { /* * Special case 1: r is too big * This usually happens when a PMC write fails, the PMC is * stopped and then it is read. * * Special case 2: PMC was reseted or has a value * that should not be possible with current r. * * In the above cases, just return 0 instead of an arbitrary * value. */ if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE) tmp = 0; else tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p); } else tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm); PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)", cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm), (uintmax_t)p); *v = tmp; return (0); } int powerpc_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; pmc_value_t vlo; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < ppc_max_pmcs, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1); vlo = v % (POWERPC_MAX_PMC_VALUE + 1); } else if (v > POWERPC_MAX_PMC_VALUE) { PMCDBG3(MDP,WRI,2, "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx", cpu, ri, (uintmax_t)v); return (EINVAL); } else vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)", cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm), (uintmax_t)vlo); powerpc_pmcn_write(ri, vlo); return (0); } int powerpc_pmc_intr(struct trapframe *tf) { struct pmc *pm; struct powerpc_cpu *pc; int cpu, error, i, retval; cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pc = powerpc_pcpu[cpu]; /* * Look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association. */ for (i = 0; i < ppc_max_pmcs; i++) { if (!POWERPC_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ /* * Always clear the PMC, to make it stop interrupting. * If pm is available and in sampling mode, use reload * count, to make PMC read after stop correct. * Otherwise, just reset the PMC. */ if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { if (pm->pm_state != PMC_STATE_RUNNING) { powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); continue; } } else { if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */ PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) + 1) % PPC_OVERFLOWCNT_MAX; PMCDBG3(MDP,INT,2, "cpu=%d ri=%d: overflowcnt=%d", cpu, i, PPC_OVERFLOWCNT(pm)); } powerpc_pmcn_write(i, 0); continue; } error = pmc_process_interrupt(PMC_HR, pm, tf); if (error != 0) { PMCDBG3(MDP,INT,3, "cpu=%d ri=%d: error %d processing interrupt", cpu, i, error); powerpc_stop_pmc(cpu, i); } /* Reload sampling count */ powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); } if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); /* * Re-enable PERF exceptions if we were able to find the interrupt * source and handle it. Otherwise, it's better to disable PERF * interrupts, to avoid the risk of processing the same interrupt * forever. */ powerpc_resume_pmc(retval != 0); if (retval == 0) log(LOG_WARNING, "pmc_intr: couldn't find interrupting PMC on cpu %d - " "disabling PERF interrupts\n", cpu); return (retval); } struct pmc_mdep * pmc_md_initialize(void) { struct pmc_mdep *pmc_mdep; int error; uint16_t vers; /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); vers = mfpvr() >> 16; pmc_mdep->pmd_switch_in = powerpc_switch_in; pmc_mdep->pmd_switch_out = powerpc_switch_out; switch (vers) { case MPC7447A: case MPC7448: case MPC7450: case MPC7455: case MPC7457: error = pmc_mpc7xxx_initialize(pmc_mdep); break; case IBM970: case IBM970FX: case IBM970MP: error = pmc_ppc970_initialize(pmc_mdep); break; case IBMPOWER8E: case IBMPOWER8NVL: case IBMPOWER8: case IBMPOWER9: error = pmc_power8_initialize(pmc_mdep); break; case FSL_E500v1: case FSL_E500v2: case FSL_E500mc: case FSL_E5500: error = pmc_e500_initialize(pmc_mdep); break; default: error = -1; break; } if (error != 0) { pmc_mdep_free(pmc_mdep); pmc_mdep = NULL; } /* Set the value for kern.hwpmc.cpuid */ snprintf(pmc_cpuid, sizeof(pmc_cpuid), "%08x", mfpvr()); return (pmc_mdep); } void pmc_md_finalize(struct pmc_mdep *md) { free(powerpc_pcpu, M_PMC); powerpc_pcpu = NULL; } int pmc_save_user_callchain(uintptr_t *cc, int maxsamples, struct trapframe *tf) { uintptr_t *osp, *sp; int frames = 0; cc[frames++] = PMC_TRAPFRAME_TO_PC(tf); sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf); osp = NULL; for (; frames < maxsamples; frames++) { if (sp <= osp) break; osp = sp; #ifdef __powerpc64__ /* Check if 32-bit mode. */ if (!(tf->srr1 & PSL_SF)) { cc[frames] = fuword32((uint32_t *)sp + 1); sp = (uintptr_t *)(uintptr_t)fuword32(sp); } else { cc[frames] = fuword(sp + 2); sp = (uintptr_t *)fuword(sp); } #else cc[frames] = fuword32((uint32_t *)sp + 1); sp = (uintptr_t *)fuword32(sp); #endif } return (frames); } diff --git a/sys/dev/hwpmc/hwpmc_soft.c b/sys/dev/hwpmc/hwpmc_soft.c index c96a41457a66..3b602cfee89d 100644 --- a/sys/dev/hwpmc/hwpmc_soft.c +++ b/sys/dev/hwpmc/hwpmc_soft.c @@ -1,511 +1,506 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "hwpmc_soft.h" /* * Software PMC support. */ #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) struct soft_descr { struct pmc_descr pm_descr; /* "base class" */ }; static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = { #define SOFT_PMCDESCR(N) \ { \ .pm_descr = \ { \ .pd_name = #N, \ .pd_class = PMC_CLASS_SOFT, \ .pd_caps = SOFT_CAPS, \ .pd_width = 64 \ }, \ } SOFT_PMCDESCR(SOFT0), SOFT_PMCDESCR(SOFT1), SOFT_PMCDESCR(SOFT2), SOFT_PMCDESCR(SOFT3), SOFT_PMCDESCR(SOFT4), SOFT_PMCDESCR(SOFT5), SOFT_PMCDESCR(SOFT6), SOFT_PMCDESCR(SOFT7), SOFT_PMCDESCR(SOFT8), SOFT_PMCDESCR(SOFT9), SOFT_PMCDESCR(SOFT10), SOFT_PMCDESCR(SOFT11), SOFT_PMCDESCR(SOFT12), SOFT_PMCDESCR(SOFT13), SOFT_PMCDESCR(SOFT14), SOFT_PMCDESCR(SOFT15) }; /* * Per-CPU data structure. */ struct soft_cpu { struct pmc_hw soft_hw[SOFT_NPMCS]; pmc_value_t soft_values[SOFT_NPMCS]; }; static struct soft_cpu **soft_pcpu; static int soft_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event ev; struct pmc_soft *ps; (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_SOFT) return (EINVAL); if ((pm->pm_caps & SOFT_CAPS) == 0) return (EINVAL); if ((pm->pm_caps & ~SOFT_CAPS) != 0) return (EPERM); ev = pm->pm_event; if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) return (EINVAL); /* Check if event is registered. */ ps = pmc_soft_ev_acquire(ev); if (ps == NULL) return (EINVAL); pmc_soft_ev_release(ps); /* Module unload is protected by pmc SX lock. */ if (ps->ps_alloc != NULL) ps->ps_alloc(); return (0); } static int soft_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct soft_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; pd = &soft_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return (error); - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int soft_get_config(int cpu, int ri, struct pmc **ppm) { (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; return (0); } static int soft_pcpu_fini(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); free(soft_pcpu[cpu], M_PMC); soft_pcpu[cpu] = NULL; ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] ri=%d", __LINE__, ri)); pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; return (0); } static int soft_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, n; struct pmc_cpu *pc; struct soft_cpu *soft_pc; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", __LINE__)); soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); soft_pcpu[cpu] = soft_pc; phw = soft_pc->soft_hw; first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; for (n = 0; n < SOFT_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + first_ri] = phw; } return (0); } static int soft_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm __diagused; const struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); PMCDBG1(MDP,REA,1,"soft-read id=%d", ri); *v = soft_pcpu[cpu]->soft_values[ri]; return (0); } static int soft_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm __diagused; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); soft_pcpu[cpu]->soft_values[ri] = v; return (0); } static int soft_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; enum pmc_event ev; struct pmc_soft *ps; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); phw = &soft_pcpu[cpu]->soft_hw[ri]; KASSERT(phw->phw_pmc == NULL, ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); ev = pmc->pm_event; /* Check if event is registered. */ ps = pmc_soft_ev_acquire(ev); KASSERT(ps != NULL, ("[soft,%d] unregistered event %d", __LINE__, ev)); pmc_soft_ev_release(ps); /* Module unload is protected by pmc SX lock. */ if (ps->ps_release != NULL) ps->ps_release(); return (0); } static int soft_start_pmc(int cpu, int ri) { struct pmc *pm; struct soft_cpu *pc; struct pmc_soft *ps; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pc = soft_pcpu[cpu]; pm = pc->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); ps = pmc_soft_ev_acquire(pm->pm_event); if (ps == NULL) return (EINVAL); atomic_add_int(&ps->ps_running, 1); pmc_soft_ev_release(ps); return (0); } static int soft_stop_pmc(int cpu, int ri) { struct pmc *pm; struct soft_cpu *pc; struct pmc_soft *ps; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < SOFT_NPMCS, ("[soft,%d] illegal row-index %d", __LINE__, ri)); pc = soft_pcpu[cpu]; pm = pc->soft_hw[ri].phw_pmc; KASSERT(pm, ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); ps = pmc_soft_ev_acquire(pm->pm_event); /* event unregistered ? */ if (ps != NULL) { atomic_subtract_int(&ps->ps_running, 1); pmc_soft_ev_release(ps); } return (0); } int pmc_soft_intr(struct pmckern_soft *ks) { struct pmc *pm; struct soft_cpu *pc; int ri, processed, error, user_mode; KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); processed = 0; pc = soft_pcpu[ks->pm_cpu]; for (ri = 0; ri < SOFT_NPMCS; ri++) { pm = pc->soft_hw[ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || pm->pm_event != ks->pm_ev) { continue; } processed = 1; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { if ((pc->soft_values[ri]--) <= 0) pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; else continue; user_mode = TRAPF_USERMODE(ks->pm_tf); error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf); if (error) { soft_stop_pmc(ks->pm_cpu, ri); continue; } if (user_mode) { /* * If in user mode setup AST to process * callchain out of interrupt context. */ ast_sched(curthread, TDA_HWPMC); } } else pc->soft_values[ri]++; } if (processed) counter_u64_add(pmc_stats.pm_intr_processed, 1); else counter_u64_add(pmc_stats.pm_intr_ignored, 1); return (processed); } static void ast_hwpmc(struct thread *td, int tda __unused) { /* Handle Software PMC callchain capture. */ if (PMC_IS_PENDING_CALLCHAIN(td)) PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, (void *)td->td_frame); } void pmc_soft_initialize(struct pmc_mdep *md) { struct pmc_classdep *pcd; /* Add SOFT PMCs. */ soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, M_ZERO|M_WAITOK); pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; pcd->pcd_caps = SOFT_CAPS; pcd->pcd_class = PMC_CLASS_SOFT; pcd->pcd_num = SOFT_NPMCS; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 64; pcd->pcd_allocate_pmc = soft_allocate_pmc; pcd->pcd_config_pmc = soft_config_pmc; pcd->pcd_describe = soft_describe; pcd->pcd_get_config = soft_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_init = soft_pcpu_init; pcd->pcd_pcpu_fini = soft_pcpu_fini; pcd->pcd_read_pmc = soft_read_pmc; pcd->pcd_write_pmc = soft_write_pmc; pcd->pcd_release_pmc = soft_release_pmc; pcd->pcd_start_pmc = soft_start_pmc; pcd->pcd_stop_pmc = soft_stop_pmc; md->pmd_npmc += SOFT_NPMCS; ast_register(TDA_HWPMC, ASTR_UNCOND, 0, ast_hwpmc); } void pmc_soft_finalize(struct pmc_mdep *md) { #ifdef INVARIANTS int i, ncpus; ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", __LINE__, i)); KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); #endif ast_deregister(TDA_HWPMC); free(soft_pcpu, M_PMC); soft_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_tsc.c b/sys/dev/hwpmc/hwpmc_tsc.c index d59c8908f4ca..25b2e9f8fd57 100644 --- a/sys/dev/hwpmc/hwpmc_tsc.c +++ b/sys/dev/hwpmc/hwpmc_tsc.c @@ -1,381 +1,376 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * TSC support. */ #define TSC_CAPS PMC_CAP_READ struct tsc_descr { struct pmc_descr pm_descr; /* "base class" */ }; static struct tsc_descr tsc_pmcdesc[TSC_NPMCS] = { { .pm_descr = { .pd_name = "TSC", .pd_class = PMC_CLASS_TSC, .pd_caps = TSC_CAPS, .pd_width = 64 } } }; /* * Per-CPU data structure for TSCs. */ struct tsc_cpu { struct pmc_hw tc_hw; }; static struct tsc_cpu **tsc_pcpu; static int tsc_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] illegal row index %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_TSC) return (EINVAL); if (a->pm_ev != PMC_EV_TSC_TSC || a->pm_mode != PMC_MODE_SC) return (EINVAL); return (0); } static int tsc_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[tsc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return (0); } static int tsc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; - size_t copied; const struct tsc_descr *pd; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pd = &tsc_pmcdesc[ri]; - if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return (error); - + strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); pi->pm_class = pd->pm_descr.pd_class; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int tsc_get_config(int cpu, int ri, struct pmc **ppm) { (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); *ppm = tsc_pcpu[cpu]->tc_hw.phw_pmc; return (0); } static int tsc_get_msr(int ri, uint32_t *msr) { (void) ri; KASSERT(ri >= 0 && ri < TSC_NPMCS, ("[tsc,%d] ri %d out of range", __LINE__, ri)); *msr = MSR_TSC; return (0); } static int tsc_pcpu_fini(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu[cpu] != NULL, ("[tsc,%d] null pcpu", __LINE__)); free(tsc_pcpu[cpu], M_PMC); tsc_pcpu[cpu] = NULL; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; return (0); } static int tsc_pcpu_init(struct pmc_mdep *md, int cpu) { int ri; struct pmc_cpu *pc; struct tsc_cpu *tsc_pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); KASSERT(tsc_pcpu, ("[tsc,%d] null pcpu", __LINE__)); KASSERT(tsc_pcpu[cpu] == NULL, ("[tsc,%d] non-null per-cpu", __LINE__)); tsc_pc = malloc(sizeof(struct tsc_cpu), M_PMC, M_WAITOK|M_ZERO); tsc_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | PMC_PHW_FLAG_IS_SHAREABLE; tsc_pcpu[cpu] = tsc_pc; ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; KASSERT(pmc_pcpu, ("[tsc,%d] null generic pcpu", __LINE__)); pc = pmc_pcpu[cpu]; KASSERT(pc, ("[tsc,%d] null generic per-cpu", __LINE__)); pc->pc_hwpmcs[ri] = &tsc_pc->tc_hw; return (0); } static int tsc_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; enum pmc_mode mode __diagused; const struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal ri %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; pm = phw->phw_pmc; KASSERT(pm != NULL, ("[tsc,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); KASSERT(mode == PMC_MODE_SC, ("[tsc,%d] illegal pmc mode %d", __LINE__, mode)); PMCDBG1(MDP,REA,1,"tsc-read id=%d", ri); *v = rdtsc(); return (0); } static int tsc_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw __diagused; (void) pmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); phw = &tsc_pcpu[cpu]->tc_hw; KASSERT(phw->phw_pmc == NULL, ("[tsc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); /* * Nothing to do. */ return (0); } static int tsc_start_pmc(int cpu, int ri) { (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* TSCs are always running. */ } static int tsc_stop_pmc(int cpu, int ri) { (void) cpu; (void) ri; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); return (0); /* Cannot actually stop a TSC. */ } static int tsc_write_pmc(int cpu, int ri, pmc_value_t v) { (void) cpu; (void) ri; (void) v; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); /* * The TSCs are used as timecounters by the kernel, so even * though some i386 CPUs support writeable TSCs, we don't * support writing changing TSC values through the HWPMC API. */ return (0); } int pmc_tsc_initialize(struct pmc_mdep *md, int maxcpu) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[tsc,%d] md is NULL", __LINE__)); KASSERT(md->pmd_nclass >= 1, ("[tsc,%d] dubious md->nclass %d", __LINE__, md->pmd_nclass)); tsc_pcpu = malloc(sizeof(struct tsc_cpu *) * maxcpu, M_PMC, M_ZERO|M_WAITOK); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC]; pcd->pcd_caps = PMC_CAP_READ; pcd->pcd_class = PMC_CLASS_TSC; pcd->pcd_num = TSC_NPMCS; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = 64; pcd->pcd_allocate_pmc = tsc_allocate_pmc; pcd->pcd_config_pmc = tsc_config_pmc; pcd->pcd_describe = tsc_describe; pcd->pcd_get_config = tsc_get_config; pcd->pcd_get_msr = tsc_get_msr; pcd->pcd_pcpu_init = tsc_pcpu_init; pcd->pcd_pcpu_fini = tsc_pcpu_fini; pcd->pcd_read_pmc = tsc_read_pmc; pcd->pcd_release_pmc = tsc_release_pmc; pcd->pcd_start_pmc = tsc_start_pmc; pcd->pcd_stop_pmc = tsc_stop_pmc; pcd->pcd_write_pmc = tsc_write_pmc; md->pmd_npmc += TSC_NPMCS; return (0); } void pmc_tsc_finalize(struct pmc_mdep *md) { #ifdef INVARIANTS int i, ncpus; ncpus = pmc_cpu_max(); for (i = 0; i < ncpus; i++) KASSERT(tsc_pcpu[i] == NULL, ("[tsc,%d] non-null pcpu cpu %d", __LINE__, i)); KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_class == PMC_CLASS_TSC, ("[tsc,%d] class mismatch", __LINE__)); #else (void) md; #endif free(tsc_pcpu, M_PMC); tsc_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_uncore.c b/sys/dev/hwpmc/hwpmc_uncore.c index 56373f383dda..7f61deedfd79 100644 --- a/sys/dev/hwpmc/hwpmc_uncore.c +++ b/sys/dev/hwpmc/hwpmc_uncore.c @@ -1,817 +1,805 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Uncore PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #define UCF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE) #define UCP_PMC_CAPS \ (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define SELECTSEL(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCP_CB0_EVSEL0 : UCP_EVSEL0) #define SELECTOFF(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCF_OFFSET_SB : UCF_OFFSET) static enum pmc_cputype uncore_cputype; struct uncore_cpu { volatile uint32_t pc_ucfctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_uncorepmcs[]; }; static struct uncore_cpu **uncore_pcpu; static uint64_t uncore_pmcmask; static int uncore_ucf_ri; /* relative index of fixed counters */ static int uncore_ucf_width; static int uncore_ucf_npmc; static int uncore_ucp_width; static int uncore_ucp_npmc; static int uncore_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int uncore_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct uncore_cpu *cc; struct pmc_hw *phw; int uncore_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); uncore_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + uncore_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + uncore_ri] = phw; } return (0); } static int uncore_pcpu_fini(struct pmc_mdep *md, int cpu) { int uncore_ri, n, npmc; struct pmc_cpu *pc; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); if ((cc = uncore_pcpu[cpu]) == NULL) return (0); uncore_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(SELECTSEL(uncore_cputype) + n, 0); wrmsr(UCF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + uncore_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t ucf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (uncore_ucf_width - 1))) == 0) return (0); v &= (1ULL << uncore_ucf_width) - 1; return (1ULL << uncore_ucf_width) - v; } static pmc_value_t ucf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucf_width) - rlc; } static int ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint32_t flags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > uncore_ucf_npmc) return (EINVAL); if (a->pm_class != PMC_CLASS_UCF) return (EINVAL); flags = UCF_EN; pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); return (0); } static int ucf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; return (0); } static int ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char ucf_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; - (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); - if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "UCF-%d", ri); pi->pm_class = PMC_CLASS_UCF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; return (0); } static int ucf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + uncore_ucf_ri)); tmp = rdmsr(UCF_CTR0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); return (0); } static int ucf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); } static void ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; pcd->pcd_caps = UCF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucf_allocate_pmc; pcd->pcd_config_pmc = ucf_config_pmc; pcd->pcd_describe = ucf_describe; pcd->pcd_get_config = ucf_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_noop; pcd->pcd_pcpu_init = uncore_pcpu_noop; pcd->pcd_read_pmc = ucf_read_pmc; pcd->pcd_release_pmc = ucf_release_pmc; pcd->pcd_start_pmc = ucf_start_pmc; pcd->pcd_stop_pmc = ucf_stop_pmc; pcd->pcd_write_pmc = ucf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct ucp_event_descr { enum pmc_event ucp_ev; unsigned char ucp_evcode; unsigned char ucp_umask; unsigned char ucp_flags; }; #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ #define UCP_F_WM (1 << 1) /* CPU: Westmere */ #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ #define UCP_F_HW (1 << 3) /* CPU: Haswell */ #define UCP_F_FM (1 << 4) /* Fixed mask */ #define UCP_F_ALLCPUS \ (UCP_F_I7 | UCP_F_WM) #define UCP_F_CMASK 0xFF000000 static pmc_value_t ucp_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucp_width) - 1; return (1ULL << uncore_ucp_width) - v; } static pmc_value_t ucp_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucp_width) - rlc; } /* * Counter specific event information for Sandybridge and Haswell */ static int ucp_event_sb_hw_ok_on_counter(uint8_t ev, int ri) { uint32_t mask; switch (ev) { /* * Events valid only on counter 0. */ case 0x80: case 0x83: mask = (1 << 0); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { uint8_t ev; const struct pmc_md_ucp_op_pmcallocate *ucp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); if (a->pm_class != PMC_CLASS_UCP) return (EINVAL); ucp = &a->pm_md.pm_ucp; ev = UCP_EVSEL(ucp->pm_ucp_config); switch (uncore_cputype) { case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_SANDYBRIDGE: if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: break; } pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN; return (0); } static int ucp_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; return (0); } static int ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { - int error; struct pmc_hw *phw; - char ucp_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; - (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); - if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, - NULL)) != 0) - return (error); - + snprintf(pi->pm_name, sizeof(pi->pm_name), "UCP-%d", ri); pi->pm_class = PMC_CLASS_UCP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucp_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; return (0); } static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); } static int ucp_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; uint64_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); return (0); } static int ucp_stop_pmc(int cpu, int ri) { struct pmc *pm __diagused; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); /* stop hw. */ wrmsr(SELECTSEL(uncore_cputype) + ri, 0); /* Don't need to write UC_GLOBAL_CTRL, one disable is enough. */ return (0); } static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); } static void ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucp-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; pcd->pcd_caps = UCP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucp_allocate_pmc; pcd->pcd_config_pmc = ucp_config_pmc; pcd->pcd_describe = ucp_describe; pcd->pcd_get_config = ucp_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_fini; pcd->pcd_pcpu_init = uncore_pcpu_init; pcd->pcd_read_pmc = ucp_read_pmc; pcd->pcd_release_pmc = ucp_release_pmc; pcd->pcd_start_pmc = ucp_start_pmc; pcd->pcd_stop_pmc = ucp_stop_pmc; pcd->pcd_write_pmc = ucp_write_pmc; md->pmd_npmc += npmc; } int pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) { uncore_cputype = md->pmd_cputype; uncore_pmcmask = 0; /* * Initialize programmable counters. */ uncore_ucp_npmc = 8; uncore_ucp_width = 48; uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); /* * Initialize fixed function counters, if present. */ uncore_ucf_ri = uncore_ucp_npmc; uncore_ucf_npmc = 1; uncore_ucf_width = 48; ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, uncore_ucf_ri); uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); return (0); } void pmc_uncore_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "uncore-finalize"); free(uncore_pcpu, M_PMC); uncore_pcpu = NULL; }