diff --git a/sys/dev/hwpmc/hwpmc_dmc620.c b/sys/dev/hwpmc/hwpmc_dmc620.c new file mode 100644 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_dmc620.c @@ -0,0 +1,741 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2003-2008 Joseph Koshy + * Copyright (c) 2007 The FreeBSD Foundation + * Copyright (c) 2021 Ampere Computing LLC + * + * Portions of this software were developed by A. Joseph Koshy under + * sponsorship from the FreeBSD Foundation and Google, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Support for ARM DMC-620 Memory Controller PMU */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include + +#define DMC620_TYPE_CLKDIV2 0 +#define DMC620_TYPE_CLK 1 +#define CLASS2TYPE(c) ((c) - PMC_CLASS_DMC620_PMU_CD2) + +/* Create wrapper for each class. */ +#define CLASSDEP_FN2(fn, t1, a1, t2, a2) \ + static int fn(int class, t1 a1, t2 a2); \ + static int fn ## _cd2(t1 a1, t2 a2) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2)); \ + } \ + static int fn ## _c(t1 a1, t2 a2) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2)); \ + } \ + static int fn(int class, t1 a1, t2 a2) + +#define CLASSDEP_FN3(fn, t1, a1, t2, a2, t3, a3) \ + static int fn(int class, t1 a1, t2 a2, t3 a3); \ + static int fn ## _cd2(t1 a1, t2 a2, t3 a3) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2, a3)); \ + } \ + static int fn ## _c(t1 a1, t2 a2, t3 a3) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2, a3)); \ + } \ + static int fn(int class, t1 a1, t2 a2, t3 a3) + +#define CLASSDEP_FN4(fn, t1, a1, t2, a2, t3, a3, t4, a4) \ + static int fn(int class, t1 a1, t2 a2, t3 a3, t4 a4); \ + static int fn ## _cd2(t1 a1, t2 a2, t3 a3, t4 a4) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_CD2, a1, a2, a3, a4)); \ + } \ + static int fn ## _c(t1 a1, t2 a2, t3 a3, t4 a4) \ + { \ + return (fn(PMC_CLASS_DMC620_PMU_C, a1, a2, a3, a4)); \ + } \ + static int fn(int class, t1 a1, t2 a2, t3 a3, t4 a4) + +struct dmc620_pmc { + void *arg; + int domain; +}; + +struct dmc620_descr { + struct pmc_descr pd_descr; /* "base class" */ + void *pd_rw_arg; /* Argument to use with read/write */ + struct pmc *pd_pmc; + struct pmc_hw *pd_phw; + uint32_t pd_config; + uint32_t pd_match; + uint32_t pd_mask; + uint32_t pd_evsel; /* address of EVSEL register */ + uint32_t pd_perfctr; /* address of PERFCTR register */ + +}; + +static struct dmc620_descr **dmc620_pmcdesc[2]; +static struct dmc620_pmc dmc620_pmcs[DMC620_UNIT_MAX]; +static int dmc620_npmcs = 0; + +void +dmc620_pmc_register(int unit, void *arg, int domain) +{ + + if (unit >= DMC620_UNIT_MAX) { + /* TODO */ + return; + } + + dmc620_pmcs[unit].arg = arg; + dmc620_pmcs[unit].domain = domain; + dmc620_npmcs++; +} + +void +dmc620_pmc_unregister(int unit) +{ + + dmc620_pmcs[unit].arg = NULL; + dmc620_npmcs--; +} + +int +pmc_dmc620_nclasses() +{ + + if (dmc620_npmcs > 0) + return (2); + return (0); +} + +static inline struct dmc620_descr * +dmc620desc(int class, int cpu, int ri) +{ + int c; + + c = CLASS2TYPE(class); + KASSERT((c & 0xfffffffe) == 0, ("[dmc620,%d] 'c' can only be 0 or 1. " + "now %d", __LINE__, c)); + + return (dmc620_pmcdesc[c][ri]); +} + +static inline int +cntr(int class, int ri) +{ + int c; + + c = CLASS2TYPE(class); + KASSERT((c & 0xfffffffe) == 0, ("[dmc620,%d] 'c' can only be 0 or 1. " + "now %d", __LINE__, c)); + + if (c == DMC620_TYPE_CLKDIV2) + return (ri % DMC620_CLKDIV2_COUNTERS_N); + return ((ri % DMC620_CLK_COUNTERS_N) + DMC620_CLKDIV2_COUNTERS_N); +} + +static inline int +class2mdep(int class) +{ + + switch (class) { + case PMC_CLASS_DMC620_PMU_CD2: + return (PMC_MDEP_CLASS_INDEX_DMC620_CD2); + case PMC_CLASS_DMC620_PMU_C: + return (PMC_MDEP_CLASS_INDEX_DMC620_C); + } + return (-1); +} + +static inline int +class_ri2unit(int class, int ri) +{ + + if (class == PMC_CLASS_DMC620_PMU_CD2) + return (ri / DMC620_CLKDIV2_COUNTERS_N); + else + return (ri / DMC620_CLK_COUNTERS_N); +} + +/* + * read a pmc register + */ + +CLASSDEP_FN3(dmc620_read_pmc, int, cpu, int, ri, pmc_value_t *, v) +{ + struct dmc620_descr *desc; + struct pmc *pm; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + desc = dmc620desc(class, cpu, ri); + pm = desc->pd_phw->phw_pmc; + + KASSERT(pm != NULL, + ("[dmc620,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, + cpu, ri)); + + PMCDBG3(MDP,REA,1,"%s id=%d class=%d", __func__, ri, class); + + /* + * Should emulate 64bits, because 32 bits counter overflows faster than + * pmcstat default period. + */ + /* Always CPU0. Single controller for all CPUs. */ + *v = ((uint64_t)pm->pm_pcpu_state[0].pps_overflowcnt << 32) | + pmu_dmc620_rd4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_VALUE_LO); + + PMCDBG3(MDP, REA, 2, "%s id=%d -> %jd", __func__, ri, *v); + + return (0); +} + +/* + * Write a pmc register. + */ + +CLASSDEP_FN3(dmc620_write_pmc, int, cpu, int, ri, pmc_value_t, v) +{ + struct dmc620_descr *desc; + struct pmc *pm; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + desc = dmc620desc(class, cpu, ri); + pm = desc->pd_phw->phw_pmc; + + KASSERT(pm != NULL, + ("[dmc620,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, + cpu, ri)); + + PMCDBG4(MDP, WRI, 1, "%s cpu=%d ri=%d v=%jx", __func__, cpu, ri, v); + + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_VALUE_LO, v); + return (0); +} + +/* + * configure hardware pmc according to the configuration recorded in + * pmc 'pm'. + */ + +CLASSDEP_FN3(dmc620_config_pmc, int, cpu, int, ri, struct pmc *, pm) +{ + struct pmc_hw *phw; + + PMCDBG4(MDP, CFG, 1, "%s cpu=%d ri=%d pm=%p", __func__, cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + phw = dmc620desc(class, cpu, ri)->pd_phw; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[dmc620,%d] pm=%p phw->pm=%p hwpmc not unconfigured", + __LINE__, pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + return (0); +} + +/* + * Retrieve a configured PMC pointer from hardware state. + */ + +CLASSDEP_FN3(dmc620_get_config, int, cpu, int, ri, struct pmc **, ppm) +{ + + *ppm = dmc620desc(class, cpu, ri)->pd_phw->phw_pmc; + + return (0); +} + +/* + * Check if a given allocation is feasible. + */ + +CLASSDEP_FN4(dmc620_allocate_pmc, int, cpu, int, ri, struct pmc *,pm, + const struct pmc_op_pmcallocate *, a) +{ + const struct pmc_descr *pd; + uint64_t caps, control; + enum pmc_event pe; + uint8_t e; + + (void) cpu; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + pd = &dmc620desc(class, cpu, ri)->pd_descr; + if (dmc620_pmcs[class_ri2unit(class, ri)].domain != + pcpu_find(cpu)->pc_domain) + return (EINVAL); + + /* check class match */ + if (pd->pd_class != a->pm_class) + return (EINVAL); + + caps = pm->pm_caps; + + PMCDBG3(MDP, ALL, 1, "%s ri=%d caps=0x%x", __func__, ri, caps); + + pe = a->pm_ev; + if (class == PMC_CLASS_DMC620_PMU_CD2) + e = pe - PMC_EV_DMC620_PMU_CD2_FIRST; + else + e = pe - PMC_EV_DMC620_PMU_C_FIRST; + + control = (e << DMC620_COUNTER_CONTROL_EVENT_SHIFT) & + DMC620_COUNTER_CONTROL_EVENT_MASK; + + if (caps & PMC_CAP_INVERT) + control |= DMC620_COUNTER_CONTROL_INVERT; + + pm->pm_md.pm_dmc620.pm_control = control; + pm->pm_md.pm_dmc620.pm_match = a->pm_md.pm_dmc620.pm_dmc620_match; + pm->pm_md.pm_dmc620.pm_mask = a->pm_md.pm_dmc620.pm_dmc620_mask; + + PMCDBG3(MDP, ALL, 2, "%s ri=%d -> control=0x%x", __func__, ri, control); + + return (0); +} + +/* + * Release machine dependent state associated with a PMC. This is a + * no-op on this architecture. + * + */ + +/* ARGSUSED0 */ +CLASSDEP_FN3(dmc620_release_pmc, int, cpu, int, ri, struct pmc *, pmc) +{ + struct pmc_hw *phw; + + (void) pmc; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + phw = dmc620desc(class, cpu, ri)->pd_phw; + + KASSERT(phw->phw_pmc == NULL, + ("[dmc620,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + return (0); +} + +/* + * start a PMC. + */ + +CLASSDEP_FN2(dmc620_start_pmc, int, cpu, int, ri) +{ + struct dmc620_descr *desc; + struct pmc_hw *phw; + uint64_t control; + struct pmc *pm; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + desc = dmc620desc(class, cpu, ri); + phw = desc->pd_phw; + pm = phw->phw_pmc; + + KASSERT(pm != NULL, + ("[dmc620,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, + cpu, ri)); + + PMCDBG3(MDP, STA, 1, "%s cpu=%d ri=%d", __func__, cpu, ri); + + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_MASK_LO, pm->pm_md.pm_dmc620.pm_mask & 0xffffffff); + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_MASK_HI, pm->pm_md.pm_dmc620.pm_mask >> 32); + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_MATCH_LO, pm->pm_md.pm_dmc620.pm_match & 0xffffffff); + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_MATCH_HI, pm->pm_md.pm_dmc620.pm_match >> 32); + /* turn on the PMC ENABLE bit */ + control = pm->pm_md.pm_dmc620.pm_control | DMC620_COUNTER_CONTROL_ENABLE; + + PMCDBG2(MDP, STA, 2, "%s control=0x%x", __func__, control); + + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_CONTROL, control); + return (0); +} + +/* + * Stop a PMC. + */ + +CLASSDEP_FN2(dmc620_stop_pmc, int, cpu, int, ri) +{ + struct dmc620_descr *desc; + struct pmc_hw *phw; + struct pmc *pm; + uint64_t control; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + desc = dmc620desc(class, cpu, ri); + phw = desc->pd_phw; + pm = phw->phw_pmc; + + KASSERT(pm != NULL, + ("[dmc620,%d] cpu%d,pmc%d no PMC to stop", __LINE__, + cpu, ri)); + + PMCDBG2(MDP, STO, 1, "%s ri=%d", __func__, ri); + + /* turn off the PMC ENABLE bit */ + control = pm->pm_md.pm_dmc620.pm_control & ~DMC620_COUNTER_CONTROL_ENABLE; + pmu_dmc620_wr4(desc->pd_rw_arg, cntr(class, ri), + DMC620_COUNTER_CONTROL, control); + + return (0); +} + +/* + * describe a PMC + */ +CLASSDEP_FN4(dmc620_describe, int, cpu, int, ri, struct pmc_info *, pi, + struct pmc **, ppmc) +{ + struct pmc_hw *phw; + size_t copied; + int error; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0, ("[dmc620,%d] row-index %d out of range", __LINE__, + ri)); + + phw = dmc620desc(class, cpu, ri)->pd_phw; + + if ((error = copystr(dmc620desc(class, cpu, ri)->pd_descr.pd_name, + pi->pm_name, PMC_NAME_MAX, &copied)) != 0) + return (error); + + pi->pm_class = dmc620desc(class, cpu, ri)->pd_descr.pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = TRUE; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = FALSE; + *ppmc = NULL; + } + + return (0); +} + +/* + * processor dependent initialization. + */ + +CLASSDEP_FN2(dmc620_pcpu_init, struct pmc_mdep *, md, int, cpu) +{ + int first_ri, n, npmc; + struct pmc_hw *phw; + struct pmc_cpu *pc; + int mdep_class; + + mdep_class = class2mdep(class); + KASSERT(mdep_class != -1, ("[dmc620,%d] wrong class %d", __LINE__, + class)); + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] insane cpu number %d", __LINE__, cpu)); + + PMCDBG1(MDP, INI, 1, "dmc620-init cpu=%d", cpu); + + /* + * Set the content of the hardware descriptors to a known + * state and initialize pointers in the MI per-cpu descriptor. + */ + + pc = pmc_pcpu[cpu]; + first_ri = md->pmd_classdep[mdep_class].pcd_ri; + npmc = md->pmd_classdep[mdep_class].pcd_num; + + for (n = 0; n < npmc; n++, phw++) { + phw = dmc620desc(class, cpu, n)->pd_phw; + phw->phw_state = PMC_PHW_CPU_TO_STATE(cpu) | + PMC_PHW_INDEX_TO_STATE(n); + /* Set enabled only if unit present. */ + if (dmc620_pmcs[class_ri2unit(class, n)].arg != NULL) + phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; + phw->phw_pmc = NULL; + pc->pc_hwpmcs[n + first_ri] = phw; + } + return (0); +} + +/* + * processor dependent cleanup prior to the KLD + * being unloaded + */ + +CLASSDEP_FN2(dmc620_pcpu_fini, struct pmc_mdep *, md, int, cpu) +{ + + return (0); +} + +int +dmc620_intr(struct trapframe *tf, int class, int unit, int i) +{ + struct pmc_cpu *pc; + struct pmc_hw *phw; + struct pmc *pm; + int error, cpu, ri; + + ri = i + unit * ((class == PMC_CLASS_DMC620_PMU_CD2) ? + DMC620_CLKDIV2_COUNTERS_N : DMC620_CLK_COUNTERS_N); + cpu = curcpu; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[dmc620,%d] CPU %d out of range", __LINE__, cpu)); + pc = pmc_pcpu[cpu]; + KASSERT(pc != NULL, ("pc != NULL")); + + phw = dmc620desc(class, cpu, ri)->pd_phw; + KASSERT(phw != NULL, ("phw != NULL")); + pm = phw->phw_pmc; + if (pm == NULL) + return (0); + + if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { + /* Always CPU0. */ + pm->pm_pcpu_state[0].pps_overflowcnt += 1; + return (0); + } + + if (pm->pm_state != PMC_STATE_RUNNING) + return (0); + + error = pmc_process_interrupt(PMC_HR, pm, tf); + if (error) + dmc620_stop_pmc(class, cpu, ri); + + /* Reload sampling count */ + dmc620_write_pmc(class, cpu, ri, pm->pm_sc.pm_reloadcount); + + return (0); +} + +/* + * Initialize ourselves. + */ + +int +pmc_dmc620_initialize_cd2(struct pmc_mdep *md) +{ + struct pmc_classdep *pcd; + int i, npmc, unit; + + KASSERT(md != NULL, ("[dmc620,%d] md is NULL", __LINE__)); + KASSERT(dmc620_npmcs <= DMC620_UNIT_MAX, + ("[dmc620,%d] dmc620_npmcs too big", __LINE__)); + + PMCDBG0(MDP,INI,1, "dmc620-initialize"); + + npmc = DMC620_CLKDIV2_COUNTERS_N * dmc620_npmcs; + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2]; + + pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | + PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER | + PMC_CAP_INTERRUPT | PMC_CAP_DOMWIDE; + pcd->pcd_class = PMC_CLASS_DMC620_PMU_CD2; + pcd->pcd_num = npmc; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 32; + + pcd->pcd_allocate_pmc = dmc620_allocate_pmc_cd2; + pcd->pcd_config_pmc = dmc620_config_pmc_cd2; + pcd->pcd_describe = dmc620_describe_cd2; + pcd->pcd_get_config = dmc620_get_config_cd2; + pcd->pcd_get_msr = NULL; + pcd->pcd_pcpu_fini = dmc620_pcpu_fini_cd2; + pcd->pcd_pcpu_init = dmc620_pcpu_init_cd2; + pcd->pcd_read_pmc = dmc620_read_pmc_cd2; + pcd->pcd_release_pmc = dmc620_release_pmc_cd2; + pcd->pcd_start_pmc = dmc620_start_pmc_cd2; + pcd->pcd_stop_pmc = dmc620_stop_pmc_cd2; + pcd->pcd_write_pmc = dmc620_write_pmc_cd2; + + md->pmd_npmc += npmc; + dmc620_pmcdesc[0] = malloc(sizeof(struct dmc620_descr *) * npmc * + DMC620_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); + for (i = 0; i < npmc; i++) { + dmc620_pmcdesc[0][i] = malloc(sizeof(struct dmc620_descr), + M_PMC, M_WAITOK|M_ZERO); + + unit = i / DMC620_CLKDIV2_COUNTERS_N; + KASSERT(unit >= 0, ("unit >= 0")); + KASSERT(dmc620_pmcs[unit].arg != NULL, ("arg != NULL")); + + dmc620_pmcdesc[0][i]->pd_rw_arg = dmc620_pmcs[unit].arg; + dmc620_pmcdesc[0][i]->pd_descr.pd_class = + PMC_CLASS_DMC620_PMU_CD2; + dmc620_pmcdesc[0][i]->pd_descr.pd_caps = pcd->pcd_caps; + dmc620_pmcdesc[0][i]->pd_phw = malloc(sizeof(struct pmc_hw), + M_PMC, M_WAITOK|M_ZERO); + snprintf(dmc620_pmcdesc[0][i]->pd_descr.pd_name, 63, + "DMC620_CD2_%d", i); + } + + return (0); +} + +int +pmc_dmc620_initialize_c(struct pmc_mdep *md) +{ + struct pmc_classdep *pcd; + int i, npmc, unit; + + KASSERT(md != NULL, ("[dmc620,%d] md is NULL", __LINE__)); + KASSERT(dmc620_npmcs <= DMC620_UNIT_MAX, + ("[dmc620,%d] dmc620_npmcs too big", __LINE__)); + + PMCDBG0(MDP,INI,1, "dmc620-initialize"); + + npmc = DMC620_CLK_COUNTERS_N * dmc620_npmcs; + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C]; + + pcd->pcd_caps = PMC_CAP_SYSTEM | PMC_CAP_READ | + PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER | + PMC_CAP_INTERRUPT | PMC_CAP_DOMWIDE; + pcd->pcd_class = PMC_CLASS_DMC620_PMU_C; + pcd->pcd_num = npmc; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 32; + + pcd->pcd_allocate_pmc = dmc620_allocate_pmc_c; + pcd->pcd_config_pmc = dmc620_config_pmc_c; + pcd->pcd_describe = dmc620_describe_c; + pcd->pcd_get_config = dmc620_get_config_c; + pcd->pcd_get_msr = NULL; + pcd->pcd_pcpu_fini = dmc620_pcpu_fini_c; + pcd->pcd_pcpu_init = dmc620_pcpu_init_c; + pcd->pcd_read_pmc = dmc620_read_pmc_c; + pcd->pcd_release_pmc = dmc620_release_pmc_c; + pcd->pcd_start_pmc = dmc620_start_pmc_c; + pcd->pcd_stop_pmc = dmc620_stop_pmc_c; + pcd->pcd_write_pmc = dmc620_write_pmc_c; + + md->pmd_npmc += npmc; + dmc620_pmcdesc[1] = malloc(sizeof(struct dmc620_descr *) * npmc * + DMC620_PMU_DEFAULT_UNITS_N, M_PMC, M_WAITOK|M_ZERO); + for (i = 0; i < npmc; i++) { + dmc620_pmcdesc[1][i] = malloc(sizeof(struct dmc620_descr), + M_PMC, M_WAITOK|M_ZERO); + + unit = i / DMC620_CLK_COUNTERS_N; + KASSERT(unit >= 0, ("unit >= 0")); + KASSERT(dmc620_pmcs[unit].arg != NULL, ("arg != NULL")); + + dmc620_pmcdesc[1][i]->pd_rw_arg = dmc620_pmcs[unit].arg; + dmc620_pmcdesc[1][i]->pd_descr.pd_class = PMC_CLASS_DMC620_PMU_C; + dmc620_pmcdesc[1][i]->pd_descr.pd_caps = pcd->pcd_caps; + dmc620_pmcdesc[1][i]->pd_phw = malloc(sizeof(struct pmc_hw), + M_PMC, M_WAITOK|M_ZERO); + snprintf(dmc620_pmcdesc[1][i]->pd_descr.pd_name, 63, + "DMC620_C_%d", i); + } + + return (0); +} + +void +pmc_dmc620_finalize_cd2(struct pmc_mdep *md) +{ + struct pmc_classdep *pcd; + int i, npmc; + + KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2].pcd_class == + PMC_CLASS_DMC620_PMU_CD2, ("[dmc620,%d] pmc class mismatch", + __LINE__)); + + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_CD2]; + + npmc = pcd->pcd_num; + for (i = 0; i < npmc; i++) { + free(dmc620_pmcdesc[0][i]->pd_phw, M_PMC); + free(dmc620_pmcdesc[0][i], M_PMC); + } + free(dmc620_pmcdesc[0], M_PMC); + dmc620_pmcdesc[0] = NULL; +} + +void +pmc_dmc620_finalize_c(struct pmc_mdep *md) +{ + struct pmc_classdep *pcd; + int i, npmc; + + KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C].pcd_class == + PMC_CLASS_DMC620_PMU_C, ("[dmc620,%d] pmc class mismatch", + __LINE__)); + + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_DMC620_C]; + + npmc = pcd->pcd_num; + for (i = 0; i < npmc; i++) { + free(dmc620_pmcdesc[1][i]->pd_phw, M_PMC); + free(dmc620_pmcdesc[1][i], M_PMC); + } + free(dmc620_pmcdesc[1], M_PMC); + dmc620_pmcdesc[1] = NULL; +} diff --git a/sys/dev/hwpmc/pmu_dmc620.c b/sys/dev/hwpmc/pmu_dmc620.c new file mode 100644 --- /dev/null +++ b/sys/dev/hwpmc/pmu_dmc620.c @@ -0,0 +1,279 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Ampere Computing LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_hwpmc_hooks.h" +#include "opt_acpi.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +static char *pmu_dmc620_ids[] = { + "ARMHD620", + NULL +}; + +static struct resource_spec pmu_dmc620_res_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, + { -1, 0 } +}; + +struct pmu_dmc620_softc { + device_t sc_dev; + int sc_unit; + int sc_domain; + struct resource *sc_res[2]; + void *sc_ih; + uint32_t sc_clkdiv2_conters_hi[DMC620_CLKDIV2_COUNTERS_N]; + uint32_t sc_clk_conters_hi[DMC620_CLK_COUNTERS_N]; + uint32_t sc_saved_control[DMC620_COUNTERS_N]; +}; + +#define RD4(sc, r) bus_read_4((sc)->sc_res[0], (r)) +#define WR4(sc, r, v) bus_write_4((sc)->sc_res[0], (r), (v)) +#define MD4(sc, r, c, s) WR4((sc), (r), RD4((sc), (r)) & ~(c) | (s)) + +#define CD2MD4(sc, u, r, c, s) MD4((sc), DMC620_CLKDIV2_REG((u), (r)), (c), (s)) +#define CMD4(sc, u, r, c, s) MD4((sc), DMC620_CLK_REG((u), (r)), (c), (s)) + +static int pmu_dmc620_counter_overflow_intr(void *arg); + +uint32_t +pmu_dmc620_rd4(void *arg, u_int cntr, off_t reg) +{ + struct pmu_dmc620_softc *sc; + uint32_t val; + + sc = (struct pmu_dmc620_softc *)arg; + KASSERT(cntr < DMC620_COUNTERS_N, ("Wrong counter unit %d", cntr)); + + val = RD4(sc, DMC620_REG(cntr, reg)); + return (val); +} + +void +pmu_dmc620_wr4(void *arg, u_int cntr, off_t reg, uint32_t val) +{ + struct pmu_dmc620_softc *sc; + + sc = (struct pmu_dmc620_softc *)arg; + KASSERT(cntr < DMC620_COUNTERS_N, ("Wrong counter unit %d", cntr)); + + WR4(sc, DMC620_REG(cntr, reg), val); +} + +static int +pmu_dmc620_acpi_probe(device_t dev) +{ + int err; + + err = ACPI_ID_PROBE(device_get_parent(dev), dev, pmu_dmc620_ids, NULL); + if (err <= 0) + device_set_desc(dev, "ARM DMC-620 Memory Controller PMU"); + + return (err); +} + +static int +pmu_dmc620_acpi_attach(device_t dev) +{ + struct pmu_dmc620_softc *sc; + int domain, i, u; + const char *dname; + + dname = device_get_name(dev); + sc = device_get_softc(dev); + sc->sc_dev = dev; + u = device_get_unit(dev); + sc->sc_unit = u; + + /* + * Ampere Altra support NUMA emulation, but DMC-620 PMU units have no + * mapping. Emulate this with kenv/hints. + * Format "hint.pmu_dmc620.3.domain=1". + */ + if ((resource_int_value(dname, u, "domain", &domain) == 0 || + bus_get_domain(dev, &domain) == 0) && domain < MAXMEMDOM) { + sc->sc_domain = domain; + } + device_printf(dev, "domain=%d\n", domain); + + i = bus_alloc_resources(dev, pmu_dmc620_res_spec, sc->sc_res); + if (i != 0) { + device_printf(dev, "cannot allocate resources for device (%d)\n", + i); + return (i); + } + /* Disable counter before enable interrupt. */ + for (i = 0; i < DMC620_CLKDIV2_COUNTERS_N; i++) { + CD2MD4(sc, i, DMC620_COUNTER_CONTROL, + DMC620_COUNTER_CONTROL_ENABLE, 0); + } + for (i = 0; i < DMC620_CLK_COUNTERS_N; i++) { + CMD4(sc, i, DMC620_COUNTER_CONTROL, + DMC620_COUNTER_CONTROL_ENABLE, 0); + } + + /* Clear intr status. */ + WR4(sc, DMC620_OVERFLOW_STATUS_CLKDIV2, 0); + WR4(sc, DMC620_OVERFLOW_STATUS_CLK, 0); + + if (sc->sc_res[1] != NULL && bus_setup_intr(dev, sc->sc_res[1], + INTR_TYPE_MISC | INTR_MPSAFE, pmu_dmc620_counter_overflow_intr, + NULL, sc, &sc->sc_ih)) { + bus_release_resources(dev, pmu_dmc620_res_spec, sc->sc_res); + device_printf(dev, "cannot setup interrupt handler\n"); + return (ENXIO); + } + dmc620_pmc_register(u, sc, domain); + return (0); +} + +static int +pmu_dmc620_acpi_detach(device_t dev) +{ + struct pmu_dmc620_softc *sc; + + sc = device_get_softc(dev); + dmc620_pmc_unregister(device_get_unit(dev)); + if (sc->sc_res[1] != NULL) { + bus_teardown_intr(dev, sc->sc_res[1], sc->sc_ih); + } + bus_release_resources(dev, pmu_dmc620_res_spec, sc->sc_res); + + return (0); +} + +static void +pmu_dmc620_clkdiv2_overflow(struct trapframe *tf, struct pmu_dmc620_softc *sc, + u_int i) +{ + + atomic_add_32(&sc->sc_clkdiv2_conters_hi[i], 1); + /* Call dmc620 handler directly, because hook busy by arm64_intr. */ + dmc620_intr(tf, PMC_CLASS_DMC620_PMU_CD2, sc->sc_unit, i); +} + +static void +pmu_dmc620_clk_overflow(struct trapframe *tf, struct pmu_dmc620_softc *sc, + u_int i) +{ + + atomic_add_32(&sc->sc_clk_conters_hi[i], 1); + /* Call dmc620 handler directly, because hook busy by arm64_intr. */ + dmc620_intr(tf, PMC_CLASS_DMC620_PMU_C, sc->sc_unit, i); + +} + +static int +pmu_dmc620_counter_overflow_intr(void *arg) +{ + uint32_t clkdiv2_stat, clk_stat; + struct pmu_dmc620_softc *sc; + struct trapframe *tf; + u_int i; + + tf = PCPU_GET(curthread)->td_intr_frame; + sc = (struct pmu_dmc620_softc *) arg; + clkdiv2_stat = RD4(sc, DMC620_OVERFLOW_STATUS_CLKDIV2); + clk_stat = RD4(sc, DMC620_OVERFLOW_STATUS_CLK); + + if ((clkdiv2_stat == 0) && (clk_stat == 0)) + return (FILTER_STRAY); + /* Stop and save states of all counters. */ + for (i = 0; i < DMC620_COUNTERS_N; i++) { + sc->sc_saved_control[i] = RD4(sc, DMC620_REG(i, + DMC620_COUNTER_CONTROL)); + WR4(sc, DMC620_REG(i, DMC620_COUNTER_CONTROL), + sc->sc_saved_control[i] & ~DMC620_COUNTER_CONTROL_ENABLE); + } + + if (clkdiv2_stat != 0) { + for (i = 0; i < DMC620_CLKDIV2_COUNTERS_N; i++) { + if ((clkdiv2_stat & (1 << i)) == 0) + continue; + pmu_dmc620_clkdiv2_overflow(tf, sc, i); + } + WR4(sc, DMC620_OVERFLOW_STATUS_CLKDIV2, 0); + } + if (clk_stat != 0) { + for (i = 0; i < DMC620_CLK_COUNTERS_N; i++) { + if ((clk_stat & (1 << i)) == 0) + continue; + pmu_dmc620_clk_overflow(tf, sc, i); + } + WR4(sc, DMC620_OVERFLOW_STATUS_CLK, 0); + } + + /* Restore states of all counters. */ + for (i = 0; i < DMC620_COUNTERS_N; i++) { + WR4(sc, DMC620_REG(i, DMC620_COUNTER_CONTROL), + sc->sc_saved_control[i]); + } + + return (FILTER_HANDLED); +} + +static device_method_t pmu_dmc620_acpi_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, pmu_dmc620_acpi_probe), + DEVMETHOD(device_attach, pmu_dmc620_acpi_attach), + DEVMETHOD(device_detach, pmu_dmc620_acpi_detach), + + /* End */ + DEVMETHOD_END +}; + +static driver_t pmu_dmc620_acpi_driver = { + "pmu_dmc620", + pmu_dmc620_acpi_methods, + sizeof(struct pmu_dmc620_softc), +}; +static devclass_t pmu_dmc620_acpi_devclass; + +DRIVER_MODULE(pmu_dmc620, acpi, pmu_dmc620_acpi_driver, + pmu_dmc620_acpi_devclass, 0, 0); +/* Reverse dependency. hwpmc needs DMC-620 on ARM64. */ +MODULE_DEPEND(pmc, pmu_dmc620, 1, 1, 1); +MODULE_VERSION(pmu_dmc620, 1); diff --git a/sys/modules/hwpmc/Makefile b/sys/modules/hwpmc/Makefile --- a/sys/modules/hwpmc/Makefile +++ b/sys/modules/hwpmc/Makefile @@ -3,6 +3,7 @@ # .PATH: ${SRCTOP}/sys/dev/hwpmc +.PATH: ${SRCTOP}/sys/arm64/arm64 KMOD= hwpmc @@ -10,7 +11,8 @@ SRCS+= vnode_if.h .if ${MACHINE_CPUARCH} == "aarch64" -SRCS+= hwpmc_arm64.c hwpmc_arm64_md.c +SRCS+= hwpmc_arm64.c hwpmc_arm64_md.c +SRCS+= hwpmc_dmc620.c pmu_dmc620.c .endif .if ${MACHINE_CPUARCH} == "amd64"