Index: head/sys/dev/hwpmc/hwpmc_amd.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_amd.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_amd.c (revision 282658)
@@ -1,1033 +1,1033 @@
/*-
* Copyright (c) 2003-2008 Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
/* Support for the AMD K7 and later processors */
#include <sys/param.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#ifdef HWPMC_DEBUG
enum pmc_class amd_pmc_class;
#endif
/* AMD K7 & K8 PMCs */
struct amd_descr {
struct pmc_descr pm_descr; /* "base class" */
uint32_t pm_evsel; /* address of EVSEL register */
uint32_t pm_perfctr; /* address of PERFCTR register */
};
static struct amd_descr amd_pmcdesc[AMD_NPMCS] =
{
{
.pm_descr =
{
.pd_name = "",
.pd_class = -1,
.pd_caps = AMD_PMC_CAPS,
.pd_width = 48
},
.pm_evsel = AMD_PMC_EVSEL_0,
.pm_perfctr = AMD_PMC_PERFCTR_0
},
{
.pm_descr =
{
.pd_name = "",
.pd_class = -1,
.pd_caps = AMD_PMC_CAPS,
.pd_width = 48
},
.pm_evsel = AMD_PMC_EVSEL_1,
.pm_perfctr = AMD_PMC_PERFCTR_1
},
{
.pm_descr =
{
.pd_name = "",
.pd_class = -1,
.pd_caps = AMD_PMC_CAPS,
.pd_width = 48
},
.pm_evsel = AMD_PMC_EVSEL_2,
.pm_perfctr = AMD_PMC_PERFCTR_2
},
{
.pm_descr =
{
.pd_name = "",
.pd_class = -1,
.pd_caps = AMD_PMC_CAPS,
.pd_width = 48
},
.pm_evsel = AMD_PMC_EVSEL_3,
.pm_perfctr = AMD_PMC_PERFCTR_3
}
};
struct amd_event_code_map {
enum pmc_event pe_ev; /* enum value */
uint8_t pe_code; /* encoded event mask */
uint8_t pe_mask; /* bits allowed in unit mask */
};
const struct amd_event_code_map amd_event_codes[] = {
#if defined(__i386__) /* 32 bit Athlon (K7) only */
{ PMC_EV_K7_DC_ACCESSES, 0x40, 0 },
{ PMC_EV_K7_DC_MISSES, 0x41, 0 },
{ PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI },
{ PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI },
{ PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI },
{ PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 },
{ PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 },
{ PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 },
{ PMC_EV_K7_IC_FETCHES, 0x80, 0 },
{ PMC_EV_K7_IC_MISSES, 0x81, 0 },
{ PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 },
{ PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 },
{ PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 },
{ PMC_EV_K7_RETIRED_OPS, 0xC1, 0 },
{ PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 },
{ PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 },
{ PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 },
{ PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 },
{ PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 },
{ PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 },
{ PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 },
{ PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 },
{ PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 },
#endif
{ PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F },
{ PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 },
{ PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 },
{ PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F },
{ PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE,
0x21, 0x00 },
{ PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 },
{ PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 },
{ PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 },
{ PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 },
{ PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 },
{ PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 },
{ PMC_EV_K8_DC_ACCESS, 0x40, 0x00 },
{ PMC_EV_K8_DC_MISS, 0x41, 0x00 },
{ PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F },
{ PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F },
{ PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F },
{ PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 },
{ PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 },
{ PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 },
{ PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 },
{ PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 },
{ PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 },
{ PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 },
{ PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 },
{ PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 },
{ PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F },
{ PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 },
{ PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 },
{ PMC_EV_K8_IC_FETCH, 0x80, 0x00 },
{ PMC_EV_K8_IC_MISS, 0x81, 0x00 },
{ PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 },
{ PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 },
{ PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 },
{ PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 },
{ PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 },
{ PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 },
{ PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 },
{ PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 },
{ PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 },
{ PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 },
{ PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 },
{ PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 },
{ PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 },
{ PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 },
{ PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 },
{ PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 },
{ PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 },
{ PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 },
{ PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE,
0xCA, 0x00 },
{ PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F },
{ PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS,
0xCC, 0x07 },
{ PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 },
{ PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 },
{ PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 },
{ PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE,
0xD2, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL,
0xD5, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL,
0xD6, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET,
0xD9, 0x00 },
{ PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING,
0xDA, 0x00 },
{ PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F },
{ PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 },
{ PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 },
{ PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 },
{ PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 },
{ PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 },
{ PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 },
{ PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED,
0xE2, 0x00 },
{ PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 },
{ PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F },
{ PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F },
{ PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F },
{ PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F },
{ PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F },
{ PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F }
};
const int amd_event_codes_size =
sizeof(amd_event_codes) / sizeof(amd_event_codes[0]);
/*
* Per-processor information
*/
struct amd_cpu {
struct pmc_hw pc_amdpmcs[AMD_NPMCS];
};
static struct amd_cpu **amd_pcpu;
/*
* read a pmc register
*/
static int
amd_read_pmc(int cpu, int ri, pmc_value_t *v)
{
enum pmc_mode mode;
const struct amd_descr *pd;
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
KASSERT(amd_pcpu[cpu],
("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu));
pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
pd = &amd_pmcdesc[ri];
KASSERT(pm != NULL,
("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri));
mode = PMC_TO_MODE(pm);
- PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class);
+ PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class);
#ifdef HWPMC_DEBUG
KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
("[amd,%d] unknown PMC class (%d)", __LINE__,
pd->pm_descr.pd_class));
#endif
tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */
- PMCDBG(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp);
+ PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(mode)) {
/* Sign extend 48 bit value to 64 bits. */
tmp = (pmc_value_t) (((int64_t) tmp << 16) >> 16);
tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
}
*v = tmp;
- PMCDBG(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v);
+ PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v);
return 0;
}
/*
* Write a PMC MSR.
*/
static int
amd_write_pmc(int cpu, int ri, pmc_value_t v)
{
const struct amd_descr *pd;
enum pmc_mode mode;
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
pd = &amd_pmcdesc[ri];
KASSERT(pm != NULL,
("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__,
cpu, ri));
mode = PMC_TO_MODE(pm);
#ifdef HWPMC_DEBUG
KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
("[amd,%d] unknown PMC class (%d)", __LINE__,
pd->pm_descr.pd_class));
#endif
/* use 2's complement of the count for sampling mode PMCs */
if (PMC_IS_SAMPLING_MODE(mode))
v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
- PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
/* write the PMC value */
wrmsr(pd->pm_perfctr, v);
return 0;
}
/*
* configure hardware pmc according to the configuration recorded in
* pmc 'pm'.
*/
static int
amd_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
amd_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
return 0;
}
/*
* Machine dependent actions taken during the context switch in of a
* thread.
*/
static int
amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc;
- PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
+ PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
(pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
/* enable the RDPMC instruction if needed */
if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
load_cr4(rcr4() | CR4_PCE);
return 0;
}
/*
* Machine dependent actions taken during the context switch out of a
* thread.
*/
static int
amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc;
(void) pp; /* can be NULL */
- PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ?
+ PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ?
(pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0);
/* always turn off the RDPMC instruction */
load_cr4(rcr4() & ~CR4_PCE);
return 0;
}
/*
* Check if a given allocation is feasible.
*/
static int
amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int i;
uint32_t allowed_unitmask, caps, config, unitmask;
enum pmc_event pe;
const struct pmc_descr *pd;
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row index %d", __LINE__, ri));
pd = &amd_pmcdesc[ri].pm_descr;
/* check class match */
if (pd->pd_class != a->pm_class)
return EINVAL;
caps = pm->pm_caps;
- PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps);
+ PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps);
if ((pd->pd_caps & caps) != caps)
return EPERM;
pe = a->pm_ev;
/* map ev to the correct event mask code */
config = allowed_unitmask = 0;
for (i = 0; i < amd_event_codes_size; i++)
if (amd_event_codes[i].pe_ev == pe) {
config =
AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code);
allowed_unitmask =
AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask);
break;
}
if (i == amd_event_codes_size)
return EINVAL;
unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK;
if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
return EINVAL;
if (unitmask && (caps & PMC_CAP_QUALIFIER))
config |= unitmask;
if (caps & PMC_CAP_THRESHOLD)
config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK;
/* set at least one of the 'usr' or 'os' caps */
if (caps & PMC_CAP_USER)
config |= AMD_PMC_USR;
if (caps & PMC_CAP_SYSTEM)
config |= AMD_PMC_OS;
if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
config |= (AMD_PMC_USR|AMD_PMC_OS);
if (caps & PMC_CAP_EDGE)
config |= AMD_PMC_EDGE;
if (caps & PMC_CAP_INVERT)
config |= AMD_PMC_INVERT;
if (caps & PMC_CAP_INTERRUPT)
config |= AMD_PMC_INT;
pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */
- PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
/*
* Release machine dependent state associated with a PMC. This is a
* no-op on this architecture.
*
*/
/* ARGSUSED0 */
static int
amd_release_pmc(int cpu, int ri, struct pmc *pmc)
{
#ifdef HWPMC_DEBUG
const struct amd_descr *pd;
#endif
struct pmc_hw *phw;
(void) pmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
KASSERT(phw->phw_pmc == NULL,
("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
#ifdef HWPMC_DEBUG
pd = &amd_pmcdesc[ri];
if (pd->pm_descr.pd_class == amd_pmc_class)
KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
("[amd,%d] PMC %d released while active", __LINE__, ri));
#endif
return 0;
}
/*
* start a PMC.
*/
static int
amd_start_pmc(int cpu, int ri)
{
uint32_t config;
struct pmc *pm;
struct pmc_hw *phw;
const struct amd_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
pm = phw->phw_pmc;
pd = &amd_pmcdesc[ri];
KASSERT(pm != NULL,
("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__,
cpu, ri));
- PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri);
KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__,
ri, cpu, pd->pm_descr.pd_name));
/* turn on the PMC ENABLE bit */
config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
- PMCDBG(MDP,STA,2,"amd-start config=0x%x", config);
+ PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config);
wrmsr(pd->pm_evsel, config);
return 0;
}
/*
* Stop a PMC.
*/
static int
amd_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct pmc_hw *phw;
const struct amd_descr *pd;
uint64_t config;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
pm = phw->phw_pmc;
pd = &amd_pmcdesc[ri];
KASSERT(pm != NULL,
("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__,
cpu, ri));
KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel),
("[amd,%d] PMC%d, CPU%d \"%s\" already stopped",
__LINE__, ri, cpu, pd->pm_descr.pd_name));
- PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri);
+ PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri);
/* turn off the PMC ENABLE bit */
config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE;
wrmsr(pd->pm_evsel, config);
return 0;
}
/*
* Interrupt handler. This function needs to return '1' if the
* interrupt was this CPU's PMCs or '0' otherwise. It is not allowed
* to sleep or do anything a 'fast' interrupt handler is not allowed
* to do.
*/
static int
amd_intr(int cpu, struct trapframe *tf)
{
int i, error, retval;
uint32_t config, evsel, perfctr;
struct pmc *pm;
struct amd_cpu *pac;
pmc_value_t v;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] out of range CPU %d", __LINE__, cpu));
- PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = amd_pcpu[cpu];
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*
* If multiple PMCs interrupt at the same time, the AMD64
* processor appears to deliver as many NMIs as there are
* outstanding PMC interrupts. So we process only one NMI
* interrupt at a time.
*/
for (i = 0; retval == 0 && i < AMD_NPMCS; i++) {
if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!AMD_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the PMC, reload count. */
evsel = AMD_PMC_EVSEL_0 + i;
perfctr = AMD_PMC_PERFCTR_0 + i;
v = pm->pm_sc.pm_reloadcount;
config = rdmsr(evsel);
KASSERT((config & ~AMD_PMC_ENABLE) ==
(pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE),
("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__,
config, pm->pm_md.pm_amd.pm_amd_evsel));
wrmsr(evsel, config & ~AMD_PMC_ENABLE);
wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
/* Restart the counter if logging succeeded. */
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error == 0)
wrmsr(evsel, config | AMD_PMC_ENABLE);
}
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (retval);
}
/*
* describe a PMC
*/
static int
amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
size_t copied;
const struct amd_descr *pd;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] row-index %d out of range", __LINE__, ri));
phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
pd = &amd_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return error;
pi->pm_class = pd->pm_descr.pd_class;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return 0;
}
/*
* i386 specific entry points
*/
/*
* return the MSR address of the given PMC.
*/
static int
amd_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < AMD_NPMCS,
("[amd,%d] ri %d out of range", __LINE__, ri));
*msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0;
return (0);
}
/*
* processor dependent initialization.
*/
static int
amd_pcpu_init(struct pmc_mdep *md, int cpu)
{
int classindex, first_ri, n;
struct pmc_cpu *pc;
struct amd_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] insane cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu);
amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC,
M_WAITOK|M_ZERO);
/*
* Set the content of the hardware descriptors to a known
* state and initialize pointers in the MI per-cpu descriptor.
*/
pc = pmc_pcpu[cpu];
#if defined(__amd64__)
classindex = PMC_MDEP_CLASS_INDEX_K8;
#elif defined(__i386__)
classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ?
PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7;
#endif
first_ri = md->pmd_classdep[classindex].pcd_ri;
KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__));
for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
return (0);
}
/*
* processor dependent cleanup prior to the KLD
* being unloaded
*/
static int
amd_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int classindex, first_ri, i;
uint32_t evsel;
struct pmc_cpu *pc;
struct amd_cpu *pac;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] insane cpu number (%d)", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu);
/*
* First, turn off all PMCs on this CPU.
*/
for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */
evsel = rdmsr(AMD_PMC_EVSEL_0 + i);
evsel &= ~AMD_PMC_ENABLE;
wrmsr(AMD_PMC_EVSEL_0 + i, evsel);
}
/*
* Next, free up allocated space.
*/
if ((pac = amd_pcpu[cpu]) == NULL)
return (0);
amd_pcpu[cpu] = NULL;
#ifdef HWPMC_DEBUG
for (i = 0; i < AMD_NPMCS; i++) {
KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL,
("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i));
KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i),
("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i));
}
#endif
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__));
#if defined(__amd64__)
classindex = PMC_MDEP_CLASS_INDEX_K8;
#elif defined(__i386__)
classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 :
PMC_MDEP_CLASS_INDEX_K7;
#endif
first_ri = md->pmd_classdep[classindex].pcd_ri;
/*
* Reset pointers in the MI 'per-cpu' state.
*/
for (i = 0; i < AMD_NPMCS; i++) {
pc->pc_hwpmcs[i + first_ri] = NULL;
}
free(pac, M_PMC);
return (0);
}
/*
* Initialize ourselves.
*/
struct pmc_mdep *
pmc_amd_initialize(void)
{
int classindex, error, i, ncpus;
struct pmc_classdep *pcd;
enum pmc_cputype cputype;
struct pmc_mdep *pmc_mdep;
enum pmc_class class;
char *name;
/*
* The presence of hardware performance counters on the AMD
* Athlon, Duron or later processors, is _not_ indicated by
* any of the processor feature flags set by the 'CPUID'
* instruction, so we only check the 'instruction family'
* field returned by CPUID for instruction family >= 6.
*/
name = NULL;
switch (cpu_id & 0xF00) {
#if defined(__i386__)
case 0x600: /* Athlon(tm) processor */
classindex = PMC_MDEP_CLASS_INDEX_K7;
cputype = PMC_CPU_AMD_K7;
class = PMC_CLASS_K7;
name = "K7";
break;
#endif
case 0xF00: /* Athlon64/Opteron processor */
classindex = PMC_MDEP_CLASS_INDEX_K8;
cputype = PMC_CPU_AMD_K8;
class = PMC_CLASS_K8;
name = "K8";
break;
default:
(void) printf("pmc: Unknown AMD CPU.\n");
return NULL;
}
#ifdef HWPMC_DEBUG
amd_pmc_class = class;
#endif
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
*/
amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC,
M_WAITOK|M_ZERO);
/*
* These processors have two classes of PMCs: the TSC and
* programmable PMCs.
*/
pmc_mdep = pmc_mdep_alloc(2);
pmc_mdep->pmd_cputype = cputype;
ncpus = pmc_cpu_max();
/* Initialize the TSC. */
error = pmc_tsc_initialize(pmc_mdep, ncpus);
if (error)
goto error;
/* Initialize AMD K7 and K8 PMC handling. */
pcd = &pmc_mdep->pmd_classdep[classindex];
pcd->pcd_caps = AMD_PMC_CAPS;
pcd->pcd_class = class;
pcd->pcd_num = AMD_NPMCS;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 48;
/* fill in the correct pmc name and class */
for (i = 0; i < AMD_NPMCS; i++) {
(void) snprintf(amd_pmcdesc[i].pm_descr.pd_name,
sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d",
name, i);
amd_pmcdesc[i].pm_descr.pd_class = class;
}
pcd->pcd_allocate_pmc = amd_allocate_pmc;
pcd->pcd_config_pmc = amd_config_pmc;
pcd->pcd_describe = amd_describe;
pcd->pcd_get_config = amd_get_config;
pcd->pcd_get_msr = amd_get_msr;
pcd->pcd_pcpu_fini = amd_pcpu_fini;
pcd->pcd_pcpu_init = amd_pcpu_init;
pcd->pcd_read_pmc = amd_read_pmc;
pcd->pcd_release_pmc = amd_release_pmc;
pcd->pcd_start_pmc = amd_start_pmc;
pcd->pcd_stop_pmc = amd_stop_pmc;
pcd->pcd_write_pmc = amd_write_pmc;
pmc_mdep->pmd_pcpu_init = NULL;
pmc_mdep->pmd_pcpu_fini = NULL;
pmc_mdep->pmd_intr = amd_intr;
pmc_mdep->pmd_switch_in = amd_switch_in;
pmc_mdep->pmd_switch_out = amd_switch_out;
pmc_mdep->pmd_npmc += AMD_NPMCS;
- PMCDBG(MDP,INI,0,"%s","amd-initialize");
+ PMCDBG0(MDP,INI,0,"amd-initialize");
return (pmc_mdep);
error:
if (error) {
free(pmc_mdep, M_PMC);
pmc_mdep = NULL;
}
return (NULL);
}
/*
* Finalization code for AMD CPUs.
*/
void
pmc_amd_finalize(struct pmc_mdep *md)
{
#if defined(INVARIANTS)
int classindex, i, ncpus, pmcclass;
#endif
pmc_tsc_finalize(md);
KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer",
__LINE__));
#if defined(INVARIANTS)
switch (md->pmd_cputype) {
#if defined(__i386__)
case PMC_CPU_AMD_K7:
classindex = PMC_MDEP_CLASS_INDEX_K7;
pmcclass = PMC_CLASS_K7;
break;
#endif
default:
classindex = PMC_MDEP_CLASS_INDEX_K8;
pmcclass = PMC_CLASS_K8;
}
KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass,
("[amd,%d] pmc class mismatch", __LINE__));
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu",
__LINE__));
#endif
free(amd_pcpu, M_PMC);
amd_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_core.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_core.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_core.c (revision 282658)
@@ -1,2818 +1,2818 @@
/*-
* Copyright (c) 2008 Joseph Koshy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Intel Core PMCs.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/intr_machdep.h>
#if (__FreeBSD_version >= 1100000)
#include <x86/apicvar.h>
#else
#include <machine/apicvar.h>
#endif
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#define CORE_CPUID_REQUEST 0xA
#define CORE_CPUID_REQUEST_SIZE 0x4
#define CORE_CPUID_EAX 0x0
#define CORE_CPUID_EBX 0x1
#define CORE_CPUID_ECX 0x2
#define CORE_CPUID_EDX 0x3
#define IAF_PMC_CAPS \
(PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
PMC_CAP_USER | PMC_CAP_SYSTEM)
#define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30))
#define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
#define EV_IS_NOTARCH 0
#define EV_IS_ARCH_SUPP 1
#define EV_IS_ARCH_NOTSUPP -1
/*
* "Architectural" events defined by Intel. The values of these
* symbols correspond to positions in the bitmask returned by
* the CPUID.0AH instruction.
*/
enum core_arch_events {
CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5,
CORE_AE_BRANCH_MISSES_RETIRED = 6,
CORE_AE_INSTRUCTION_RETIRED = 1,
CORE_AE_LLC_MISSES = 4,
CORE_AE_LLC_REFERENCE = 3,
CORE_AE_UNHALTED_REFERENCE_CYCLES = 2,
CORE_AE_UNHALTED_CORE_CYCLES = 0
};
static enum pmc_cputype core_cputype;
struct core_cpu {
volatile uint32_t pc_resync;
volatile uint32_t pc_iafctrl; /* Fixed function control. */
volatile uint64_t pc_globalctrl; /* Global control register. */
struct pmc_hw pc_corepmcs[];
};
static struct core_cpu **core_pcpu;
static uint32_t core_architectural_events;
static uint64_t core_pmcmask;
static int core_iaf_ri; /* relative index of fixed counters */
static int core_iaf_width;
static int core_iaf_npmc;
static int core_iap_width;
static int core_iap_npmc;
static int
core_pcpu_noop(struct pmc_mdep *md, int cpu)
{
(void) md;
(void) cpu;
return (0);
}
static int
core_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct core_cpu *cc;
struct pmc_hw *phw;
int core_ri, n, npmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[iaf,%d] insane cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"core-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
if (core_cputype != PMC_CPU_INTEL_CORE)
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
M_PMC, M_WAITOK | M_ZERO);
core_pcpu[cpu] = cc;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL && cc != NULL,
("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) |
PMC_PHW_INDEX_TO_STATE(n + core_ri);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + core_ri] = phw;
}
return (0);
}
static int
core_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int core_ri, n, npmc;
struct pmc_cpu *pc;
struct core_cpu *cc;
uint64_t msr = 0;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] insane cpu number (%d)", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
if ((cc = core_pcpu[cpu]) == NULL)
return (0);
core_pcpu[cpu] = NULL;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
cpu));
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
for (n = 0; n < npmc; n++) {
msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK;
wrmsr(IAP_EVSEL0 + n, msr);
}
if (core_cputype != PMC_CPU_INTEL_CORE) {
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
wrmsr(IAF_CTRL, msr);
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
}
for (n = 0; n < npmc; n++)
pc->pc_hwpmcs[n + core_ri] = NULL;
free(cc, M_PMC);
return (0);
}
/*
* Fixed function counters.
*/
static pmc_value_t
iaf_perfctr_value_to_reload_count(pmc_value_t v)
{
v &= (1ULL << core_iaf_width) - 1;
return (1ULL << core_iaf_width) - v;
}
static pmc_value_t
iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << core_iaf_width) - rlc;
}
static int
iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
uint32_t caps, flags, validflags;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
- PMCDBG(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
+ PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
if (ri < 0 || ri > core_iaf_npmc)
return (EINVAL);
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_IAF ||
(caps & IAF_PMC_CAPS) != caps)
return (EINVAL);
ev = pm->pm_event;
if (ev < PMC_EV_IAF_FIRST || ev > PMC_EV_IAF_LAST)
return (EINVAL);
if (ev == PMC_EV_IAF_INSTR_RETIRED_ANY && ri != 0)
return (EINVAL);
if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_CORE && ri != 1)
return (EINVAL);
if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_REF && ri != 2)
return (EINVAL);
flags = a->pm_md.pm_iaf.pm_iaf_flags;
validflags = IAF_MASK;
if (core_cputype != PMC_CPU_INTEL_ATOM &&
core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT)
validflags &= ~IAF_ANY;
if ((flags & ~validflags) != 0)
return (EINVAL);
if (caps & PMC_CAP_INTERRUPT)
flags |= IAF_PMI;
if (caps & PMC_CAP_SYSTEM)
flags |= IAF_OS;
if (caps & PMC_CAP_USER)
flags |= IAF_USR;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
flags |= (IAF_OS | IAF_USR);
pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
- PMCDBG(MDP,ALL,2, "iaf-allocate config=0x%jx",
+ PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
(uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
return (0);
}
static int
iaf_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
cpu));
core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
return (0);
}
static int
iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char iaf_name[PMC_NAME_MAX];
phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
(void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri);
if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return (error);
pi->pm_class = PMC_CLASS_IAF;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
iaf_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
return (0);
}
static int
iaf_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[iaf,%d] ri %d out of range", __LINE__, ri));
*msr = IAF_RI_TO_MSR(ri);
return (0);
}
static int
iaf_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
ri, ri + core_iaf_ri));
tmp = rdpmc(IAF_RI_TO_MSR(ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = iaf_perfctr_value_to_reload_count(tmp);
else
*v = tmp;
- PMCDBG(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
+ PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
IAF_RI_TO_MSR(ri), *v);
return (0);
}
static int
iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
{
- PMCDBG(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
+ PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
("[core,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
iaf_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct core_cpu *iafc;
uint64_t msr = 0;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
iafc = core_pcpu[cpu];
pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
do {
iafc->pc_resync = 0;
iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
IAF_GLOBAL_CTRL_MASK));
} while (iafc->pc_resync != 0);
- PMCDBG(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
+ PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
return (0);
}
static int
iaf_stop_pmc(int cpu, int ri)
{
uint32_t fc;
struct core_cpu *iafc;
uint64_t msr = 0;
- PMCDBG(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
iafc = core_pcpu[cpu];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
fc = (IAF_MASK << (ri * 4));
if (core_cputype != PMC_CPU_INTEL_ATOM &&
core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT)
fc &= ~IAF_ANY;
iafc->pc_iafctrl &= ~fc;
- PMCDBG(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
+ PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
do {
iafc->pc_resync = 0;
iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
IAF_GLOBAL_CTRL_MASK));
} while (iafc->pc_resync != 0);
- PMCDBG(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
+ PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
return (0);
}
static int
iaf_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct core_cpu *cc;
struct pmc *pm;
uint64_t msr;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
cc = core_pcpu[cpu];
pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = iaf_reload_count_to_perfctr_value(v);
/* Turn off fixed counters */
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
wrmsr(IAF_CTRL, msr);
wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
/* Turn on fixed counters */
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
- PMCDBG(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
+ PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
"pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
(uintmax_t) rdmsr(IAF_CTRL),
(uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
return (0);
}
static void
iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "iaf-initialize");
+ PMCDBG0(MDP,INI,1, "iaf-initialize");
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
pcd->pcd_caps = IAF_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_IAF;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = iaf_allocate_pmc;
pcd->pcd_config_pmc = iaf_config_pmc;
pcd->pcd_describe = iaf_describe;
pcd->pcd_get_config = iaf_get_config;
pcd->pcd_get_msr = iaf_get_msr;
pcd->pcd_pcpu_fini = core_pcpu_noop;
pcd->pcd_pcpu_init = core_pcpu_noop;
pcd->pcd_read_pmc = iaf_read_pmc;
pcd->pcd_release_pmc = iaf_release_pmc;
pcd->pcd_start_pmc = iaf_start_pmc;
pcd->pcd_stop_pmc = iaf_stop_pmc;
pcd->pcd_write_pmc = iaf_write_pmc;
md->pmd_npmc += npmc;
}
/*
* Intel programmable PMCs.
*/
/*
* Event descriptor tables.
*
* For each event id, we track:
*
* 1. The CPUs that the event is valid for.
*
* 2. If the event uses a fixed UMASK, the value of the umask field.
* If the event doesn't use a fixed UMASK, a mask of legal bits
* to check against.
*/
struct iap_event_descr {
enum pmc_event iap_ev;
unsigned char iap_evcode;
unsigned char iap_umask;
unsigned int iap_flags;
};
#define IAP_F_CC (1 << 0) /* CPU: Core */
#define IAP_F_CC2 (1 << 1) /* CPU: Core2 family */
#define IAP_F_CC2E (1 << 2) /* CPU: Core2 Extreme only */
#define IAP_F_CA (1 << 3) /* CPU: Atom */
#define IAP_F_I7 (1 << 4) /* CPU: Core i7 */
#define IAP_F_I7O (1 << 4) /* CPU: Core i7 (old) */
#define IAP_F_WM (1 << 5) /* CPU: Westmere */
#define IAP_F_SB (1 << 6) /* CPU: Sandy Bridge */
#define IAP_F_IB (1 << 7) /* CPU: Ivy Bridge */
#define IAP_F_SBX (1 << 8) /* CPU: Sandy Bridge Xeon */
#define IAP_F_IBX (1 << 9) /* CPU: Ivy Bridge Xeon */
#define IAP_F_HW (1 << 10) /* CPU: Haswell */
#define IAP_F_CAS (1 << 11) /* CPU: Atom Silvermont */
#define IAP_F_HWX (1 << 12) /* CPU: Haswell Xeon */
#define IAP_F_BW (1 << 13) /* CPU: Broadwell */
#define IAP_F_FM (1 << 14) /* Fixed mask */
#define IAP_F_ALLCPUSCORE2 \
(IAP_F_CC | IAP_F_CC2 | IAP_F_CC2E | IAP_F_CA)
/* Sub fields of UMASK that this event supports. */
#define IAP_M_CORE (1 << 0) /* Core specificity */
#define IAP_M_AGENT (1 << 1) /* Agent specificity */
#define IAP_M_PREFETCH (1 << 2) /* Prefetch */
#define IAP_M_MESI (1 << 3) /* MESI */
#define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */
#define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */
#define IAP_M_TRANSITION (1 << 6) /* Transition */
#define IAP_F_CORE (0x3 << 14) /* Core specificity */
#define IAP_F_AGENT (0x1 << 13) /* Agent specificity */
#define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */
#define IAP_F_MESI (0xF << 8) /* MESI */
#define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */
#define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */
#define IAP_F_TRANSITION (0x1 << 12) /* Transition */
#define IAP_PREFETCH_RESERVED (0x2 << 12)
#define IAP_CORE_THIS (0x1 << 14)
#define IAP_CORE_ALL (0x3 << 14)
#define IAP_F_CMASK 0xFF000000
static struct iap_event_descr iap_events[] = {
#undef IAPDESCR
#define IAPDESCR(N,EV,UM,FLAGS) { \
.iap_ev = PMC_EV_IAP_EVENT_##N, \
.iap_evcode = (EV), \
.iap_umask = (UM), \
.iap_flags = (FLAGS) \
}
IAPDESCR(02H_01H, 0x02, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(02H_81H, 0x02, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(03H_00H, 0x03, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(03H_01H, 0x03, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
IAP_F_SBX | IAP_F_CAS),
IAPDESCR(03H_02H, 0x03, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(03H_04H, 0x03, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O |
IAP_F_CAS),
IAPDESCR(03H_08H, 0x03, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB |
IAP_F_SBX | IAP_F_CAS | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(03H_10H, 0x03, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB |
IAP_F_SBX | IAP_F_CAS),
IAPDESCR(03H_20H, 0x03, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
IAPDESCR(03H_40H, 0x03, 0x40, IAP_F_CAS),
IAPDESCR(03H_80H, 0x03, 0x80, IAP_F_CAS),
IAPDESCR(04H_00H, 0x04, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CAS),
IAPDESCR(04H_01H, 0x04, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O |
IAP_F_CAS),
IAPDESCR(04H_02H, 0x04, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
IAPDESCR(04H_04H, 0x04, 0x04, IAP_F_CAS),
IAPDESCR(04H_07H, 0x04, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(04H_08H, 0x04, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
IAPDESCR(04H_10H, 0x04, 0x10, IAP_F_CAS),
IAPDESCR(04H_20H, 0x04, 0x20, IAP_F_CAS),
IAPDESCR(04H_40H, 0x04, 0x40, IAP_F_CAS),
IAPDESCR(04H_80H, 0x04, 0x80, IAP_F_CAS),
IAPDESCR(05H_00H, 0x05, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(05H_01H, 0x05, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(05H_02H, 0x05, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(05H_03H, 0x05, 0x03, IAP_F_FM | IAP_F_I7O | IAP_F_CAS),
IAPDESCR(06H_00H, 0x06, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2 |
IAP_F_CC2E | IAP_F_CA),
IAPDESCR(06H_01H, 0x06, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(06H_02H, 0x06, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(06H_04H, 0x06, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(06H_08H, 0x06, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(06H_0FH, 0x06, 0x0F, IAP_F_FM | IAP_F_I7O),
IAPDESCR(07H_00H, 0x07, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
IAPDESCR(07H_01H, 0x07, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
IAP_F_HW | IAP_F_HWX),
IAPDESCR(07H_02H, 0x07, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(07H_03H, 0x07, 0x03, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(07H_06H, 0x07, 0x06, IAP_F_FM | IAP_F_CA),
IAPDESCR(07H_08H, 0x07, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_SB |
IAP_F_SBX),
IAPDESCR(08H_01H, 0x08, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_02H, 0x08, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_04H, 0x08, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_05H, 0x08, 0x05, IAP_F_FM | IAP_F_CA),
IAPDESCR(08H_06H, 0x08, 0x06, IAP_F_FM | IAP_F_CA),
IAPDESCR(08H_07H, 0x08, 0x07, IAP_F_FM | IAP_F_CA),
IAPDESCR(08H_08H, 0x08, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(08H_09H, 0x08, 0x09, IAP_F_FM | IAP_F_CA),
IAPDESCR(08H_0EH, 0x08, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_10H, 0x08, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(08H_40H, 0x08, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_60H, 0x08, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_80H, 0x08, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
IAPDESCR(08H_81H, 0x08, 0x81, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
IAPDESCR(08H_82H, 0x08, 0x82, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
IAPDESCR(08H_84H, 0x08, 0x84, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
IAPDESCR(08H_88H, 0x08, 0x88, IAP_F_IB | IAP_F_IBX),
IAPDESCR(09H_01H, 0x09, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
IAPDESCR(09H_02H, 0x09, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
IAPDESCR(09H_04H, 0x09, 0x04, IAP_F_FM | IAP_F_I7O),
IAPDESCR(09H_08H, 0x09, 0x08, IAP_F_FM | IAP_F_I7O),
IAPDESCR(0BH_01H, 0x0B, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0BH_02H, 0x0B, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0BH_10H, 0x0B, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0CH_01H, 0x0C, 0x01, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 |
IAP_F_WM),
IAPDESCR(0CH_02H, 0x0C, 0x02, IAP_F_FM | IAP_F_CC2),
IAPDESCR(0CH_03H, 0x0C, 0x03, IAP_F_FM | IAP_F_CA),
IAPDESCR(0DH_03H, 0x0D, 0x03, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_HW |
IAP_F_IB | IAP_F_IBX | IAP_F_HWX),
IAPDESCR(0DH_40H, 0x0D, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(0EH_01H, 0x0E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(0EH_02H, 0x0E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(0FH_01H, 0x0F, 0x01, IAP_F_FM | IAP_F_I7),
IAPDESCR(0FH_02H, 0x0F, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0FH_08H, 0x0F, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0FH_10H, 0x0F, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0FH_20H, 0x0F, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(0FH_80H, 0x0F, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(10H_00H, 0x10, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(10H_01H, 0x10, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX ),
IAPDESCR(10H_02H, 0x10, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(10H_04H, 0x10, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(10H_08H, 0x10, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(10H_10H, 0x10, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(10H_20H, 0x10, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(10H_40H, 0x10, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(10H_80H, 0x10, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(10H_81H, 0x10, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(11H_00H, 0x11, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
IAPDESCR(11H_01H, 0x11, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_SB |
IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(11H_02H, 0x11, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
IAPDESCR(11H_81H, 0x11, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(12H_00H, 0x12, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(12H_01H, 0x12, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_02H, 0x12, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_04H, 0x12, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_08H, 0x12, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_10H, 0x12, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_20H, 0x12, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_40H, 0x12, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(12H_81H, 0x12, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(13H_00H, 0x13, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(13H_01H, 0x13, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM),
IAPDESCR(13H_02H, 0x13, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(13H_04H, 0x13, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(13H_07H, 0x13, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(13H_81H, 0x13, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(14H_00H, 0x14, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
IAPDESCR(14H_01H, 0x14, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(14H_02H, 0x14, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(17H_01H, 0x17, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_SBX),
IAPDESCR(18H_00H, 0x18, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(18H_01H, 0x18, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(19H_00H, 0x19, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(19H_01H, 0x19, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(19H_02H, 0x19, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(1DH_01H, 0x1D, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(1DH_02H, 0x1D, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(1DH_04H, 0x1D, 0x04, IAP_F_FM | IAP_F_I7O),
IAPDESCR(1EH_01H, 0x1E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(20H_01H, 0x20, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(21H, 0x21, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(22H, 0x22, IAP_M_CORE, IAP_F_CC2),
IAPDESCR(23H, 0x23, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(24H, 0x24, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
IAPDESCR(24H_01H, 0x24, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_02H, 0x24, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(24H_03H, 0x24, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_04H, 0x24, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_08H, 0x24, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_0CH, 0x24, 0x0C, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_10H, 0x24, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_20H, 0x24, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_21H, 0x24, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_22H, 0x24, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_24H, 0x24, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_27H, 0x24, 0x27, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_30H, 0x24, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_40H, 0x24, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_41H, 0x24, 0x41, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_42H, 0x24, 0x42, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_44H, 0x24, 0x44, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_50H, 0x24, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_80H, 0x24, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_C0H, 0x24, 0xC0, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(24H_E1H, 0x24, 0xE1, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_E2H, 0x24, 0xE2, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_E4H, 0x24, 0xE4, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_E7H, 0x24, 0xE7, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_AAH, 0x24, 0xAA, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(24H_F8H, 0x24, 0xF8, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_3FH, 0x24, 0x3F, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(25H, 0x25, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(26H, 0x26, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
IAPDESCR(26H_01H, 0x26, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_02H, 0x26, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_04H, 0x26, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_08H, 0x26, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_0FH, 0x26, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_10H, 0x26, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_20H, 0x26, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_40H, 0x26, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_80H, 0x26, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_F0H, 0x26, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(26H_FFH, 0x26, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H, 0x27, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
IAPDESCR(27H_01H, 0x27, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(27H_02H, 0x27, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_04H, 0x27, 0x04, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
IAP_F_SBX),
IAPDESCR(27H_08H, 0x27, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(27H_0EH, 0x27, 0x0E, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_0FH, 0x27, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(27H_10H, 0x27, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_20H, 0x27, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_40H, 0x27, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_50H, 0x27, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(27H_80H, 0x27, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_E0H, 0x27, 0xE0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(27H_F0H, 0x27, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(28H, 0x28, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2),
IAPDESCR(28H_01H, 0x28, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(28H_02H, 0x28, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SBX),
IAPDESCR(28H_04H, 0x28, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(28H_08H, 0x28, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(28H_0FH, 0x28, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI, IAP_F_CC),
IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
IAP_F_CA | IAP_F_CC2),
IAPDESCR(2AH, 0x2A, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2),
IAPDESCR(2BH, 0x2B, IAP_M_CORE | IAP_M_MESI, IAP_F_CA | IAP_F_CC2),
IAPDESCR(2EH, 0x2E, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
IAP_F_ALLCPUSCORE2),
IAPDESCR(2EH_01H, 0x2E, 0x01, IAP_F_FM | IAP_F_WM),
IAPDESCR(2EH_02H, 0x2E, 0x02, IAP_F_FM | IAP_F_WM),
IAPDESCR(2EH_41H, 0x2E, 0x41, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(2EH_4FH, 0x2E, 0x4F, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(30H, 0x30, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
IAP_F_ALLCPUSCORE2),
IAPDESCR(30H_00H, 0x30, 0x00, IAP_F_CAS),
IAPDESCR(31H_00H, 0x31, 0x00, IAP_F_CAS),
IAPDESCR(32H, 0x32, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_CC),
IAPDESCR(32H, 0x32, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(3AH, 0x3A, IAP_M_TRANSITION, IAP_F_CC),
IAPDESCR(3AH_00H, 0x3A, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(3BH_C0H, 0x3B, 0xC0, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(3CH_00H, 0x3C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(3CH_01H, 0x3C, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(3CH_02H, 0x3C, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(3DH_01H, 0x3D, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(40H, 0x40, IAP_M_MESI, IAP_F_CC | IAP_F_CC2),
IAPDESCR(40H_01H, 0x40, 0x01, IAP_F_FM | IAP_F_I7),
IAPDESCR(40H_02H, 0x40, 0x02, IAP_F_FM | IAP_F_I7),
IAPDESCR(40H_04H, 0x40, 0x04, IAP_F_FM | IAP_F_I7),
IAPDESCR(40H_08H, 0x40, 0x08, IAP_F_FM | IAP_F_I7),
IAPDESCR(40H_0FH, 0x40, 0x0F, IAP_F_FM | IAP_F_I7),
IAPDESCR(40H_21H, 0x40, 0x21, IAP_F_FM | IAP_F_CA),
IAPDESCR(41H, 0x41, IAP_M_MESI, IAP_F_CC | IAP_F_CC2),
IAPDESCR(41H_01H, 0x41, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(41H_02H, 0x41, 0x02, IAP_F_FM | IAP_F_I7),
IAPDESCR(41H_04H, 0x41, 0x04, IAP_F_FM | IAP_F_I7),
IAPDESCR(41H_08H, 0x41, 0x08, IAP_F_FM | IAP_F_I7),
IAPDESCR(41H_0FH, 0x41, 0x0F, IAP_F_FM | IAP_F_I7O),
IAPDESCR(41H_22H, 0x41, 0x22, IAP_F_FM | IAP_F_CA),
IAPDESCR(42H, 0x42, IAP_M_MESI, IAP_F_ALLCPUSCORE2),
IAPDESCR(42H_01H, 0x42, 0x01, IAP_F_FM | IAP_F_I7),
IAPDESCR(42H_02H, 0x42, 0x02, IAP_F_FM | IAP_F_I7),
IAPDESCR(42H_04H, 0x42, 0x04, IAP_F_FM | IAP_F_I7),
IAPDESCR(42H_08H, 0x42, 0x08, IAP_F_FM | IAP_F_I7),
IAPDESCR(42H_10H, 0x42, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(43H_01H, 0x43, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7),
IAPDESCR(43H_02H, 0x43, 0x02, IAP_F_FM | IAP_F_CA |
IAP_F_CC2 | IAP_F_I7),
IAPDESCR(44H_02H, 0x44, 0x02, IAP_F_FM | IAP_F_CC),
IAPDESCR(45H_0FH, 0x45, 0x0F, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(46H_00H, 0x46, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(47H_00H, 0x47, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(48H_00H, 0x48, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(48H_01H, 0x48, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(48H_02H, 0x48, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(49H_00H, 0x49, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(49H_01H, 0x49, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_02H, 0x49, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_04H, 0x49, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_0EH, 0x49, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_10H, 0x49, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_20H, 0x49, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_40H, 0x49, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_60H, 0x49, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7 | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(4BH_00H, 0x4B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(4BH_01H, 0x4B, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7O),
IAPDESCR(4BH_02H, 0x4B, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(4BH_03H, 0x4B, 0x03, IAP_F_FM | IAP_F_CC),
IAPDESCR(4BH_08H, 0x4B, 0x08, IAP_F_FM | IAP_F_I7O),
IAPDESCR(4CH_00H, 0x4C, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(4CH_01H, 0x4C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(4CH_02H, 0x4C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(4DH_01H, 0x4D, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(4EH_01H, 0x4E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(4EH_02H, 0x4E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(4EH_04H, 0x4E, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(4EH_10H, 0x4E, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(4FH_00H, 0x4F, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(4FH_02H, 0x4F, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(4FH_04H, 0x4F, 0x04, IAP_F_FM | IAP_F_I7O),
IAPDESCR(4FH_08H, 0x4F, 0x08, IAP_F_FM | IAP_F_I7O),
IAPDESCR(4FH_10H, 0x4F, 0x10, IAP_F_FM | IAP_F_WM),
IAPDESCR(51H_01H, 0x51, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(51H_02H, 0x51, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(51H_04H, 0x51, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(51H_08H, 0x51, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(52H_01H, 0x52, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(53H_01H, 0x53, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(59H_20H, 0x59, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(59H_40H, 0x59, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(59H_80H, 0x59, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(5BH_0CH, 0x5B, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(5BH_0FH, 0x5B, 0x0F, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(5BH_40H, 0x5B, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(5BH_4FH, 0x5B, 0x4F, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(5CH_01H, 0x5C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(5CH_02H, 0x5C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(5EH_01H, 0x5E, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(5FH_01H, 0x5F, 0x01, IAP_F_FM | IAP_F_IB ), /* IB not in manual */
IAPDESCR(5FH_04H, 0x5F, 0x04, IAP_F_IBX | IAP_F_IB),
IAPDESCR(60H, 0x60, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(60H_01H, 0x60, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(60H_02H, 0x60, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(60H_04H, 0x60, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(60H_08H, 0x60, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(61H, 0x61, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
IAPDESCR(61H_00H, 0x61, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(62H, 0x62, IAP_M_AGENT, IAP_F_ALLCPUSCORE2),
IAPDESCR(62H_00H, 0x62, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(63H, 0x63, IAP_M_AGENT | IAP_M_CORE,
IAP_F_CA | IAP_F_CC2),
IAPDESCR(63H, 0x63, IAP_M_CORE, IAP_F_CC),
IAPDESCR(63H_01H, 0x63, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(63H_02H, 0x63, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(64H, 0x64, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(64H_40H, 0x64, 0x40, IAP_F_FM | IAP_F_CC),
IAPDESCR(65H, 0x65, IAP_M_AGENT | IAP_M_CORE,
IAP_F_CA | IAP_F_CC2),
IAPDESCR(65H, 0x65, IAP_M_CORE, IAP_F_CC),
IAPDESCR(66H, 0x66, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(67H, 0x67, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(67H, 0x67, IAP_M_AGENT, IAP_F_CC),
IAPDESCR(68H, 0x68, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(69H, 0x69, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(6AH, 0x6A, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(6BH, 0x6B, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(6CH, 0x6C, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(6CH_01H, 0x6C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(6DH, 0x6D, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(6DH, 0x6D, IAP_M_CORE, IAP_F_CC),
IAPDESCR(6EH, 0x6E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(6EH, 0x6E, IAP_M_CORE, IAP_F_CC),
IAPDESCR(6FH, 0x6F, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(6FH, 0x6F, IAP_M_CORE, IAP_F_CC),
IAPDESCR(70H, 0x70, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(70H, 0x70, IAP_M_CORE, IAP_F_CC),
IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_SNOOPRESPONSE,
IAP_F_CA | IAP_F_CC2),
IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_MESI, IAP_F_CC),
IAPDESCR(78H, 0x78, IAP_M_CORE, IAP_F_CC),
IAPDESCR(78H, 0x78, IAP_M_CORE | IAP_M_SNOOPTYPE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(79H_02H, 0x79, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_04H, 0x79, 0x04, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_08H, 0x79, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_10H, 0x79, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_20H, 0x79, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_24H, 0x79, 0x24, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_30H, 0x79, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(79H_24H, 0x79, 0x24, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(7AH, 0x7A, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
IAPDESCR(7BH, 0x7B, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
IAPDESCR(7DH, 0x7D, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
IAPDESCR(7EH, 0x7E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(7EH_00H, 0x7E, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(7FH, 0x7F, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
IAPDESCR(80H_00H, 0x80, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(80H_01H, 0x80, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_CAS),
IAPDESCR(80H_02H, 0x80, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(80H_03H, 0x80, 0x03, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM | IAP_F_CAS),
IAPDESCR(80H_04H, 0x80, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_IBX),
IAPDESCR(81H_00H, 0x81, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(81H_01H, 0x81, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(81H_02H, 0x81, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(82H_01H, 0x82, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(82H_02H, 0x82, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(82H_04H, 0x82, 0x04, IAP_F_FM | IAP_F_CA),
IAPDESCR(82H_10H, 0x82, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(82H_12H, 0x82, 0x12, IAP_F_FM | IAP_F_CC2),
IAPDESCR(82H_40H, 0x82, 0x40, IAP_F_FM | IAP_F_CC2),
IAPDESCR(83H_01H, 0x83, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(83H_02H, 0x83, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(85H_00H, 0x85, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(85H_01H, 0x85, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_02H, 0x85, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_04H, 0x85, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_0EH, 0x85, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_10H, 0x85, 0x10, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_20H, 0x85, 0x20, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_40H, 0x85, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_60H, 0x85, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(85H_80H, 0x85, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
IAPDESCR(86H_00H, 0x86, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(87H_00H, 0x87, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(87H_01H, 0x87, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(87H_02H, 0x87, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(87H_04H, 0x87, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(87H_08H, 0x87, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(87H_0FH, 0x87, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_00H, 0x88, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(88H_01H, 0x88, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_02H, 0x88, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_04H, 0x88, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_07H, 0x88, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_08H, 0x88, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_10H, 0x88, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_20H, 0x88, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_30H, 0x88, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_40H, 0x88, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_7FH, 0x88, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(88H_41H, 0x88, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_81H, 0x88, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_82H, 0x88, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_84H, 0x88, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_88H, 0x88, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_90H, 0x88, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_A0H, 0x88, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(88H_FFH, 0x88, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_00H, 0x89, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(89H_01H, 0x89, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_02H, 0x89, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_04H, 0x89, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_07H, 0x89, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_08H, 0x89, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_10H, 0x89, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_20H, 0x89, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_30H, 0x89, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_40H, 0x89, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_7FH, 0x89, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(89H_41H, 0x89, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_81H, 0x89, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_82H, 0x89, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_84H, 0x89, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_88H, 0x89, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_90H, 0x89, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_A0H, 0x89, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(89H_FFH, 0x89, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(8AH_00H, 0x8A, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(8BH_00H, 0x8B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(8CH_00H, 0x8C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(8DH_00H, 0x8D, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(8EH_00H, 0x8E, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(8FH_00H, 0x8F, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(90H_00H, 0x90, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(91H_00H, 0x91, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(92H_00H, 0x92, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(93H_00H, 0x93, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(94H_00H, 0x94, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(97H_00H, 0x97, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(98H_00H, 0x98, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(9CH_01H, 0x9C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A0H_00H, 0xA0, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(A1H_01H, 0xA1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_02H, 0xA1, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_04H, 0xA1, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_08H, 0xA1, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_0CH, 0xA1, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(A1H_10H, 0xA1, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_20H, 0xA1, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_30H, 0xA1, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(A1H_40H, 0xA1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A1H_80H, 0xA1, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A2H_00H, 0xA2, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(A2H_01H, 0xA2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A2H_02H, 0xA2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(A2H_04H, 0xA2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A2H_08H, 0xA2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A2H_10H, 0xA2, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A2H_20H, 0xA2, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(A2H_40H, 0xA2, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(A2H_80H, 0xA2, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(A3H_01H, 0xA3, 0x01, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A3H_02H, 0xA3, 0x02, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A3H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB),
IAPDESCR(A3H_05H, 0xA3, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A3H_08H, 0xA3, 0x08, IAP_F_FM | IAP_F_IBX | IAP_F_HW | IAP_F_IB | IAP_F_HWX),
IAPDESCR(A3H_0CH, 0xA3, 0x08, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(A6H_01H, 0xA6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(A7H_01H, 0xA7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(A8H_01H, 0xA8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IBX |
IAP_F_IB |IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(AAH_01H, 0xAA, 0x01, IAP_F_FM | IAP_F_CC2),
IAPDESCR(AAH_02H, 0xAA, 0x02, IAP_F_FM | IAP_F_CA),
IAPDESCR(AAH_03H, 0xAA, 0x03, IAP_F_FM | IAP_F_CA),
IAPDESCR(AAH_08H, 0xAA, 0x08, IAP_F_FM | IAP_F_CC2),
IAPDESCR(ABH_01H, 0xAB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(ABH_02H, 0xAB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(ACH_02H, 0xAC, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(ACH_08H, 0xAC, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(ACH_0AH, 0xAC, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(AEH_01H, 0xAE, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B0H_00H, 0xB0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(B0H_01H, 0xB0, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B0H_02H, 0xB0, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B0H_04H, 0xB0, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B0H_08H, 0xB0, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B0H_10H, 0xB0, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
IAPDESCR(B0H_20H, 0xB0, 0x20, IAP_F_FM | IAP_F_I7O),
IAPDESCR(B0H_40H, 0xB0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B0H_80H, 0xB0, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_WM | IAP_F_I7O),
IAPDESCR(B1H_00H, 0xB1, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(B1H_01H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(B1H_02H, 0xB1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(B1H_04H, 0xB1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_08H, 0xB1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_10H, 0xB1, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_1FH, 0xB1, 0x1F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_20H, 0xB1, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_3FH, 0xB1, 0x3F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_40H, 0xB1, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B1H_80H, 0xB1, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM),
IAPDESCR(B2H_01H, 0xB2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(B3H_01H, 0xB3, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_WM | IAP_F_I7O),
IAPDESCR(B3H_02H, 0xB3, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_WM | IAP_F_I7O),
IAPDESCR(B3H_04H, 0xB3, 0x04, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_WM | IAP_F_I7O),
IAPDESCR(B3H_08H, 0xB3, 0x08, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(B3H_10H, 0xB3, 0x10, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(B3H_20H, 0xB3, 0x20, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(B3H_81H, 0xB3, 0x81, IAP_F_FM | IAP_F_CA),
IAPDESCR(B3H_82H, 0xB3, 0x82, IAP_F_FM | IAP_F_CA),
IAPDESCR(B3H_84H, 0xB3, 0x84, IAP_F_FM | IAP_F_CA),
IAPDESCR(B3H_88H, 0xB3, 0x88, IAP_F_FM | IAP_F_CA),
IAPDESCR(B3H_90H, 0xB3, 0x90, IAP_F_FM | IAP_F_CA),
IAPDESCR(B3H_A0H, 0xB3, 0xA0, IAP_F_FM | IAP_F_CA),
IAPDESCR(B4H_01H, 0xB4, 0x01, IAP_F_FM | IAP_F_WM),
IAPDESCR(B4H_02H, 0xB4, 0x02, IAP_F_FM | IAP_F_WM),
IAPDESCR(B4H_04H, 0xB4, 0x04, IAP_F_FM | IAP_F_WM),
IAPDESCR(B6H_01H, 0xB6, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(B6H_04H, 0xB6, 0x04, IAP_F_CAS),
IAPDESCR(B7H_01H, 0xB7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS |
IAP_F_HWX),
IAPDESCR(B7H_02H, 0xB7, 0x02, IAP_F_CAS),
IAPDESCR(B8H_01H, 0xB8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B8H_02H, 0xB8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(B8H_04H, 0xB8, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(BAH_01H, 0xBA, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(BAH_02H, 0xBA, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(BBH_01H, 0xBB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_11H, 0xBC, 0x11, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_12H, 0xBC, 0x12, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_14H, 0xBC, 0x14, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_18H, 0xBC, 0x18, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_21H, 0xBC, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_22H, 0xBC, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_24H, 0xBC, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BCH_28H, 0xBC, 0x28, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BDH_01H, 0xBD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BDH_20H, 0xBD, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(BFH_05H, 0xBF, 0x05, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(C0H_00H, 0xC0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C0H_01H, 0xC0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C0H_02H, 0xC0, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB),
IAPDESCR(C0H_04H, 0xC0, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C0H_08H, 0xC0, 0x08, IAP_F_FM | IAP_F_CC2E),
IAPDESCR(C1H_00H, 0xC1, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(C1H_01H, 0xC1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C1H_02H, 0xC1, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(C1H_08H, 0xC1, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C1H_10H, 0xC1, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C1H_20H, 0xC1, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(C1H_40H, 0xC1, 0x40, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C1H_80H, 0xC1, 0x80, IAP_F_IB | IAP_F_IBX),
IAPDESCR(C1H_FEH, 0xC1, 0xFE, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C2H_00H, 0xC2, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(C2H_01H, 0xC2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C2H_02H, 0xC2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C2H_04H, 0xC2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C2H_07H, 0xC2, 0x07, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C2H_08H, 0xC2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C2H_0FH, 0xC2, 0x0F, IAP_F_FM | IAP_F_CC2),
IAPDESCR(C2H_10H, 0xC2, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CAS),
IAPDESCR(C3H_00H, 0xC3, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(C3H_01H, 0xC3, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_CAS),
IAPDESCR(C3H_02H, 0xC3, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C3H_04H, 0xC3, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C3H_08H, 0xC3, 0x08, IAP_F_CAS),
IAPDESCR(C3H_10H, 0xC3, 0x10, IAP_F_FM | IAP_F_I7O),
IAPDESCR(C3H_20H, 0xC3, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_00H, 0xC4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C4H_01H, 0xC4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_02H, 0xC4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_04H, 0xC4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_08H, 0xC4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
IAP_F_HWX),
IAPDESCR(C4H_0CH, 0xC4, 0x0C, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C4H_0FH, 0xC4, 0x0F, IAP_F_FM | IAP_F_CA),
IAPDESCR(C4H_10H, 0xC4, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_20H, 0xC4, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_40H, 0xC4, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C4H_7EH, 0xC4, 0x7E, IAP_F_CAS),
IAPDESCR(C4H_BFH, 0xC4, 0xBF, IAP_F_CAS),
IAPDESCR(C4H_EBH, 0xC4, 0xEB, IAP_F_CAS),
IAPDESCR(C4H_F7H, 0xC4, 0xF7, IAP_F_CAS),
IAPDESCR(C4H_F9H, 0xC4, 0xF9, IAP_F_CAS),
IAPDESCR(C4H_FBH, 0xC4, 0xFB, IAP_F_CAS),
IAPDESCR(C4H_FDH, 0xC4, 0xFD, IAP_F_CAS),
IAPDESCR(C4H_FEH, 0xC4, 0xFE, IAP_F_CAS),
IAPDESCR(C5H_00H, 0xC5, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(C5H_01H, 0xC5, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C5H_02H, 0xC5, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(C5H_04H, 0xC5, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C5H_10H, 0xC5, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(C5H_20H, 0xC5, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(C5H_7EH, 0xC5, 0x7E, IAP_F_CAS),
IAPDESCR(C5H_BFH, 0xC5, 0xBF, IAP_F_CAS),
IAPDESCR(C5H_EBH, 0xC5, 0xEB, IAP_F_CAS),
IAPDESCR(C5H_F7H, 0xC5, 0xF7, IAP_F_CAS),
IAPDESCR(C5H_F9H, 0xC5, 0xF9, IAP_F_CAS),
IAPDESCR(C5H_FBH, 0xC5, 0xFB, IAP_F_CAS),
IAPDESCR(C5H_FDH, 0xC5, 0xFD, IAP_F_CAS),
IAPDESCR(C5H_FEH, 0xC5, 0xFE, IAP_F_CAS),
IAPDESCR(C6H_00H, 0xC6, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(C6H_01H, 0xC6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C6H_02H, 0xC6, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C7H_00H, 0xC7, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(C7H_01H, 0xC7, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C7H_02H, 0xC7, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C7H_04H, 0xC7, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C7H_08H, 0xC7, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C7H_10H, 0xC7, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(C7H_1FH, 0xC7, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(C8H_00H, 0xC8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(C8H_20H, 0xC8, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(C9H_00H, 0xC9, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(CAH_00H, 0xCA, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(CAH_01H, 0xCA, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
IAPDESCR(CAH_02H, 0xCA, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CAH_04H, 0xCA, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CAH_08H, 0xCA, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CAH_10H, 0xCA, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CAH_1EH, 0xCA, 0x1E, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CAH_20H, 0xCA, 0x20, IAP_F_CAS),
IAPDESCR(CAH_3FH, 0xCA, 0x3F, IAP_F_CAS),
IAPDESCR(CAH_50H, 0xCA, 0x50, IAP_F_CAS),
IAPDESCR(CBH_01H, 0xCB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_CAS),
IAPDESCR(CBH_02H, 0xCB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(CBH_04H, 0xCB, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(CBH_08H, 0xCB, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(CBH_10H, 0xCB, 0x10, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 |
IAP_F_WM),
IAPDESCR(CBH_1FH, 0xCB, 0x1F, IAP_F_CAS),
IAPDESCR(CBH_40H, 0xCB, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(CBH_80H, 0xCB, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(CCH_00H, 0xCC, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(CCH_01H, 0xCC, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(CCH_02H, 0xCC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(CCH_03H, 0xCC, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(CCH_20H, 0xCC, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(CDH_00H, 0xCD, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(CDH_01H, 0xCD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX),
IAPDESCR(CDH_02H, 0xCD, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX),
IAPDESCR(CEH_00H, 0xCE, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(CFH_00H, 0xCF, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
/* Sandy Bridge / Sandy Bridge Xeon - 11, 12, 21, 41, 42, 81, 82 */
IAPDESCR(D0H_00H, 0xD0, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(D0H_01H, 0xD0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(D0H_11H, 0xD0, 0x11, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D0H_12H, 0xD0, 0x12, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D0H_21H, 0xD0, 0x21, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
IAPDESCR(D0H_41H, 0xD0, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D0H_42H, 0xD0, 0x42, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D0H_81H, 0xD0, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D0H_82H, 0xD0, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_01H, 0xD1, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_02H, 0xD1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_04H, 0xD1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_08H, 0xD1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_10H, 0xD1, 0x10, IAP_F_HW | IAP_F_IB | IAP_F_IBX | IAP_F_HWX),
IAPDESCR(D1H_20H, 0xD1, 0x20, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB |
IAP_F_HW | IAP_F_HWX),
IAPDESCR(D1H_40H, 0xD1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_01H, 0xD2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_02H, 0xD2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_04H, 0xD2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_08H, 0xD2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_0FH, 0xD2, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D2H_10H, 0xD2, 0x10, IAP_F_FM | IAP_F_CC2E),
IAPDESCR(D3H_01H, 0xD3, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_SBX |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(D3H_03H, 0xD3, 0x03, IAP_F_IBX),
IAPDESCR(D3H_04H, 0xD3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX), /* Not defined for IBX */
IAPDESCR(D3H_0CH, 0xD3, 0x0C, IAP_F_IBX),
IAPDESCR(D3H_10H, 0xD3, 0x10, IAP_F_IBX ),
IAPDESCR(D3H_20H, 0xD3, 0x20, IAP_F_IBX ),
IAPDESCR(D4H_01H, 0xD4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(D4H_02H, 0xD4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(D4H_04H, 0xD4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D4H_08H, 0xD4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D4H_0FH, 0xD4, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D5H_01H, 0xD5, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
IAP_F_I7 | IAP_F_WM),
IAPDESCR(D5H_02H, 0xD5, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D5H_04H, 0xD5, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D5H_08H, 0xD5, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D5H_0FH, 0xD5, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(D7H_00H, 0xD7, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(D8H_00H, 0xD8, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(D8H_01H, 0xD8, 0x01, IAP_F_FM | IAP_F_CC),
IAPDESCR(D8H_02H, 0xD8, 0x02, IAP_F_FM | IAP_F_CC),
IAPDESCR(D8H_03H, 0xD8, 0x03, IAP_F_FM | IAP_F_CC),
IAPDESCR(D8H_04H, 0xD8, 0x04, IAP_F_FM | IAP_F_CC),
IAPDESCR(D9H_00H, 0xD9, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(D9H_01H, 0xD9, 0x01, IAP_F_FM | IAP_F_CC),
IAPDESCR(D9H_02H, 0xD9, 0x02, IAP_F_FM | IAP_F_CC),
IAPDESCR(D9H_03H, 0xD9, 0x03, IAP_F_FM | IAP_F_CC),
IAPDESCR(DAH_00H, 0xDA, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(DAH_01H, 0xDA, 0x01, IAP_F_FM | IAP_F_CC),
IAPDESCR(DAH_02H, 0xDA, 0x02, IAP_F_FM | IAP_F_CC),
IAPDESCR(DBH_00H, 0xDB, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(DBH_01H, 0xDB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(DCH_01H, 0xDC, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(DCH_02H, 0xDC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(DCH_04H, 0xDC, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(DCH_08H, 0xDC, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(DCH_10H, 0xDC, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(DCH_1FH, 0xDC, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
IAPDESCR(E0H_00H, 0xE0, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
IAPDESCR(E0H_01H, 0xE0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM),
IAPDESCR(E2H_00H, 0xE2, 0x00, IAP_F_FM | IAP_F_CC),
IAPDESCR(E4H_00H, 0xE4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(E4H_01H, 0xE4, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(E5H_01H, 0xE5, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(E6H_00H, 0xE6, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
IAPDESCR(E6H_01H, 0xE6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
IAP_F_WM | IAP_F_SBX | IAP_F_CAS),
IAPDESCR(E6H_02H, 0xE6, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(E6H_08H, 0xE6, 0x08, IAP_F_CAS),
IAPDESCR(E6H_10H, 0xE6, 0x10, IAP_F_CAS),
IAPDESCR(E6H_1FH, 0xE6, 0x1F, IAP_F_FM | IAP_F_IB |
IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(E7H_01H, 0xE7, 0x01, IAP_F_CAS),
IAPDESCR(E8H_01H, 0xE8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(E8H_02H, 0xE8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(E8H_03H, 0xE8, 0x03, IAP_F_FM | IAP_F_I7O),
IAPDESCR(ECH_01H, 0xEC, 0x01, IAP_F_FM | IAP_F_WM),
IAPDESCR(F0H_00H, 0xF0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(F0H_01H, 0xF0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_02H, 0xF0, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_04H, 0xF0, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_08H, 0xF0, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_10H, 0xF0, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_20H, 0xF0, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_40H, 0xF0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F0H_80H, 0xF0, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F1H_01H, 0xF1, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F1H_02H, 0xF1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F1H_04H, 0xF1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F1H_07H, 0xF1, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F2H_01H, 0xF2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(F2H_02H, 0xF2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(F2H_04H, 0xF2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(F2H_05H, 0xF2, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F2H_06H, 0xF2, 0x06, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
IAPDESCR(F2H_08H, 0xF2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
IAPDESCR(F2H_0AH, 0xF2, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX |
IAP_F_IBX),
IAPDESCR(F2H_0FH, 0xF2, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(F3H_01H, 0xF3, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F3H_02H, 0xF3, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F3H_04H, 0xF3, 0x04, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F3H_08H, 0xF3, 0x08, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F3H_10H, 0xF3, 0x10, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F3H_20H, 0xF3, 0x20, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F4H_01H, 0xF4, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F4H_02H, 0xF4, 0x02, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F4H_04H, 0xF4, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
IAPDESCR(F4H_08H, 0xF4, 0x08, IAP_F_FM | IAP_F_I7O),
IAPDESCR(F4H_10H, 0xF4, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
IAP_F_SB | IAP_F_SBX),
IAPDESCR(F6H_01H, 0xF6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
IAPDESCR(F7H_01H, 0xF7, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(F7H_02H, 0xF7, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(F7H_04H, 0xF7, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(F8H_00H, 0xF8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
IAPDESCR(F8H_01H, 0xF8, 0x01, IAP_F_FM | IAP_F_I7O),
IAPDESCR(FDH_01H, 0xFD, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_02H, 0xFD, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_04H, 0xFD, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_08H, 0xFD, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_10H, 0xFD, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_20H, 0xFD, 0x20, IAP_F_FM | IAP_F_WM | IAP_F_I7),
IAPDESCR(FDH_40H, 0xFD, 0x40, IAP_F_FM | IAP_F_WM | IAP_F_I7),
};
static const int niap_events = sizeof(iap_events) / sizeof(iap_events[0]);
static pmc_value_t
iap_perfctr_value_to_reload_count(pmc_value_t v)
{
v &= (1ULL << core_iap_width) - 1;
return (1ULL << core_iap_width) - v;
}
static pmc_value_t
iap_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << core_iap_width) - rlc;
}
static int
iap_pmc_has_overflowed(int ri)
{
uint64_t v;
/*
* We treat a Core (i.e., Intel architecture v1) PMC as has
* having overflowed if its MSB is zero.
*/
v = rdpmc(ri);
return ((v & (1ULL << (core_iap_width - 1))) == 0);
}
/*
* Check an event against the set of supported architectural events.
*
* If the event is not architectural EV_IS_NOTARCH is returned.
* If the event is architectural and supported on this CPU, the correct
* event+umask mapping is returned in map, and EV_IS_ARCH_SUPP is returned.
* Otherwise, the function returns EV_IS_ARCH_NOTSUPP.
*/
static int
iap_is_event_architectural(enum pmc_event pe, enum pmc_event *map)
{
enum core_arch_events ae;
switch (pe) {
case PMC_EV_IAP_ARCH_UNH_COR_CYC:
ae = CORE_AE_UNHALTED_CORE_CYCLES;
*map = PMC_EV_IAP_EVENT_3CH_00H;
break;
case PMC_EV_IAP_ARCH_INS_RET:
ae = CORE_AE_INSTRUCTION_RETIRED;
*map = PMC_EV_IAP_EVENT_C0H_00H;
break;
case PMC_EV_IAP_ARCH_UNH_REF_CYC:
ae = CORE_AE_UNHALTED_REFERENCE_CYCLES;
*map = PMC_EV_IAP_EVENT_3CH_01H;
break;
case PMC_EV_IAP_ARCH_LLC_REF:
ae = CORE_AE_LLC_REFERENCE;
*map = PMC_EV_IAP_EVENT_2EH_4FH;
break;
case PMC_EV_IAP_ARCH_LLC_MIS:
ae = CORE_AE_LLC_MISSES;
*map = PMC_EV_IAP_EVENT_2EH_41H;
break;
case PMC_EV_IAP_ARCH_BR_INS_RET:
ae = CORE_AE_BRANCH_INSTRUCTION_RETIRED;
*map = PMC_EV_IAP_EVENT_C4H_00H;
break;
case PMC_EV_IAP_ARCH_BR_MIS_RET:
ae = CORE_AE_BRANCH_MISSES_RETIRED;
*map = PMC_EV_IAP_EVENT_C5H_00H;
break;
default: /* Non architectural event. */
return (EV_IS_NOTARCH);
}
return (((core_architectural_events & (1 << ae)) == 0) ?
EV_IS_ARCH_NOTSUPP : EV_IS_ARCH_SUPP);
}
static int
iap_event_corei7_ok_on_counter(enum pmc_event pe, int ri)
{
uint32_t mask;
switch (pe) {
/*
* Events valid only on counter 0, 1.
*/
case PMC_EV_IAP_EVENT_40H_01H:
case PMC_EV_IAP_EVENT_40H_02H:
case PMC_EV_IAP_EVENT_40H_04H:
case PMC_EV_IAP_EVENT_40H_08H:
case PMC_EV_IAP_EVENT_40H_0FH:
case PMC_EV_IAP_EVENT_41H_02H:
case PMC_EV_IAP_EVENT_41H_04H:
case PMC_EV_IAP_EVENT_41H_08H:
case PMC_EV_IAP_EVENT_42H_01H:
case PMC_EV_IAP_EVENT_42H_02H:
case PMC_EV_IAP_EVENT_42H_04H:
case PMC_EV_IAP_EVENT_42H_08H:
case PMC_EV_IAP_EVENT_43H_01H:
case PMC_EV_IAP_EVENT_43H_02H:
case PMC_EV_IAP_EVENT_51H_01H:
case PMC_EV_IAP_EVENT_51H_02H:
case PMC_EV_IAP_EVENT_51H_04H:
case PMC_EV_IAP_EVENT_51H_08H:
case PMC_EV_IAP_EVENT_63H_01H:
case PMC_EV_IAP_EVENT_63H_02H:
mask = 0x3;
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
iap_event_westmere_ok_on_counter(enum pmc_event pe, int ri)
{
uint32_t mask;
switch (pe) {
/*
* Events valid only on counter 0.
*/
case PMC_EV_IAP_EVENT_60H_01H:
case PMC_EV_IAP_EVENT_60H_02H:
case PMC_EV_IAP_EVENT_60H_04H:
case PMC_EV_IAP_EVENT_60H_08H:
case PMC_EV_IAP_EVENT_B3H_01H:
case PMC_EV_IAP_EVENT_B3H_02H:
case PMC_EV_IAP_EVENT_B3H_04H:
mask = 0x1;
break;
/*
* Events valid only on counter 0, 1.
*/
case PMC_EV_IAP_EVENT_4CH_01H:
case PMC_EV_IAP_EVENT_4EH_01H:
case PMC_EV_IAP_EVENT_4EH_02H:
case PMC_EV_IAP_EVENT_4EH_04H:
case PMC_EV_IAP_EVENT_51H_01H:
case PMC_EV_IAP_EVENT_51H_02H:
case PMC_EV_IAP_EVENT_51H_04H:
case PMC_EV_IAP_EVENT_51H_08H:
case PMC_EV_IAP_EVENT_63H_01H:
case PMC_EV_IAP_EVENT_63H_02H:
mask = 0x3;
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
iap_event_sb_sbx_ib_ibx_ok_on_counter(enum pmc_event pe, int ri)
{
uint32_t mask;
switch (pe) {
/* Events valid only on counter 0. */
case PMC_EV_IAP_EVENT_B7H_01H:
mask = 0x1;
break;
/* Events valid only on counter 1. */
case PMC_EV_IAP_EVENT_C0H_01H:
mask = 0x2;
break;
/* Events valid only on counter 2. */
case PMC_EV_IAP_EVENT_48H_01H:
case PMC_EV_IAP_EVENT_A2H_02H:
case PMC_EV_IAP_EVENT_A3H_08H:
mask = 0x4;
break;
/* Events valid only on counter 3. */
case PMC_EV_IAP_EVENT_BBH_01H:
case PMC_EV_IAP_EVENT_CDH_01H:
case PMC_EV_IAP_EVENT_CDH_02H:
mask = 0x8;
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
iap_event_ok_on_counter(enum pmc_event pe, int ri)
{
uint32_t mask;
switch (pe) {
/*
* Events valid only on counter 0.
*/
case PMC_EV_IAP_EVENT_10H_00H:
case PMC_EV_IAP_EVENT_14H_00H:
case PMC_EV_IAP_EVENT_18H_00H:
case PMC_EV_IAP_EVENT_B3H_01H:
case PMC_EV_IAP_EVENT_B3H_02H:
case PMC_EV_IAP_EVENT_B3H_04H:
case PMC_EV_IAP_EVENT_C1H_00H:
case PMC_EV_IAP_EVENT_CBH_01H:
case PMC_EV_IAP_EVENT_CBH_02H:
mask = (1 << 0);
break;
/*
* Events valid only on counter 1.
*/
case PMC_EV_IAP_EVENT_11H_00H:
case PMC_EV_IAP_EVENT_12H_00H:
case PMC_EV_IAP_EVENT_13H_00H:
mask = (1 << 1);
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int arch, n, model;
enum pmc_event ev, map;
struct iap_event_descr *ie;
uint32_t c, caps, config, cpuflag, evsel, mask;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index value %d", __LINE__, ri));
/* check requested capabilities */
caps = a->pm_caps;
if ((IAP_PMC_CAPS & caps) != caps)
return (EPERM);
map = 0; /* XXX: silent GCC warning */
arch = iap_is_event_architectural(pm->pm_event, &map);
if (arch == EV_IS_ARCH_NOTSUPP)
return (EOPNOTSUPP);
else if (arch == EV_IS_ARCH_SUPP)
ev = map;
else
ev = pm->pm_event;
/*
* A small number of events are not supported in all the
* processors based on a given microarchitecture.
*/
if (ev == PMC_EV_IAP_EVENT_0FH_01H || ev == PMC_EV_IAP_EVENT_0FH_80H) {
model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4);
if (core_cputype == PMC_CPU_INTEL_COREI7 && model != 0x2E)
return (EINVAL);
}
switch (core_cputype) {
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_NEHALEM_EX:
if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
case PMC_CPU_INTEL_BROADWELL:
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
case PMC_CPU_INTEL_IVYBRIDGE:
case PMC_CPU_INTEL_IVYBRIDGE_XEON:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_HASWELL_XEON:
if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
case PMC_CPU_INTEL_WESTMERE:
case PMC_CPU_INTEL_WESTMERE_EX:
if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
default:
if (iap_event_ok_on_counter(ev, ri) == 0)
return (EINVAL);
}
/*
* Look for an event descriptor with matching CPU and event id
* fields.
*/
switch (core_cputype) {
default:
case PMC_CPU_INTEL_ATOM:
cpuflag = IAP_F_CA;
break;
case PMC_CPU_INTEL_ATOM_SILVERMONT:
cpuflag = IAP_F_CAS;
break;
case PMC_CPU_INTEL_BROADWELL:
cpuflag = IAP_F_BW;
break;
case PMC_CPU_INTEL_CORE:
cpuflag = IAP_F_CC;
break;
case PMC_CPU_INTEL_CORE2:
cpuflag = IAP_F_CC2;
break;
case PMC_CPU_INTEL_CORE2EXTREME:
cpuflag = IAP_F_CC2 | IAP_F_CC2E;
break;
case PMC_CPU_INTEL_COREI7:
cpuflag = IAP_F_I7;
break;
case PMC_CPU_INTEL_HASWELL:
cpuflag = IAP_F_HW;
break;
case PMC_CPU_INTEL_HASWELL_XEON:
cpuflag = IAP_F_HWX;
break;
case PMC_CPU_INTEL_IVYBRIDGE:
cpuflag = IAP_F_IB;
break;
case PMC_CPU_INTEL_IVYBRIDGE_XEON:
cpuflag = IAP_F_IBX;
break;
case PMC_CPU_INTEL_SANDYBRIDGE:
cpuflag = IAP_F_SB;
break;
case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
cpuflag = IAP_F_SBX;
break;
case PMC_CPU_INTEL_WESTMERE:
cpuflag = IAP_F_WM;
break;
}
for (n = 0, ie = iap_events; n < niap_events; n++, ie++)
if (ie->iap_ev == ev && ie->iap_flags & cpuflag)
break;
if (n == niap_events)
return (EINVAL);
/*
* A matching event descriptor has been found, so start
* assembling the contents of the event select register.
*/
evsel = ie->iap_evcode;
config = a->pm_md.pm_iap.pm_iap_config & ~IAP_F_CMASK;
/*
* If the event uses a fixed umask value, reject any umask
* bits set by the user.
*/
if (ie->iap_flags & IAP_F_FM) {
if (IAP_UMASK(config) != 0)
return (EINVAL);
evsel |= (ie->iap_umask << 8);
} else {
/*
* Otherwise, the UMASK value needs to be taken from
* the MD fields of the allocation request. Reject
* requests that specify reserved bits.
*/
mask = 0;
if (ie->iap_umask & IAP_M_CORE) {
if ((c = (config & IAP_F_CORE)) != IAP_CORE_ALL &&
c != IAP_CORE_THIS)
return (EINVAL);
mask |= IAP_F_CORE;
}
if (ie->iap_umask & IAP_M_AGENT)
mask |= IAP_F_AGENT;
if (ie->iap_umask & IAP_M_PREFETCH) {
if ((c = (config & IAP_F_PREFETCH)) ==
IAP_PREFETCH_RESERVED)
return (EINVAL);
mask |= IAP_F_PREFETCH;
}
if (ie->iap_umask & IAP_M_MESI)
mask |= IAP_F_MESI;
if (ie->iap_umask & IAP_M_SNOOPRESPONSE)
mask |= IAP_F_SNOOPRESPONSE;
if (ie->iap_umask & IAP_M_SNOOPTYPE)
mask |= IAP_F_SNOOPTYPE;
if (ie->iap_umask & IAP_M_TRANSITION)
mask |= IAP_F_TRANSITION;
/*
* If bits outside of the allowed set of umask bits
* are set, reject the request.
*/
if (config & ~mask)
return (EINVAL);
evsel |= (config & mask);
}
/*
* Only Atom and SandyBridge CPUs support the 'ANY' qualifier.
*/
if (core_cputype == PMC_CPU_INTEL_ATOM ||
core_cputype == PMC_CPU_INTEL_ATOM_SILVERMONT ||
core_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON)
evsel |= (config & IAP_ANY);
else if (config & IAP_ANY)
return (EINVAL);
/*
* Check offcore response configuration.
*/
if (a->pm_md.pm_iap.pm_iap_rsp != 0) {
if (ev != PMC_EV_IAP_EVENT_B7H_01H &&
ev != PMC_EV_IAP_EVENT_BBH_01H)
return (EINVAL);
if (core_cputype == PMC_CPU_INTEL_COREI7 &&
ev == PMC_EV_IAP_EVENT_BBH_01H)
return (EINVAL);
if ((core_cputype == PMC_CPU_INTEL_COREI7 ||
core_cputype == PMC_CPU_INTEL_WESTMERE ||
core_cputype == PMC_CPU_INTEL_NEHALEM_EX ||
core_cputype == PMC_CPU_INTEL_WESTMERE_EX) &&
a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_I7WM)
return (EINVAL);
else if ((core_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON ||
core_cputype == PMC_CPU_INTEL_IVYBRIDGE ||
core_cputype == PMC_CPU_INTEL_IVYBRIDGE_XEON) &&
a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_SBIB)
return (EINVAL);
pm->pm_md.pm_iap.pm_iap_rsp = a->pm_md.pm_iap.pm_iap_rsp;
}
if (caps & PMC_CAP_THRESHOLD)
evsel |= (a->pm_md.pm_iap.pm_iap_config & IAP_F_CMASK);
if (caps & PMC_CAP_USER)
evsel |= IAP_USR;
if (caps & PMC_CAP_SYSTEM)
evsel |= IAP_OS;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
evsel |= (IAP_OS | IAP_USR);
if (caps & PMC_CAP_EDGE)
evsel |= IAP_EDGE;
if (caps & PMC_CAP_INVERT)
evsel |= IAP_INV;
if (caps & PMC_CAP_INTERRUPT)
evsel |= IAP_INT;
pm->pm_md.pm_iap.pm_iap_evsel = evsel;
return (0);
}
static int
iap_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
cpu));
core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
return (0);
}
static int
iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char iap_name[PMC_NAME_MAX];
phw = &core_pcpu[cpu]->pc_corepmcs[ri];
(void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri);
if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return (error);
pi->pm_class = PMC_CLASS_IAP;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
iap_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
return (0);
}
static int
iap_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[iap,%d] ri %d out of range", __LINE__, ri));
*msr = ri;
return (0);
}
static int
iap_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = rdpmc(ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = iap_perfctr_value_to_reload_count(tmp);
else
*v = tmp & ((1ULL << core_iap_width) - 1);
- PMCDBG(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
+ PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
ri, *v);
return (0);
}
static int
iap_release_pmc(int cpu, int ri, struct pmc *pm)
{
(void) pm;
- PMCDBG(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
+ PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
== NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
iap_start_pmc(int cpu, int ri)
{
struct pmc *pm;
uint32_t evsel;
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
cc = core_pcpu[cpu];
pm = cc->pc_corepmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] starting cpu%d,ri%d with no pmc configured",
__LINE__, cpu, ri));
- PMCDBG(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
evsel = pm->pm_md.pm_iap.pm_iap_evsel;
- PMCDBG(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
+ PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
cpu, ri, IAP_EVSEL0 + ri, evsel);
/* Event specific configuration. */
switch (pm->pm_event) {
case PMC_EV_IAP_EVENT_B7H_01H:
wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
break;
case PMC_EV_IAP_EVENT_BBH_01H:
wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
break;
default:
break;
}
wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
if (core_cputype == PMC_CPU_INTEL_CORE)
return (0);
do {
cc->pc_resync = 0;
cc->pc_globalctrl |= (1ULL << ri);
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
iap_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct core_cpu *cc;
uint64_t msr;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row index %d", __LINE__, ri));
cc = core_pcpu[cpu];
pm = cc->pc_corepmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
- PMCDBG(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */
if (core_cputype == PMC_CPU_INTEL_CORE)
return (0);
msr = 0;
do {
cc->pc_resync = 0;
cc->pc_globalctrl &= ~(1ULL << ri);
msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
iap_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row index %d", __LINE__, ri));
cc = core_pcpu[cpu];
pm = cc->pc_corepmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
- PMCDBG(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
+ PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
IAP_PMC0 + ri, v);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = iap_reload_count_to_perfctr_value(v);
/*
* Write the new value to the counter. The counter will be in
* a stopped state when the pcd_write() entry point is called.
*/
wrmsr(IAP_PMC0 + ri, v & ((1ULL << core_iap_width) - 1));
return (0);
}
static void
iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
int flags)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "iap-initialize");
+ PMCDBG0(MDP,INI,1, "iap-initialize");
/* Remember the set of architectural events supported. */
core_architectural_events = ~flags;
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
pcd->pcd_caps = IAP_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_IAP;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = iap_allocate_pmc;
pcd->pcd_config_pmc = iap_config_pmc;
pcd->pcd_describe = iap_describe;
pcd->pcd_get_config = iap_get_config;
pcd->pcd_get_msr = iap_get_msr;
pcd->pcd_pcpu_fini = core_pcpu_fini;
pcd->pcd_pcpu_init = core_pcpu_init;
pcd->pcd_read_pmc = iap_read_pmc;
pcd->pcd_release_pmc = iap_release_pmc;
pcd->pcd_start_pmc = iap_start_pmc;
pcd->pcd_stop_pmc = iap_stop_pmc;
pcd->pcd_write_pmc = iap_write_pmc;
md->pmd_npmc += npmc;
}
static int
core_intr(int cpu, struct trapframe *tf)
{
pmc_value_t v;
struct pmc *pm;
struct core_cpu *cc;
int error, found_interrupt, ri;
uint64_t msr;
- PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
found_interrupt = 0;
cc = core_pcpu[cpu];
for (ri = 0; ri < core_iap_npmc; ri++) {
if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
if (!iap_pmc_has_overflowed(ri))
continue;
found_interrupt = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
v = pm->pm_sc.pm_reloadcount;
v = iaf_reload_count_to_perfctr_value(v);
/*
* Stop the counter, reload it but only restart it if
* the PMC is not stalled.
*/
msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
wrmsr(IAP_EVSEL0 + ri, msr);
wrmsr(IAP_PMC0 + ri, v);
if (error)
continue;
wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel |
IAP_EN));
}
if (found_interrupt)
lapic_reenable_pmc();
atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (found_interrupt);
}
static int
core2_intr(int cpu, struct trapframe *tf)
{
int error, found_interrupt, n;
uint64_t flag, intrstatus, intrenable, msr;
struct pmc *pm;
struct core_cpu *cc;
pmc_value_t v;
- PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
/*
* The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
* PMCs have a pending PMI interrupt. We take a 'snapshot' of
* the current set of interrupting PMCs and process these
* after stopping them.
*/
intrstatus = rdmsr(IA_GLOBAL_STATUS);
intrenable = intrstatus & core_pmcmask;
- PMCDBG(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
+ PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
(uintmax_t) intrstatus);
found_interrupt = 0;
cc = core_pcpu[cpu];
KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
cc->pc_globalctrl &= ~intrenable;
cc->pc_resync = 1; /* MSRs now potentially out of sync. */
/*
* Stop PMCs and clear overflow status bits.
*/
msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
wrmsr(IA_GLOBAL_CTRL, msr);
wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
IA_GLOBAL_STATUS_FLAG_OVFBUF |
IA_GLOBAL_STATUS_FLAG_CONDCHG);
/*
* Look for interrupts from fixed function PMCs.
*/
for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
n++, flag <<= 1) {
if ((intrstatus & flag) == 0)
continue;
found_interrupt = 1;
pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
intrenable &= ~flag;
v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
/* Reload sampling count. */
wrmsr(IAF_CTR0 + n, v);
- PMCDBG(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu,
+ PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu,
error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
}
/*
* Process interrupts from the programmable counters.
*/
for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
if ((intrstatus & flag) == 0)
continue;
found_interrupt = 1;
pm = cc->pc_corepmcs[n].phw_pmc;
if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
intrenable &= ~flag;
v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
- PMCDBG(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
+ PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
(uintmax_t) v);
/* Reload sampling count. */
wrmsr(IAP_PMC0 + n, v);
}
/*
* Reenable all non-stalled PMCs.
*/
- PMCDBG(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
+ PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
(uintmax_t) intrenable);
cc->pc_globalctrl |= intrenable;
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK);
- PMCDBG(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
+ PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
"ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
(uintmax_t) rdmsr(IA_GLOBAL_CTRL),
(uintmax_t) rdmsr(IA_GLOBAL_STATUS),
(uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
if (found_interrupt)
lapic_reenable_pmc();
atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (found_interrupt);
}
int
pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
{
int cpuid[CORE_CPUID_REQUEST_SIZE];
int ipa_version, flags, nflags;
do_cpuid(CORE_CPUID_REQUEST, cpuid);
ipa_version = (version_override > 0) ? version_override :
cpuid[CORE_CPUID_EAX] & 0xFF;
core_cputype = md->pmd_cputype;
- PMCDBG(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d",
+ PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d",
core_cputype, maxcpu, ipa_version);
if (ipa_version < 1 || ipa_version > 3 ||
(core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) {
/* Unknown PMC architecture. */
printf("hwpc_core: unknown PMC architecture: %d\n",
ipa_version);
return (EPROGMISMATCH);
}
core_pmcmask = 0;
/*
* Initialize programmable counters.
*/
core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
/*
* Initialize fixed function counters, if present.
*/
if (core_cputype != PMC_CPU_INTEL_CORE) {
core_iaf_ri = core_iap_npmc;
core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
}
- PMCDBG(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
+ PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
core_iaf_ri);
core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
M_ZERO | M_WAITOK);
/*
* Choose the appropriate interrupt handler.
*/
if (ipa_version == 1)
md->pmd_intr = core_intr;
else
md->pmd_intr = core2_intr;
md->pmd_pcpu_fini = NULL;
md->pmd_pcpu_init = NULL;
return (0);
}
void
pmc_core_finalize(struct pmc_mdep *md)
{
- PMCDBG(MDP,INI,1, "%s", "core-finalize");
+ PMCDBG0(MDP,INI,1, "core-finalize");
free(core_pcpu, M_PMC);
core_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_intel.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_intel.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_intel.c (revision 282658)
@@ -1,381 +1,381 @@
/*-
* Copyright (c) 2008 Joseph Koshy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Common code for handling Intel CPUs.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#include <machine/cputypes.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
static int
intel_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc;
- PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
+ PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS);
/* allow the RDPMC instruction if needed */
if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
load_cr4(rcr4() | CR4_PCE);
- PMCDBG(MDP,SWI,1, "cr4=0x%jx", (uintmax_t) rcr4());
+ PMCDBG1(MDP,SWI,1, "cr4=0x%jx", (uintmax_t) rcr4());
return 0;
}
static int
intel_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc;
(void) pp; /* can be NULL */
- PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%jx", pc, pp,
+ PMCDBG3(MDP,SWO,1, "pc=%p pp=%p cr4=0x%jx", pc, pp,
(uintmax_t) rcr4());
/* always turn off the RDPMC instruction */
load_cr4(rcr4() & ~CR4_PCE);
return 0;
}
struct pmc_mdep *
pmc_intel_initialize(void)
{
struct pmc_mdep *pmc_mdep;
enum pmc_cputype cputype;
int error, model, nclasses, ncpus, stepping, verov;
KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
("[intel,%d] Initializing non-intel processor", __LINE__));
- PMCDBG(MDP,INI,0, "intel-initialize cpuid=0x%x", cpu_id);
+ PMCDBG1(MDP,INI,0, "intel-initialize cpuid=0x%x", cpu_id);
cputype = -1;
nclasses = 2;
error = 0;
verov = 0;
model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4);
stepping = cpu_id & 0xF;
switch (cpu_id & 0xF00) {
#if defined(__i386__)
case 0x500: /* Pentium family processors */
cputype = PMC_CPU_INTEL_P5;
break;
#endif
case 0x600: /* Pentium Pro, Celeron, Pentium II & III */
switch (model) {
#if defined(__i386__)
case 0x1:
cputype = PMC_CPU_INTEL_P6;
break;
case 0x3: case 0x5:
cputype = PMC_CPU_INTEL_PII;
break;
case 0x6: case 0x16:
cputype = PMC_CPU_INTEL_CL;
break;
case 0x7: case 0x8: case 0xA: case 0xB:
cputype = PMC_CPU_INTEL_PIII;
break;
case 0x9: case 0xD:
cputype = PMC_CPU_INTEL_PM;
break;
#endif
case 0xE:
cputype = PMC_CPU_INTEL_CORE;
break;
case 0xF:
/* Per Intel document 315338-020. */
if (stepping == 0x7) {
cputype = PMC_CPU_INTEL_CORE;
verov = 1;
} else {
cputype = PMC_CPU_INTEL_CORE2;
nclasses = 3;
}
break;
case 0x17:
cputype = PMC_CPU_INTEL_CORE2EXTREME;
nclasses = 3;
break;
case 0x1C: /* Per Intel document 320047-002. */
cputype = PMC_CPU_INTEL_ATOM;
nclasses = 3;
break;
case 0x1A:
case 0x1E: /*
* Per Intel document 253669-032 9/2009,
* pages A-2 and A-57
*/
case 0x1F: /*
* Per Intel document 253669-032 9/2009,
* pages A-2 and A-57
*/
cputype = PMC_CPU_INTEL_COREI7;
nclasses = 5;
break;
case 0x2E:
cputype = PMC_CPU_INTEL_NEHALEM_EX;
nclasses = 3;
break;
case 0x25: /* Per Intel document 253669-033US 12/2009. */
case 0x2C: /* Per Intel document 253669-033US 12/2009. */
cputype = PMC_CPU_INTEL_WESTMERE;
nclasses = 5;
break;
case 0x2F: /* Westmere-EX, seen in wild */
cputype = PMC_CPU_INTEL_WESTMERE_EX;
nclasses = 3;
break;
case 0x2A: /* Per Intel document 253669-039US 05/2011. */
cputype = PMC_CPU_INTEL_SANDYBRIDGE;
nclasses = 5;
break;
case 0x2D: /* Per Intel document 253669-044US 08/2012. */
cputype = PMC_CPU_INTEL_SANDYBRIDGE_XEON;
nclasses = 3;
break;
case 0x3A: /* Per Intel document 253669-043US 05/2012. */
cputype = PMC_CPU_INTEL_IVYBRIDGE;
nclasses = 3;
break;
case 0x3E: /* Per Intel document 325462-045US 01/2013. */
cputype = PMC_CPU_INTEL_IVYBRIDGE_XEON;
nclasses = 3;
break;
case 0x3D:
cputype = PMC_CPU_INTEL_BROADWELL;
nclasses = 3;
break;
case 0x3F: /* Per Intel document 325462-045US 09/2014. */
case 0x46: /* Per Intel document 325462-045US 09/2014. */
/* Should 46 be XEON. probably its own? */
cputype = PMC_CPU_INTEL_HASWELL_XEON;
nclasses = 3;
break;
case 0x3C: /* Per Intel document 325462-045US 01/2013. */
case 0x45: /* Per Intel document 325462-045US 09/2014. */
cputype = PMC_CPU_INTEL_HASWELL;
nclasses = 5;
break;
case 0x4D: /* Per Intel document 330061-001 01/2014. */
cputype = PMC_CPU_INTEL_ATOM_SILVERMONT;
nclasses = 3;
break;
}
break;
#if defined(__i386__) || defined(__amd64__)
case 0xF00: /* P4 */
if (model >= 0 && model <= 6) /* known models */
cputype = PMC_CPU_INTEL_PIV;
break;
}
#endif
if ((int) cputype == -1) {
printf("pmc: Unknown Intel CPU.\n");
return (NULL);
}
/* Allocate base class and initialize machine dependent struct */
pmc_mdep = pmc_mdep_alloc(nclasses);
pmc_mdep->pmd_cputype = cputype;
pmc_mdep->pmd_switch_in = intel_switch_in;
pmc_mdep->pmd_switch_out = intel_switch_out;
ncpus = pmc_cpu_max();
error = pmc_tsc_initialize(pmc_mdep, ncpus);
if (error)
goto error;
switch (cputype) {
#if defined(__i386__) || defined(__amd64__)
/*
* Intel Core, Core 2 and Atom processors.
*/
case PMC_CPU_INTEL_ATOM:
case PMC_CPU_INTEL_ATOM_SILVERMONT:
case PMC_CPU_INTEL_BROADWELL:
case PMC_CPU_INTEL_CORE:
case PMC_CPU_INTEL_CORE2:
case PMC_CPU_INTEL_CORE2EXTREME:
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_NEHALEM_EX:
case PMC_CPU_INTEL_IVYBRIDGE:
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_WESTMERE:
case PMC_CPU_INTEL_WESTMERE_EX:
case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
case PMC_CPU_INTEL_IVYBRIDGE_XEON:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_HASWELL_XEON:
error = pmc_core_initialize(pmc_mdep, ncpus, verov);
break;
/*
* Intel Pentium 4 Processors, and P4/EMT64 processors.
*/
case PMC_CPU_INTEL_PIV:
error = pmc_p4_initialize(pmc_mdep, ncpus);
break;
#endif
#if defined(__i386__)
/*
* P6 Family Processors
*/
case PMC_CPU_INTEL_P6:
case PMC_CPU_INTEL_CL:
case PMC_CPU_INTEL_PII:
case PMC_CPU_INTEL_PIII:
case PMC_CPU_INTEL_PM:
error = pmc_p6_initialize(pmc_mdep, ncpus);
break;
/*
* Intel Pentium PMCs.
*/
case PMC_CPU_INTEL_P5:
error = pmc_p5_initialize(pmc_mdep, ncpus);
break;
#endif
default:
KASSERT(0, ("[intel,%d] Unknown CPU type", __LINE__));
}
if (error) {
pmc_tsc_finalize(pmc_mdep);
goto error;
}
/*
* Init the uncore class.
*/
#if defined(__i386__) || defined(__amd64__)
switch (cputype) {
/*
* Intel Corei7 and Westmere processors.
*/
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_WESTMERE:
case PMC_CPU_INTEL_BROADWELL:
error = pmc_uncore_initialize(pmc_mdep, ncpus);
break;
default:
break;
}
#endif
error:
if (error) {
pmc_mdep_free(pmc_mdep);
pmc_mdep = NULL;
}
return (pmc_mdep);
}
void
pmc_intel_finalize(struct pmc_mdep *md)
{
pmc_tsc_finalize(md);
switch (md->pmd_cputype) {
#if defined(__i386__) || defined(__amd64__)
case PMC_CPU_INTEL_ATOM:
case PMC_CPU_INTEL_ATOM_SILVERMONT:
case PMC_CPU_INTEL_BROADWELL:
case PMC_CPU_INTEL_CORE:
case PMC_CPU_INTEL_CORE2:
case PMC_CPU_INTEL_CORE2EXTREME:
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_NEHALEM_EX:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_HASWELL_XEON:
case PMC_CPU_INTEL_IVYBRIDGE:
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_WESTMERE:
case PMC_CPU_INTEL_WESTMERE_EX:
case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
case PMC_CPU_INTEL_IVYBRIDGE_XEON:
pmc_core_finalize(md);
break;
case PMC_CPU_INTEL_PIV:
pmc_p4_finalize(md);
break;
#endif
#if defined(__i386__)
case PMC_CPU_INTEL_P6:
case PMC_CPU_INTEL_CL:
case PMC_CPU_INTEL_PII:
case PMC_CPU_INTEL_PIII:
case PMC_CPU_INTEL_PM:
pmc_p6_finalize(md);
break;
case PMC_CPU_INTEL_P5:
pmc_p5_finalize(md);
break;
#endif
default:
KASSERT(0, ("[intel,%d] unknown CPU type", __LINE__));
}
/*
* Uncore.
*/
#if defined(__i386__) || defined(__amd64__)
switch (md->pmd_cputype) {
case PMC_CPU_INTEL_BROADWELL:
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_WESTMERE:
pmc_uncore_finalize(md);
break;
default:
break;
}
#endif
}
Index: head/sys/dev/hwpmc/hwpmc_logging.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_logging.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_logging.c (revision 282658)
@@ -1,1073 +1,1073 @@
/*-
* Copyright (c) 2005-2007 Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
/*
* Logging code for hwpmc(4)
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#if (__FreeBSD_version >= 1100000)
#include <sys/capsicum.h>
#else
#include <sys/capability.h>
#endif
#include <sys/file.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/pmclog.h>
#include <sys/proc.h>
#include <sys/signalvar.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/uio.h>
#include <sys/unistd.h>
#include <sys/vnode.h>
/*
* Sysctl tunables
*/
SYSCTL_DECL(_kern_hwpmc);
/*
* kern.hwpmc.logbuffersize -- size of the per-cpu owner buffers.
*/
static int pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, logbuffersize, CTLFLAG_RDTUN,
&pmclog_buffer_size, 0, "size of log buffers in kilobytes");
/*
* kern.hwpmc.nbuffer -- number of global log buffers
*/
static int pmc_nlogbuffers = PMC_NLOGBUFFERS;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, nbuffers, CTLFLAG_RDTUN,
&pmc_nlogbuffers, 0, "number of global log buffers");
/*
* Global log buffer list and associated spin lock.
*/
TAILQ_HEAD(, pmclog_buffer) pmc_bufferlist =
TAILQ_HEAD_INITIALIZER(pmc_bufferlist);
static struct mtx pmc_bufferlist_mtx; /* spin lock */
static struct mtx pmc_kthread_mtx; /* sleep lock */
#define PMCLOG_INIT_BUFFER_DESCRIPTOR(D) do { \
const int __roundup = roundup(sizeof(*D), \
sizeof(uint32_t)); \
(D)->plb_fence = ((char *) (D)) + \
1024*pmclog_buffer_size; \
(D)->plb_base = (D)->plb_ptr = ((char *) (D)) + \
__roundup; \
} while (0)
/*
* Log file record constructors.
*/
#define _PMCLOG_TO_HEADER(T,L) \
((PMCLOG_HEADER_MAGIC << 24) | \
(PMCLOG_TYPE_ ## T << 16) | \
((L) & 0xFFFF))
/* reserve LEN bytes of space and initialize the entry header */
#define _PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do { \
uint32_t *_le; \
int _len = roundup((LEN), sizeof(uint32_t)); \
if ((_le = pmclog_reserve((PO), _len)) == NULL) { \
ACTION; \
} \
*_le = _PMCLOG_TO_HEADER(TYPE,_len); \
_le += 3 /* skip over timestamp */
#define PMCLOG_RESERVE(P,T,L) _PMCLOG_RESERVE(P,T,L,return)
#define PMCLOG_RESERVE_WITH_ERROR(P,T,L) _PMCLOG_RESERVE(P,T,L, \
error=ENOMEM;goto error)
#define PMCLOG_EMIT32(V) do { *_le++ = (V); } while (0)
#define PMCLOG_EMIT64(V) do { \
*_le++ = (uint32_t) ((V) & 0xFFFFFFFF); \
*_le++ = (uint32_t) (((V) >> 32) & 0xFFFFFFFF); \
} while (0)
/* Emit a string. Caution: does NOT update _le, so needs to be last */
#define PMCLOG_EMITSTRING(S,L) do { bcopy((S), _le, (L)); } while (0)
#define PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0)
#define PMCLOG_DESPATCH(PO) \
pmclog_release((PO)); \
} while (0)
/*
* Assertions about the log file format.
*/
CTASSERT(sizeof(struct pmclog_callchain) == 6*4 +
PMC_CALLCHAIN_DEPTH_MAX*sizeof(uintfptr_t));
CTASSERT(sizeof(struct pmclog_closelog) == 3*4);
CTASSERT(sizeof(struct pmclog_dropnotify) == 3*4);
CTASSERT(sizeof(struct pmclog_map_in) == PATH_MAX +
4*4 + sizeof(uintfptr_t));
CTASSERT(offsetof(struct pmclog_map_in,pl_pathname) ==
4*4 + sizeof(uintfptr_t));
CTASSERT(sizeof(struct pmclog_map_out) == 4*4 + 2*sizeof(uintfptr_t));
CTASSERT(sizeof(struct pmclog_pcsample) == 6*4 + sizeof(uintfptr_t));
CTASSERT(sizeof(struct pmclog_pmcallocate) == 6*4);
CTASSERT(sizeof(struct pmclog_pmcattach) == 5*4 + PATH_MAX);
CTASSERT(offsetof(struct pmclog_pmcattach,pl_pathname) == 5*4);
CTASSERT(sizeof(struct pmclog_pmcdetach) == 5*4);
CTASSERT(sizeof(struct pmclog_proccsw) == 5*4 + 8);
CTASSERT(sizeof(struct pmclog_procexec) == 5*4 + PATH_MAX +
sizeof(uintfptr_t));
CTASSERT(offsetof(struct pmclog_procexec,pl_pathname) == 5*4 +
sizeof(uintfptr_t));
CTASSERT(sizeof(struct pmclog_procexit) == 5*4 + 8);
CTASSERT(sizeof(struct pmclog_procfork) == 5*4);
CTASSERT(sizeof(struct pmclog_sysexit) == 4*4);
CTASSERT(sizeof(struct pmclog_userdata) == 4*4);
/*
* Log buffer structure
*/
struct pmclog_buffer {
TAILQ_ENTRY(pmclog_buffer) plb_next;
char *plb_base;
char *plb_ptr;
char *plb_fence;
};
/*
* Prototypes
*/
static int pmclog_get_buffer(struct pmc_owner *po);
static void pmclog_loop(void *arg);
static void pmclog_release(struct pmc_owner *po);
static uint32_t *pmclog_reserve(struct pmc_owner *po, int length);
static void pmclog_schedule_io(struct pmc_owner *po);
static void pmclog_stop_kthread(struct pmc_owner *po);
/*
* Helper functions
*/
/*
* Get a log buffer
*/
static int
pmclog_get_buffer(struct pmc_owner *po)
{
struct pmclog_buffer *plb;
mtx_assert(&po->po_mtx, MA_OWNED);
KASSERT(po->po_curbuf == NULL,
("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));
mtx_lock_spin(&pmc_bufferlist_mtx);
if ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL)
TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
- PMCDBG(LOG,GTB,1, "po=%p plb=%p", po, plb);
+ PMCDBG2(LOG,GTB,1, "po=%p plb=%p", po, plb);
#ifdef HWPMC_DEBUG
if (plb)
KASSERT(plb->plb_ptr == plb->plb_base &&
plb->plb_base < plb->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p "
"base=%p fence=%p", __LINE__, po, plb->plb_ptr,
plb->plb_base, plb->plb_fence));
#endif
po->po_curbuf = plb;
/* update stats */
atomic_add_int(&pmc_stats.pm_buffer_requests, 1);
if (plb == NULL)
atomic_add_int(&pmc_stats.pm_buffer_requests_failed, 1);
return (plb ? 0 : ENOMEM);
}
/*
* Log handler loop.
*
* This function is executed by each pmc owner's helper thread.
*/
static void
pmclog_loop(void *arg)
{
int error;
struct pmc_owner *po;
struct pmclog_buffer *lb;
struct proc *p;
struct ucred *ownercred;
struct ucred *mycred;
struct thread *td;
struct uio auio;
struct iovec aiov;
size_t nbytes;
po = (struct pmc_owner *) arg;
p = po->po_owner;
td = curthread;
mycred = td->td_ucred;
PROC_LOCK(p);
ownercred = crhold(p->p_ucred);
PROC_UNLOCK(p);
- PMCDBG(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
+ PMCDBG2(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
KASSERT(po->po_kthread == curthread->td_proc,
("[pmclog,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__,
po, po->po_kthread, curthread->td_proc));
lb = NULL;
/*
* Loop waiting for I/O requests to be added to the owner
* struct's queue. The loop is exited when the log file
* is deconfigured.
*/
mtx_lock(&pmc_kthread_mtx);
for (;;) {
/* check if we've been asked to exit */
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
break;
if (lb == NULL) { /* look for a fresh buffer to write */
mtx_lock_spin(&po->po_mtx);
if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
mtx_unlock_spin(&po->po_mtx);
/* No more buffers and shutdown required. */
if (po->po_flags & PMC_PO_SHUTDOWN) {
mtx_unlock(&pmc_kthread_mtx);
/*
* Close the file to get PMCLOG_EOF
* error in pmclog(3).
*/
fo_close(po->po_file, curthread);
mtx_lock(&pmc_kthread_mtx);
break;
}
(void) msleep(po, &pmc_kthread_mtx, PWAIT,
"pmcloop", 0);
continue;
}
TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
mtx_unlock_spin(&po->po_mtx);
}
mtx_unlock(&pmc_kthread_mtx);
/* process the request */
- PMCDBG(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
+ PMCDBG3(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
lb->plb_base, lb->plb_ptr);
/* change our thread's credentials before issuing the I/O */
aiov.iov_base = lb->plb_base;
aiov.iov_len = nbytes = lb->plb_ptr - lb->plb_base;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = -1;
auio.uio_resid = nbytes;
auio.uio_rw = UIO_WRITE;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_td = td;
/* switch thread credentials -- see kern_ktrace.c */
td->td_ucred = ownercred;
error = fo_write(po->po_file, &auio, ownercred, 0, td);
td->td_ucred = mycred;
if (error) {
/* XXX some errors are recoverable */
/* send a SIGIO to the owner and exit */
PROC_LOCK(p);
kern_psignal(p, SIGIO);
PROC_UNLOCK(p);
mtx_lock(&pmc_kthread_mtx);
po->po_error = error; /* save for flush log */
- PMCDBG(LOG,WRI,2, "po=%p error=%d", po, error);
+ PMCDBG2(LOG,WRI,2, "po=%p error=%d", po, error);
break;
}
mtx_lock(&pmc_kthread_mtx);
/* put the used buffer back into the global pool */
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
lb = NULL;
}
wakeup_one(po->po_kthread);
po->po_kthread = NULL;
mtx_unlock(&pmc_kthread_mtx);
/* return the current I/O buffer to the global pool */
if (lb) {
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/*
* Exit this thread, signalling the waiter
*/
crfree(ownercred);
kproc_exit(0);
}
/*
* Release and log entry and schedule an I/O if needed.
*/
static void
pmclog_release(struct pmc_owner *po)
{
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
/* schedule an I/O if we've filled a buffer */
if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence)
pmclog_schedule_io(po);
mtx_unlock_spin(&po->po_mtx);
- PMCDBG(LOG,REL,1, "po=%p", po);
+ PMCDBG1(LOG,REL,1, "po=%p", po);
}
/*
* Attempt to reserve 'length' bytes of space in an owner's log
* buffer. The function returns a pointer to 'length' bytes of space
* if there was enough space or returns NULL if no space was
* available. Non-null returns do so with the po mutex locked. The
* caller must invoke pmclog_release() on the pmc owner structure
* when done.
*/
static uint32_t *
pmclog_reserve(struct pmc_owner *po, int length)
{
uintptr_t newptr, oldptr;
uint32_t *lh;
struct timespec ts;
- PMCDBG(LOG,ALL,1, "po=%p len=%d", po, length);
+ PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length);
KASSERT(length % sizeof(uint32_t) == 0,
("[pmclog,%d] length not a multiple of word size", __LINE__));
mtx_lock_spin(&po->po_mtx);
/* No more data when shutdown in progress. */
if (po->po_flags & PMC_PO_SHUTDOWN) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
if (po->po_curbuf == NULL)
if (pmclog_get_buffer(po) != 0) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] po=%p no current buffer", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base &&
po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
po->po_curbuf->plb_fence));
oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
newptr = oldptr + length;
KASSERT(oldptr != (uintptr_t) NULL,
("[pmclog,%d] po=%p Null log buffer pointer", __LINE__, po));
/*
* If we have space in the current buffer, return a pointer to
* available space with the PO structure locked.
*/
if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) {
po->po_curbuf->plb_ptr = (char *) newptr;
goto done;
}
/*
* Otherwise, schedule the current buffer for output and get a
* fresh buffer.
*/
pmclog_schedule_io(po);
if (pmclog_get_buffer(po) != 0) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] po=%p no current buffer", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr != NULL,
("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__));
KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base &&
po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
po->po_curbuf->plb_fence));
oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
done:
lh = (uint32_t *) oldptr;
lh++; /* skip header */
getnanotime(&ts); /* fill in the timestamp */
*lh++ = ts.tv_sec & 0xFFFFFFFF;
*lh++ = ts.tv_nsec & 0xFFFFFFF;
return ((uint32_t *) oldptr);
}
/*
* Schedule an I/O.
*
* Transfer the current buffer to the helper kthread.
*/
static void
pmclog_schedule_io(struct pmc_owner *po)
{
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
- PMCDBG(LOG,SIO, 1, "po=%p", po);
+ PMCDBG1(LOG,SIO, 1, "po=%p", po);
mtx_assert(&po->po_mtx, MA_OWNED);
/*
* Add the current buffer to the tail of the buffer list and
* wakeup the helper.
*/
TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next);
po->po_curbuf = NULL;
wakeup_one(po);
}
/*
* Stop the helper kthread.
*/
static void
pmclog_stop_kthread(struct pmc_owner *po)
{
/*
* Close the file to force the thread out of fo_write,
* unset flag, wakeup the helper thread,
* wait for it to exit
*/
if (po->po_file != NULL)
fo_close(po->po_file, curthread);
mtx_lock(&pmc_kthread_mtx);
po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
wakeup_one(po);
if (po->po_kthread)
msleep(po->po_kthread, &pmc_kthread_mtx, PPAUSE, "pmckstp", 0);
mtx_unlock(&pmc_kthread_mtx);
}
/*
* Public functions
*/
/*
* Configure a log file for pmc owner 'po'.
*
* Parameter 'logfd' is a file handle referencing an open file in the
* owner process. This file needs to have been opened for writing.
*/
int
pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
{
int error;
struct proc *p;
cap_rights_t rights;
/*
* As long as it is possible to get a LOR between pmc_sx lock and
* proctree/allproc sx locks used for adding a new process, assure
* the former is not held here.
*/
sx_assert(&pmc_sx, SA_UNLOCKED);
- PMCDBG(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
+ PMCDBG2(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
p = po->po_owner;
/* return EBUSY if a log file was already present */
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
return (EBUSY);
KASSERT(po->po_kthread == NULL,
("[pmclog,%d] po=%p kthread (%p) already present", __LINE__, po,
po->po_kthread));
KASSERT(po->po_file == NULL,
("[pmclog,%d] po=%p file (%p) already present", __LINE__, po,
po->po_file));
/* get a reference to the file state */
error = fget_write(curthread, logfd,
cap_rights_init(&rights, CAP_WRITE), &po->po_file);
if (error)
goto error;
/* mark process as owning a log file */
po->po_flags |= PMC_PO_OWNS_LOGFILE;
error = kproc_create(pmclog_loop, po, &po->po_kthread,
RFHIGHPID, 0, "hwpmc: proc(%d)", p->p_pid);
if (error)
goto error;
/* mark process as using HWPMCs */
PROC_LOCK(p);
p->p_flag |= P_HWPMC;
PROC_UNLOCK(p);
/* create a log initialization entry */
PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
sizeof(struct pmclog_initialize));
PMCLOG_EMIT32(PMC_VERSION);
PMCLOG_EMIT32(md->pmd_cputype);
PMCLOG_DESPATCH(po);
return (0);
error:
/* shutdown the thread */
if (po->po_kthread)
pmclog_stop_kthread(po);
KASSERT(po->po_kthread == NULL, ("[pmclog,%d] po=%p kthread not "
"stopped", __LINE__, po));
if (po->po_file)
(void) fdrop(po->po_file, curthread);
po->po_file = NULL; /* clear file and error state */
po->po_error = 0;
return (error);
}
/*
* De-configure a log file. This will throw away any buffers queued
* for this owner process.
*/
int
pmclog_deconfigure_log(struct pmc_owner *po)
{
int error;
struct pmclog_buffer *lb;
- PMCDBG(LOG,CFG,1, "de-config po=%p", po);
+ PMCDBG1(LOG,CFG,1, "de-config po=%p", po);
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
return (EINVAL);
KASSERT(po->po_sscount == 0,
("[pmclog,%d] po=%p still owning SS PMCs", __LINE__, po));
KASSERT(po->po_file != NULL,
("[pmclog,%d] po=%p no log file", __LINE__, po));
/* stop the kthread, this will reset the 'OWNS_LOGFILE' flag */
pmclog_stop_kthread(po);
KASSERT(po->po_kthread == NULL,
("[pmclog,%d] po=%p kthread not stopped", __LINE__, po));
/* return all queued log buffers to the global pool */
while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) {
TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/* return the 'current' buffer to the global pool */
if ((lb = po->po_curbuf) != NULL) {
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/* drop a reference to the fd */
error = fdrop(po->po_file, curthread);
po->po_file = NULL;
po->po_error = 0;
return (error);
}
/*
* Flush a process' log buffer.
*/
int
pmclog_flush(struct pmc_owner *po)
{
int error;
struct pmclog_buffer *lb;
- PMCDBG(LOG,FLS,1, "po=%p", po);
+ PMCDBG1(LOG,FLS,1, "po=%p", po);
/*
* If there is a pending error recorded by the logger thread,
* return that.
*/
if (po->po_error)
return (po->po_error);
error = 0;
/*
* Check that we do have an active log file.
*/
mtx_lock(&pmc_kthread_mtx);
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
error = EINVAL;
goto error;
}
/*
* Schedule the current buffer if any and not empty.
*/
mtx_lock_spin(&po->po_mtx);
lb = po->po_curbuf;
if (lb && lb->plb_ptr != lb->plb_base) {
pmclog_schedule_io(po);
} else
error = ENOBUFS;
mtx_unlock_spin(&po->po_mtx);
error:
mtx_unlock(&pmc_kthread_mtx);
return (error);
}
int
pmclog_close(struct pmc_owner *po)
{
- PMCDBG(LOG,CLO,1, "po=%p", po);
+ PMCDBG1(LOG,CLO,1, "po=%p", po);
mtx_lock(&pmc_kthread_mtx);
/*
* Schedule the current buffer.
*/
mtx_lock_spin(&po->po_mtx);
if (po->po_curbuf)
pmclog_schedule_io(po);
else
wakeup_one(po);
mtx_unlock_spin(&po->po_mtx);
/*
* Initiate shutdown: no new data queued,
* thread will close file on last block.
*/
po->po_flags |= PMC_PO_SHUTDOWN;
mtx_unlock(&pmc_kthread_mtx);
return (0);
}
void
pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
{
int n, recordlen;
uint32_t flags;
struct pmc_owner *po;
- PMCDBG(LOG,SAM,1,"pm=%p pid=%d n=%d", pm, ps->ps_pid,
+ PMCDBG3(LOG,SAM,1,"pm=%p pid=%d n=%d", pm, ps->ps_pid,
ps->ps_nsamples);
recordlen = offsetof(struct pmclog_callchain, pl_pc) +
ps->ps_nsamples * sizeof(uintfptr_t);
po = pm->pm_owner;
flags = PMC_CALLCHAIN_TO_CPUFLAGS(ps->ps_cpu,ps->ps_flags);
PMCLOG_RESERVE(po, CALLCHAIN, recordlen);
PMCLOG_EMIT32(ps->ps_pid);
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(flags);
for (n = 0; n < ps->ps_nsamples; n++)
PMCLOG_EMITADDR(ps->ps_pc[n]);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_closelog(struct pmc_owner *po)
{
PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog));
PMCLOG_DESPATCH(po);
}
void
pmclog_process_dropnotify(struct pmc_owner *po)
{
PMCLOG_RESERVE(po,DROPNOTIFY,sizeof(struct pmclog_dropnotify));
PMCLOG_DESPATCH(po);
}
void
pmclog_process_map_in(struct pmc_owner *po, pid_t pid, uintfptr_t start,
const char *path)
{
int pathlen, recordlen;
KASSERT(path != NULL, ("[pmclog,%d] map-in, null path", __LINE__));
pathlen = strlen(path) + 1; /* #bytes for path name */
recordlen = offsetof(struct pmclog_map_in, pl_pathname) +
pathlen;
PMCLOG_RESERVE(po, MAP_IN, recordlen);
PMCLOG_EMIT32(pid);
PMCLOG_EMITADDR(start);
PMCLOG_EMITSTRING(path,pathlen);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_map_out(struct pmc_owner *po, pid_t pid, uintfptr_t start,
uintfptr_t end)
{
KASSERT(start <= end, ("[pmclog,%d] start > end", __LINE__));
PMCLOG_RESERVE(po, MAP_OUT, sizeof(struct pmclog_map_out));
PMCLOG_EMIT32(pid);
PMCLOG_EMITADDR(start);
PMCLOG_EMITADDR(end);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_pmcallocate(struct pmc *pm)
{
struct pmc_owner *po;
struct pmc_soft *ps;
po = pm->pm_owner;
- PMCDBG(LOG,ALL,1, "pm=%p", pm);
+ PMCDBG1(LOG,ALL,1, "pm=%p", pm);
if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) {
PMCLOG_RESERVE(po, PMCALLOCATEDYN,
sizeof(struct pmclog_pmcallocatedyn));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
ps = pmc_soft_ev_acquire(pm->pm_event);
if (ps != NULL)
PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX);
else
PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
pmc_soft_ev_release(ps);
PMCLOG_DESPATCH(po);
} else {
PMCLOG_RESERVE(po, PMCALLOCATE,
sizeof(struct pmclog_pmcallocate));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
PMCLOG_DESPATCH(po);
}
}
void
pmclog_process_pmcattach(struct pmc *pm, pid_t pid, char *path)
{
int pathlen, recordlen;
struct pmc_owner *po;
- PMCDBG(LOG,ATT,1,"pm=%p pid=%d", pm, pid);
+ PMCDBG2(LOG,ATT,1,"pm=%p pid=%d", pm, pid);
po = pm->pm_owner;
pathlen = strlen(path) + 1; /* #bytes for the string */
recordlen = offsetof(struct pmclog_pmcattach, pl_pathname) + pathlen;
PMCLOG_RESERVE(po, PMCATTACH, recordlen);
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pid);
PMCLOG_EMITSTRING(path, pathlen);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_pmcdetach(struct pmc *pm, pid_t pid)
{
struct pmc_owner *po;
- PMCDBG(LOG,ATT,1,"!pm=%p pid=%d", pm, pid);
+ PMCDBG2(LOG,ATT,1,"!pm=%p pid=%d", pm, pid);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pid);
PMCLOG_DESPATCH(po);
}
/*
* Log a context switch event to the log file.
*/
void
pmclog_process_proccsw(struct pmc *pm, struct pmc_process *pp, pmc_value_t v)
{
struct pmc_owner *po;
KASSERT(pm->pm_flags & PMC_F_LOG_PROCCSW,
("[pmclog,%d] log-process-csw called gratuitously", __LINE__));
- PMCDBG(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
+ PMCDBG3(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
v);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PROCCSW, sizeof(struct pmclog_proccsw));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT64(v);
PMCLOG_EMIT32(pp->pp_proc->p_pid);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_procexec(struct pmc_owner *po, pmc_id_t pmid, pid_t pid,
uintfptr_t startaddr, char *path)
{
int pathlen, recordlen;
- PMCDBG(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path);
+ PMCDBG3(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path);
pathlen = strlen(path) + 1; /* #bytes for the path */
recordlen = offsetof(struct pmclog_procexec, pl_pathname) + pathlen;
PMCLOG_RESERVE(po, PROCEXEC, recordlen);
PMCLOG_EMIT32(pid);
PMCLOG_EMITADDR(startaddr);
PMCLOG_EMIT32(pmid);
PMCLOG_EMITSTRING(path,pathlen);
PMCLOG_DESPATCH(po);
}
/*
* Log a process exit event (and accumulated pmc value) to the log file.
*/
void
pmclog_process_procexit(struct pmc *pm, struct pmc_process *pp)
{
int ri;
struct pmc_owner *po;
ri = PMC_TO_ROWINDEX(pm);
- PMCDBG(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
+ PMCDBG3(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
pp->pp_pmcs[ri].pp_pmcval);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PROCEXIT, sizeof(struct pmclog_procexit));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT64(pp->pp_pmcs[ri].pp_pmcval);
PMCLOG_EMIT32(pp->pp_proc->p_pid);
PMCLOG_DESPATCH(po);
}
/*
* Log a fork event.
*/
void
pmclog_process_procfork(struct pmc_owner *po, pid_t oldpid, pid_t newpid)
{
PMCLOG_RESERVE(po, PROCFORK, sizeof(struct pmclog_procfork));
PMCLOG_EMIT32(oldpid);
PMCLOG_EMIT32(newpid);
PMCLOG_DESPATCH(po);
}
/*
* Log a process exit event of the form suitable for system-wide PMCs.
*/
void
pmclog_process_sysexit(struct pmc_owner *po, pid_t pid)
{
PMCLOG_RESERVE(po, SYSEXIT, sizeof(struct pmclog_sysexit));
PMCLOG_EMIT32(pid);
PMCLOG_DESPATCH(po);
}
/*
* Write a user log entry.
*/
int
pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
{
int error;
- PMCDBG(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata);
+ PMCDBG2(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata);
error = 0;
PMCLOG_RESERVE_WITH_ERROR(po, USERDATA,
sizeof(struct pmclog_userdata));
PMCLOG_EMIT32(wl->pm_userdata);
PMCLOG_DESPATCH(po);
error:
return (error);
}
/*
* Initialization.
*
* Create a pool of log buffers and initialize mutexes.
*/
void
pmclog_initialize()
{
int n;
struct pmclog_buffer *plb;
if (pmclog_buffer_size <= 0) {
(void) printf("hwpmc: tunable logbuffersize=%d must be "
"greater than zero.\n", pmclog_buffer_size);
pmclog_buffer_size = PMC_LOG_BUFFER_SIZE;
}
if (pmc_nlogbuffers <= 0) {
(void) printf("hwpmc: tunable nlogbuffers=%d must be greater "
"than zero.\n", pmc_nlogbuffers);
pmc_nlogbuffers = PMC_NLOGBUFFERS;
}
/* create global pool of log buffers */
for (n = 0; n < pmc_nlogbuffers; n++) {
plb = malloc(1024 * pmclog_buffer_size, M_PMC,
M_WAITOK|M_ZERO);
PMCLOG_INIT_BUFFER_DESCRIPTOR(plb);
TAILQ_INSERT_HEAD(&pmc_bufferlist, plb, plb_next);
}
mtx_init(&pmc_bufferlist_mtx, "pmc-buffer-list", "pmc-leaf",
MTX_SPIN);
mtx_init(&pmc_kthread_mtx, "pmc-kthread", "pmc-sleep", MTX_DEF);
}
/*
* Shutdown logging.
*
* Destroy mutexes and release memory back the to free pool.
*/
void
pmclog_shutdown()
{
struct pmclog_buffer *plb;
mtx_destroy(&pmc_kthread_mtx);
mtx_destroy(&pmc_bufferlist_mtx);
while ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL) {
TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
free(plb, M_PMC);
}
}
Index: head/sys/dev/hwpmc/hwpmc_mips.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_mips.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_mips.c (revision 282658)
@@ -1,807 +1,807 @@
/*-
* Copyright (c) 2010, George V. Neville-Neil <gnn@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_hwpmc_hooks.h"
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/pmc_mdep.h>
#include <machine/md_var.h>
#include <machine/mips_opcode.h>
#include <machine/vmparam.h>
int mips_npmcs;
/*
* Per-processor information.
*/
struct mips_cpu {
struct pmc_hw *pc_mipspmcs;
};
static struct mips_cpu **mips_pcpu;
#if defined(__mips_n64)
# define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \
((vm_offset_t)(reg) >= MIPS_XKPHYS_START))
#else
# define MIPS_IS_VALID_KERNELADDR(reg) ((((reg) & 3) == 0) && \
((vm_offset_t)(reg) >= MIPS_KSEG0_START))
#endif
/*
* We need some reasonable default to prevent backtrace code
* from wandering too far
*/
#define MAX_FUNCTION_SIZE 0x10000
#define MAX_PROLOGUE_SIZE 0x100
static int
mips_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event pe;
uint32_t caps, config, counter;
uint32_t event;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row index %d", __LINE__, ri));
caps = a->pm_caps;
if (a->pm_class != mips_pmc_spec.ps_cpuclass)
return (EINVAL);
pe = a->pm_ev;
counter = MIPS_CTR_ALL;
event = 0;
for (i = 0; i < mips_event_codes_size; i++) {
if (mips_event_codes[i].pe_ev == pe) {
event = mips_event_codes[i].pe_code;
counter = mips_event_codes[i].pe_counter;
break;
}
}
if (i == mips_event_codes_size)
return (EINVAL);
if ((counter != MIPS_CTR_ALL) && (counter != ri))
return (EINVAL);
config = mips_get_perfctl(cpu, ri, event, caps);
pm->pm_md.pm_mips_evsel = config;
- PMCDBG(MDP,ALL,2,"mips-allocate ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"mips-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
mips_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row index %d", __LINE__, ri));
pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
tmp = mips_pmcn_read(ri);
- PMCDBG(MDP,REA,2,"mips-read id=%d -> %jd", ri, tmp);
+ PMCDBG2(MDP,REA,2,"mips-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = tmp - (1UL << (mips_pmc_spec.ps_counter_width - 1));
else
*v = tmp;
return 0;
}
static int
mips_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row-index %d", __LINE__, ri));
pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = (1UL << (mips_pmc_spec.ps_counter_width - 1)) - v;
- PMCDBG(MDP,WRI,1,"mips-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1,"mips-write cpu=%d ri=%d v=%jx", cpu, ri, v);
mips_pmcn_write(ri, v);
return 0;
}
static int
mips_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row-index %d", __LINE__, ri));
phw = &mips_pcpu[cpu]->pc_mipspmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[mips,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
static int
mips_start_pmc(int cpu, int ri)
{
uint32_t config;
struct pmc *pm;
struct pmc_hw *phw;
phw = &mips_pcpu[cpu]->pc_mipspmcs[ri];
pm = phw->phw_pmc;
config = pm->pm_md.pm_mips_evsel;
/* Enable the PMC. */
switch (ri) {
case 0:
mips_wr_perfcnt0(config);
break;
case 1:
mips_wr_perfcnt2(config);
break;
default:
break;
}
return 0;
}
static int
mips_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct pmc_hw *phw;
phw = &mips_pcpu[cpu]->pc_mipspmcs[ri];
pm = phw->phw_pmc;
/*
* Disable the PMCs.
*
* Clearing the entire register turns the counter off as well
* as removes the previously sampled event.
*/
switch (ri) {
case 0:
mips_wr_perfcnt0(0);
break;
case 1:
mips_wr_perfcnt2(0);
break;
default:
break;
}
return 0;
}
static int
mips_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row-index %d", __LINE__, ri));
phw = &mips_pcpu[cpu]->pc_mipspmcs[ri];
KASSERT(phw->phw_pmc == NULL,
("[mips,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
return 0;
}
static int
mips_pmc_intr(int cpu, struct trapframe *tf)
{
int error;
int retval, ri;
struct pmc *pm;
struct mips_cpu *pc;
uint32_t r0, r2;
pmc_value_t r;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] CPU %d out of range", __LINE__, cpu));
retval = 0;
pc = mips_pcpu[cpu];
/* Stop PMCs without clearing the counter */
r0 = mips_rd_perfcnt0();
mips_wr_perfcnt0(r0 & ~(0x1f));
r2 = mips_rd_perfcnt2();
mips_wr_perfcnt2(r2 & ~(0x1f));
for (ri = 0; ri < mips_npmcs; ri++) {
pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
if (pm == NULL)
continue;
if (! PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
r = mips_pmcn_read(ri);
/* If bit 31 is set, the counter has overflowed */
if ((r & (1UL << (mips_pmc_spec.ps_counter_width - 1))) == 0)
continue;
retval = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error) {
/* Clear/disable the relevant counter */
if (ri == 0)
r0 = 0;
else if (ri == 1)
r2 = 0;
mips_stop_pmc(cpu, ri);
}
/* Reload sampling count */
mips_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount);
}
/*
* Re-enable the PMC counters where they left off.
*
* Any counter which overflowed will have its sample count
* reloaded in the loop above.
*/
mips_wr_perfcnt0(r0);
mips_wr_perfcnt2(r2);
return retval;
}
static int
mips_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char mips_name[PMC_NAME_MAX];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d], illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] row-index %d out of range", __LINE__, ri));
phw = &mips_pcpu[cpu]->pc_mipspmcs[ri];
snprintf(mips_name, sizeof(mips_name), "MIPS-%d", ri);
if ((error = copystr(mips_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return error;
pi->pm_class = mips_pmc_spec.ps_cpuclass;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
mips_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
return 0;
}
/*
* XXX don't know what we should do here.
*/
static int
mips_pmc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
return 0;
}
static int
mips_pmc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
return 0;
}
static int
mips_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct pmc_cpu *pc;
struct mips_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] wrong cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"mips-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"mips-init cpu=%d", cpu);
mips_pcpu[cpu] = pac = malloc(sizeof(struct mips_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_mipspmcs = malloc(sizeof(struct pmc_hw) * mips_npmcs,
M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS].pcd_ri;
KASSERT(pc != NULL, ("[mips,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_mipspmcs; i < mips_npmcs; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/*
* Clear the counter control register which has the effect
* of disabling counting.
*/
for (i = 0; i < mips_npmcs; i++)
mips_pmcn_write(i, 0);
return 0;
}
static int
mips_pcpu_fini(struct pmc_mdep *md, int cpu)
{
return 0;
}
struct pmc_mdep *
pmc_mips_initialize()
{
struct pmc_mdep *pmc_mdep;
struct pmc_classdep *pcd;
/*
* TODO: Use More bit of PerfCntlX register to detect actual
* number of counters
*/
mips_npmcs = 2;
- PMCDBG(MDP,INI,1,"mips-init npmcs=%d", mips_npmcs);
+ PMCDBG1(MDP,INI,1,"mips-init npmcs=%d", mips_npmcs);
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
*/
mips_pcpu = malloc(sizeof(struct mips_cpu *) * pmc_cpu_max(), M_PMC,
M_WAITOK|M_ZERO);
/* Just one class */
pmc_mdep = pmc_mdep_alloc(1);
pmc_mdep->pmd_cputype = mips_pmc_spec.ps_cputype;
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS];
pcd->pcd_caps = mips_pmc_spec.ps_capabilities;
pcd->pcd_class = mips_pmc_spec.ps_cpuclass;
pcd->pcd_num = mips_npmcs;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = mips_pmc_spec.ps_counter_width;
pcd->pcd_allocate_pmc = mips_allocate_pmc;
pcd->pcd_config_pmc = mips_config_pmc;
pcd->pcd_pcpu_fini = mips_pcpu_fini;
pcd->pcd_pcpu_init = mips_pcpu_init;
pcd->pcd_describe = mips_describe;
pcd->pcd_get_config = mips_get_config;
pcd->pcd_read_pmc = mips_read_pmc;
pcd->pcd_release_pmc = mips_release_pmc;
pcd->pcd_start_pmc = mips_start_pmc;
pcd->pcd_stop_pmc = mips_stop_pmc;
pcd->pcd_write_pmc = mips_write_pmc;
pmc_mdep->pmd_intr = mips_pmc_intr;
pmc_mdep->pmd_switch_in = mips_pmc_switch_in;
pmc_mdep->pmd_switch_out = mips_pmc_switch_out;
pmc_mdep->pmd_npmc += mips_npmcs;
return (pmc_mdep);
}
void
pmc_mips_finalize(struct pmc_mdep *md)
{
(void) md;
}
#ifdef HWPMC_MIPS_BACKTRACE
static int
pmc_next_frame(register_t *pc, register_t *sp)
{
InstFmt i;
uintptr_t va;
uint32_t instr, mask;
int more, stksize;
register_t ra = 0;
/* Jump here after a nonstandard (interrupt handler) frame */
stksize = 0;
/* check for bad SP: could foul up next frame */
if (!MIPS_IS_VALID_KERNELADDR(*sp)) {
goto error;
}
/* check for bad PC */
if (!MIPS_IS_VALID_KERNELADDR(*pc)) {
goto error;
}
/*
* Find the beginning of the current subroutine by scanning
* backwards from the current PC for the end of the previous
* subroutine.
*/
va = *pc - sizeof(int);
while (1) {
instr = *((uint32_t *)va);
/* [d]addiu sp,sp,-X */
if (((instr & 0xffff8000) == 0x27bd8000)
|| ((instr & 0xffff8000) == 0x67bd8000))
break;
/* jr ra */
if (instr == 0x03e00008) {
/* skip over branch-delay slot instruction */
va += 2 * sizeof(int);
break;
}
va -= sizeof(int);
}
/* skip over nulls which might separate .o files */
while ((instr = *((uint32_t *)va)) == 0)
va += sizeof(int);
/* scan forwards to find stack size and any saved registers */
stksize = 0;
more = 3;
mask = 0;
for (; more; va += sizeof(int),
more = (more == 3) ? 3 : more - 1) {
/* stop if hit our current position */
if (va >= *pc)
break;
instr = *((uint32_t *)va);
i.word = instr;
switch (i.JType.op) {
case OP_SPECIAL:
switch (i.RType.func) {
case OP_JR:
case OP_JALR:
more = 2; /* stop after next instruction */
break;
case OP_SYSCALL:
case OP_BREAK:
more = 1; /* stop now */
};
break;
case OP_BCOND:
case OP_J:
case OP_JAL:
case OP_BEQ:
case OP_BNE:
case OP_BLEZ:
case OP_BGTZ:
more = 2; /* stop after next instruction */
break;
case OP_COP0:
case OP_COP1:
case OP_COP2:
case OP_COP3:
switch (i.RType.rs) {
case OP_BCx:
case OP_BCy:
more = 2; /* stop after next instruction */
};
break;
case OP_SW:
case OP_SD:
/*
* SP is being saved using S8(FP). Most likely it indicates
* that SP is modified in the function and we can't get
* its value safely without emulating code backward
* So just bail out on functions like this
*/
if ((i.IType.rs == 30) && (i.IType.rt = 29))
return (-1);
/* look for saved registers on the stack */
if (i.IType.rs != 29)
break;
/* only restore the first one */
if (mask & (1 << i.IType.rt))
break;
mask |= (1 << i.IType.rt);
if (i.IType.rt == 31)
ra = *((register_t *)(*sp + (short)i.IType.imm));
break;
case OP_ADDI:
case OP_ADDIU:
case OP_DADDI:
case OP_DADDIU:
/* look for stack pointer adjustment */
if (i.IType.rs != 29 || i.IType.rt != 29)
break;
stksize = -((short)i.IType.imm);
}
}
if (!MIPS_IS_VALID_KERNELADDR(ra))
return (-1);
*pc = ra;
*sp += stksize;
return (0);
error:
return (-1);
}
static int
pmc_next_uframe(register_t *pc, register_t *sp, register_t *ra)
{
int offset, registers_on_stack;
uint32_t opcode, mask;
register_t function_start;
int stksize;
InstFmt i;
registers_on_stack = 0;
mask = 0;
function_start = 0;
offset = 0;
stksize = 0;
while (offset < MAX_FUNCTION_SIZE) {
opcode = fuword32((void *)(*pc - offset));
/* [d]addiu sp, sp, -X*/
if (((opcode & 0xffff8000) == 0x27bd8000)
|| ((opcode & 0xffff8000) == 0x67bd8000)) {
function_start = *pc - offset;
registers_on_stack = 1;
break;
}
/* lui gp, X */
if ((opcode & 0xffff8000) == 0x3c1c0000) {
/*
* Function might start with this instruction
* Keep an eye on "jr ra" and sp correction
* with positive value further on
*/
function_start = *pc - offset;
}
if (function_start) {
/*
* Stop looking further. Possible end of
* function instruction: it means there is no
* stack modifications, sp is unchanged
*/
/* [d]addiu sp,sp,X */
if (((opcode & 0xffff8000) == 0x27bd0000)
|| ((opcode & 0xffff8000) == 0x67bd0000))
break;
if (opcode == 0x03e00008)
break;
}
offset += sizeof(int);
}
if (!function_start)
return (-1);
if (registers_on_stack) {
offset = 0;
while ((offset < MAX_PROLOGUE_SIZE)
&& ((function_start + offset) < *pc)) {
i.word = fuword32((void *)(function_start + offset));
switch (i.JType.op) {
case OP_SW:
/* look for saved registers on the stack */
if (i.IType.rs != 29)
break;
/* only restore the first one */
if (mask & (1 << i.IType.rt))
break;
mask |= (1 << i.IType.rt);
if (i.IType.rt == 31)
*ra = fuword32((void *)(*sp + (short)i.IType.imm));
break;
#if defined(__mips_n64)
case OP_SD:
/* look for saved registers on the stack */
if (i.IType.rs != 29)
break;
/* only restore the first one */
if (mask & (1 << i.IType.rt))
break;
mask |= (1 << i.IType.rt);
/* ra */
if (i.IType.rt == 31)
*ra = fuword64((void *)(*sp + (short)i.IType.imm));
break;
#endif
case OP_ADDI:
case OP_ADDIU:
case OP_DADDI:
case OP_DADDIU:
/* look for stack pointer adjustment */
if (i.IType.rs != 29 || i.IType.rt != 29)
break;
stksize = -((short)i.IType.imm);
}
offset += sizeof(int);
}
}
/*
* We reached the end of backtrace
*/
if (*pc == *ra)
return (-1);
*pc = *ra;
*sp += stksize;
return (0);
}
#endif /* HWPMC_MIPS_BACKTRACE */
struct pmc_mdep *
pmc_md_initialize()
{
return pmc_mips_initialize();
}
void
pmc_md_finalize(struct pmc_mdep *md)
{
return pmc_mips_finalize(md);
}
int
pmc_save_kernel_callchain(uintptr_t *cc, int nframes,
struct trapframe *tf)
{
register_t pc, ra, sp;
int frames = 0;
pc = tf->pc;
sp = tf->sp;
ra = tf->ra;
cc[frames++] = pc;
#ifdef HWPMC_MIPS_BACKTRACE
/*
* Unwind, and unwind, and unwind
*/
while (1) {
if (frames >= nframes)
break;
if (pmc_next_frame(&pc, &sp) < 0)
break;
cc[frames++] = pc;
}
#endif
return (frames);
}
int
pmc_save_user_callchain(uintptr_t *cc, int nframes,
struct trapframe *tf)
{
register_t pc, ra, sp;
int frames = 0;
pc = tf->pc;
sp = tf->sp;
ra = tf->ra;
cc[frames++] = pc;
#ifdef HWPMC_MIPS_BACKTRACE
/*
* Unwind, and unwind, and unwind
*/
while (1) {
if (frames >= nframes)
break;
if (pmc_next_uframe(&pc, &sp, &ra) < 0)
break;
cc[frames++] = pc;
}
#endif
return (frames);
}
Index: head/sys/dev/hwpmc/hwpmc_mips24k.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_mips24k.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_mips24k.c (revision 282658)
@@ -1,229 +1,229 @@
/*-
* Copyright (c) 2010 George V. Neville-Neil <gnn@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/pmc_mdep.h>
#define MIPS24K_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \
PMC_CAP_SYSTEM | PMC_CAP_EDGE | \
PMC_CAP_THRESHOLD | PMC_CAP_READ | \
PMC_CAP_WRITE | PMC_CAP_INVERT | \
PMC_CAP_QUALIFIER)
#define MIPS24K_PMC_INTERRUPT_ENABLE 0x10 /* Enable interrupts */
#define MIPS24K_PMC_USER_ENABLE 0x08 /* Count in USER mode */
#define MIPS24K_PMC_SUPER_ENABLE 0x04 /* Count in SUPERVISOR mode */
#define MIPS24K_PMC_KERNEL_ENABLE 0x02 /* Count in KERNEL mode */
#define MIPS24K_PMC_ENABLE (MIPS24K_PMC_USER_ENABLE | \
MIPS24K_PMC_SUPER_ENABLE | \
MIPS24K_PMC_KERNEL_ENABLE)
#define MIPS24K_PMC_SELECT 5 /* Which bit position the event starts at. */
const struct mips_event_code_map mips_event_codes[] = {
{ PMC_EV_MIPS24K_CYCLE, MIPS_CTR_ALL, 0},
{ PMC_EV_MIPS24K_INSTR_EXECUTED, MIPS_CTR_ALL, 1},
{ PMC_EV_MIPS24K_BRANCH_COMPLETED, MIPS_CTR_0, 2},
{ PMC_EV_MIPS24K_BRANCH_MISPRED, MIPS_CTR_1, 2},
{ PMC_EV_MIPS24K_RETURN, MIPS_CTR_0, 3},
{ PMC_EV_MIPS24K_RETURN_MISPRED, MIPS_CTR_1, 3},
{ PMC_EV_MIPS24K_RETURN_NOT_31, MIPS_CTR_0, 4},
{ PMC_EV_MIPS24K_RETURN_NOTPRED, MIPS_CTR_1, 4},
{ PMC_EV_MIPS24K_ITLB_ACCESS, MIPS_CTR_0, 5},
{ PMC_EV_MIPS24K_ITLB_MISS, MIPS_CTR_1, 5},
{ PMC_EV_MIPS24K_DTLB_ACCESS, MIPS_CTR_0, 6},
{ PMC_EV_MIPS24K_DTLB_MISS, MIPS_CTR_1, 6},
{ PMC_EV_MIPS24K_JTLB_IACCESS, MIPS_CTR_0, 7},
{ PMC_EV_MIPS24K_JTLB_IMISS, MIPS_CTR_1, 7},
{ PMC_EV_MIPS24K_JTLB_DACCESS, MIPS_CTR_0, 8},
{ PMC_EV_MIPS24K_JTLB_DMISS, MIPS_CTR_1, 8},
{ PMC_EV_MIPS24K_IC_FETCH, MIPS_CTR_0, 9},
{ PMC_EV_MIPS24K_IC_MISS, MIPS_CTR_1, 9},
{ PMC_EV_MIPS24K_DC_LOADSTORE, MIPS_CTR_0, 10},
{ PMC_EV_MIPS24K_DC_WRITEBACK, MIPS_CTR_1, 10},
{ PMC_EV_MIPS24K_DC_MISS, MIPS_CTR_ALL, 11},
/* 12 reserved */
{ PMC_EV_MIPS24K_STORE_MISS, MIPS_CTR_0, 13},
{ PMC_EV_MIPS24K_LOAD_MISS, MIPS_CTR_1, 13},
{ PMC_EV_MIPS24K_INTEGER_COMPLETED, MIPS_CTR_0, 14},
{ PMC_EV_MIPS24K_FP_COMPLETED, MIPS_CTR_1, 14},
{ PMC_EV_MIPS24K_LOAD_COMPLETED, MIPS_CTR_0, 15},
{ PMC_EV_MIPS24K_STORE_COMPLETED, MIPS_CTR_1, 15},
{ PMC_EV_MIPS24K_BARRIER_COMPLETED, MIPS_CTR_0, 16},
{ PMC_EV_MIPS24K_MIPS16_COMPLETED, MIPS_CTR_1, 16},
{ PMC_EV_MIPS24K_NOP_COMPLETED, MIPS_CTR_0, 17},
{ PMC_EV_MIPS24K_INTEGER_MULDIV_COMPLETED, MIPS_CTR_1, 17},
{ PMC_EV_MIPS24K_RF_STALL, MIPS_CTR_0, 18},
{ PMC_EV_MIPS24K_INSTR_REFETCH, MIPS_CTR_1, 18},
{ PMC_EV_MIPS24K_STORE_COND_COMPLETED, MIPS_CTR_0, 19},
{ PMC_EV_MIPS24K_STORE_COND_FAILED, MIPS_CTR_1, 19},
{ PMC_EV_MIPS24K_ICACHE_REQUESTS, MIPS_CTR_0, 20},
{ PMC_EV_MIPS24K_ICACHE_HIT, MIPS_CTR_1, 20},
{ PMC_EV_MIPS24K_L2_WRITEBACK, MIPS_CTR_0, 21},
{ PMC_EV_MIPS24K_L2_ACCESS, MIPS_CTR_1, 21},
{ PMC_EV_MIPS24K_L2_MISS, MIPS_CTR_0, 22},
{ PMC_EV_MIPS24K_L2_ERR_CORRECTED, MIPS_CTR_1, 22},
{ PMC_EV_MIPS24K_EXCEPTIONS, MIPS_CTR_0, 23},
/* Event 23 on COP0 1/3 is undefined */
{ PMC_EV_MIPS24K_RF_CYCLES_STALLED, MIPS_CTR_0, 24},
{ PMC_EV_MIPS24K_IFU_CYCLES_STALLED, MIPS_CTR_0, 25},
{ PMC_EV_MIPS24K_ALU_CYCLES_STALLED, MIPS_CTR_1, 25},
/* Events 26 through 32 undefined or reserved to customers */
{ PMC_EV_MIPS24K_UNCACHED_LOAD, MIPS_CTR_0, 33},
{ PMC_EV_MIPS24K_UNCACHED_STORE, MIPS_CTR_1, 33},
{ PMC_EV_MIPS24K_CP2_REG_TO_REG_COMPLETED, MIPS_CTR_0, 35},
{ PMC_EV_MIPS24K_MFTC_COMPLETED, MIPS_CTR_1, 35},
/* Event 36 reserved */
{ PMC_EV_MIPS24K_IC_BLOCKED_CYCLES, MIPS_CTR_0, 37},
{ PMC_EV_MIPS24K_DC_BLOCKED_CYCLES, MIPS_CTR_1, 37},
{ PMC_EV_MIPS24K_L2_IMISS_STALL_CYCLES, MIPS_CTR_0, 38},
{ PMC_EV_MIPS24K_L2_DMISS_STALL_CYCLES, MIPS_CTR_1, 38},
{ PMC_EV_MIPS24K_DMISS_CYCLES, MIPS_CTR_0, 39},
{ PMC_EV_MIPS24K_L2_MISS_CYCLES, MIPS_CTR_1, 39},
{ PMC_EV_MIPS24K_UNCACHED_BLOCK_CYCLES, MIPS_CTR_0, 40},
{ PMC_EV_MIPS24K_MDU_STALL_CYCLES, MIPS_CTR_0, 41},
{ PMC_EV_MIPS24K_FPU_STALL_CYCLES, MIPS_CTR_1, 41},
{ PMC_EV_MIPS24K_CP2_STALL_CYCLES, MIPS_CTR_0, 42},
{ PMC_EV_MIPS24K_COREXTEND_STALL_CYCLES, MIPS_CTR_1, 42},
{ PMC_EV_MIPS24K_ISPRAM_STALL_CYCLES, MIPS_CTR_0, 43},
{ PMC_EV_MIPS24K_DSPRAM_STALL_CYCLES, MIPS_CTR_1, 43},
{ PMC_EV_MIPS24K_CACHE_STALL_CYCLES, MIPS_CTR_0, 44},
/* Event 44 undefined on 1/3 */
{ PMC_EV_MIPS24K_LOAD_TO_USE_STALLS, MIPS_CTR_0, 45},
{ PMC_EV_MIPS24K_BASE_MISPRED_STALLS, MIPS_CTR_1, 45},
{ PMC_EV_MIPS24K_CPO_READ_STALLS, MIPS_CTR_0, 46},
{ PMC_EV_MIPS24K_BRANCH_MISPRED_CYCLES, MIPS_CTR_1, 46},
/* Event 47 reserved */
{ PMC_EV_MIPS24K_IFETCH_BUFFER_FULL, MIPS_CTR_0, 48},
{ PMC_EV_MIPS24K_FETCH_BUFFER_ALLOCATED, MIPS_CTR_1, 48},
{ PMC_EV_MIPS24K_EJTAG_ITRIGGER, MIPS_CTR_0, 49},
{ PMC_EV_MIPS24K_EJTAG_DTRIGGER, MIPS_CTR_1, 49},
{ PMC_EV_MIPS24K_FSB_LT_QUARTER, MIPS_CTR_0, 50},
{ PMC_EV_MIPS24K_FSB_QUARTER_TO_HALF, MIPS_CTR_1, 50},
{ PMC_EV_MIPS24K_FSB_GT_HALF, MIPS_CTR_0, 51},
{ PMC_EV_MIPS24K_FSB_FULL_PIPELINE_STALLS, MIPS_CTR_1, 51},
{ PMC_EV_MIPS24K_LDQ_LT_QUARTER, MIPS_CTR_0, 52},
{ PMC_EV_MIPS24K_LDQ_QUARTER_TO_HALF, MIPS_CTR_1, 52},
{ PMC_EV_MIPS24K_LDQ_GT_HALF, MIPS_CTR_0, 53},
{ PMC_EV_MIPS24K_LDQ_FULL_PIPELINE_STALLS, MIPS_CTR_1, 53},
{ PMC_EV_MIPS24K_WBB_LT_QUARTER, MIPS_CTR_0, 54},
{ PMC_EV_MIPS24K_WBB_QUARTER_TO_HALF, MIPS_CTR_1, 54},
{ PMC_EV_MIPS24K_WBB_GT_HALF, MIPS_CTR_0, 55},
{ PMC_EV_MIPS24K_WBB_FULL_PIPELINE_STALLS, MIPS_CTR_1, 55},
/* Events 56-63 reserved */
{ PMC_EV_MIPS24K_REQUEST_LATENCY, MIPS_CTR_0, 61},
{ PMC_EV_MIPS24K_REQUEST_COUNT, MIPS_CTR_1, 61}
};
const int mips_event_codes_size =
sizeof(mips_event_codes) / sizeof(mips_event_codes[0]);
struct mips_pmc_spec mips_pmc_spec = {
.ps_cpuclass = PMC_CLASS_MIPS24K,
.ps_cputype = PMC_CPU_MIPS_24K,
.ps_capabilities = MIPS24K_PMC_CAPS,
.ps_counter_width = 32
};
/*
* Performance Count Register N
*/
uint64_t
mips_pmcn_read(unsigned int pmc)
{
uint32_t reg = 0;
KASSERT(pmc < mips_npmcs, ("[mips24k,%d] illegal PMC number %d",
__LINE__, pmc));
/* The counter value is the next value after the control register. */
switch (pmc) {
case 0:
reg = mips_rd_perfcnt1();
break;
case 1:
reg = mips_rd_perfcnt3();
break;
default:
return 0;
}
return (reg);
}
uint64_t
mips_pmcn_write(unsigned int pmc, uint64_t reg)
{
KASSERT(pmc < mips_npmcs, ("[mips24k,%d] illegal PMC number %d",
__LINE__, pmc));
switch (pmc) {
case 0:
mips_wr_perfcnt1(reg);
break;
case 1:
mips_wr_perfcnt3(reg);
break;
default:
return 0;
}
return (reg);
}
uint32_t
mips_get_perfctl(int cpu, int ri, uint32_t event, uint32_t caps)
{
uint32_t config;
config = event;
config <<= MIPS24K_PMC_SELECT;
if (caps & PMC_CAP_SYSTEM)
config |= (MIPS24K_PMC_SUPER_ENABLE |
MIPS24K_PMC_KERNEL_ENABLE);
if (caps & PMC_CAP_USER)
config |= MIPS24K_PMC_USER_ENABLE;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
config |= MIPS24K_PMC_ENABLE;
if (caps & PMC_CAP_INTERRUPT)
config |= MIPS24K_PMC_INTERRUPT_ENABLE;
- PMCDBG(MDP,ALL,2,"mips24k-get_perfctl ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"mips24k-get_perfctl ri=%d -> config=0x%x", ri, config);
return (config);
}
Index: head/sys/dev/hwpmc/hwpmc_mod.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_mod.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_mod.c (revision 282658)
@@ -1,5117 +1,5117 @@
/*-
* Copyright (c) 2003-2008 Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/eventhandler.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/pmclog.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/vnode.h>
#include <sys/linker.h> /* needs to be after <sys/malloc.h> */
#include <machine/atomic.h>
#include <machine/md_var.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include "hwpmc_soft.h"
/*
* Types
*/
enum pmc_flags {
PMC_FLAG_NONE = 0x00, /* do nothing */
PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */
PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
};
/*
* The offset in sysent where the syscall is allocated.
*/
static int pmc_syscall_num = NO_SYSCALL;
struct pmc_cpu **pmc_pcpu; /* per-cpu state */
pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */
#define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
struct mtx_pool *pmc_mtxpool;
static int *pmc_pmcdisp; /* PMC row dispositions */
#define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0)
#define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0)
#define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0)
#define PMC_MARK_ROW_FREE(R) do { \
pmc_pmcdisp[(R)] = 0; \
} while (0)
#define PMC_MARK_ROW_STANDALONE(R) do { \
KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
__LINE__)); \
atomic_add_int(&pmc_pmcdisp[(R)], -1); \
KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()), \
("[pmc,%d] row disposition error", __LINE__)); \
} while (0)
#define PMC_UNMARK_ROW_STANDALONE(R) do { \
atomic_add_int(&pmc_pmcdisp[(R)], 1); \
KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
__LINE__)); \
} while (0)
#define PMC_MARK_ROW_THREAD(R) do { \
KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
__LINE__)); \
atomic_add_int(&pmc_pmcdisp[(R)], 1); \
} while (0)
#define PMC_UNMARK_ROW_THREAD(R) do { \
atomic_add_int(&pmc_pmcdisp[(R)], -1); \
KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
__LINE__)); \
} while (0)
/* various event handlers */
static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
pmc_kld_unload_tag;
/* Module statistics */
struct pmc_op_getdriverstats pmc_stats;
/* Machine/processor dependent operations */
static struct pmc_mdep *md;
/*
* Hash tables mapping owner processes and target threads to PMCs.
*/
struct mtx pmc_processhash_mtx; /* spin mutex */
static u_long pmc_processhashmask;
static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash;
/*
* Hash table of PMC owner descriptors. This table is protected by
* the shared PMC "sx" lock.
*/
static u_long pmc_ownerhashmask;
static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash;
/*
* List of PMC owners with system-wide sampling PMCs.
*/
static LIST_HEAD(, pmc_owner) pmc_ss_owners;
/*
* A map of row indices to classdep structures.
*/
static struct pmc_classdep **pmc_rowindex_to_classdep;
/*
* Prototypes
*/
#ifdef HWPMC_DEBUG
static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
static int pmc_debugflags_parse(char *newstr, char *fence);
#endif
static int load(struct module *module, int cmd, void *arg);
static int pmc_attach_process(struct proc *p, struct pmc *pm);
static struct pmc *pmc_allocate_pmc_descriptor(void);
static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
int cpu);
static int pmc_can_attach(struct pmc *pm, struct proc *p);
static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
static void pmc_cleanup(void);
static int pmc_detach_process(struct proc *p, struct pmc *pm);
static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
int flags);
static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
static void pmc_destroy_pmc_descriptor(struct pmc *pm);
static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
pmc_id_t pmc);
static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
uint32_t mode);
static void pmc_force_context_switch(void);
static void pmc_link_target_process(struct pmc *pm,
struct pmc_process *pp);
static void pmc_log_all_process_mappings(struct pmc_owner *po);
static void pmc_log_kernel_mappings(struct pmc *pm);
static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
static void pmc_maybe_remove_owner(struct pmc_owner *po);
static void pmc_process_csw_in(struct thread *td);
static void pmc_process_csw_out(struct thread *td);
static void pmc_process_exit(void *arg, struct proc *p);
static void pmc_process_fork(void *arg, struct proc *p1,
struct proc *p2, int n);
static void pmc_process_samples(int cpu, int soft);
static void pmc_release_pmc_descriptor(struct pmc *pmc);
static void pmc_remove_owner(struct pmc_owner *po);
static void pmc_remove_process_descriptor(struct pmc_process *pp);
static void pmc_restore_cpu_binding(struct pmc_binding *pb);
static void pmc_save_cpu_binding(struct pmc_binding *pb);
static void pmc_select_cpu(int cpu);
static int pmc_start(struct pmc *pm);
static int pmc_stop(struct pmc *pm);
static int pmc_syscall_handler(struct thread *td, void *syscall_args);
static void pmc_unlink_target_process(struct pmc *pmc,
struct pmc_process *pp);
static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
static struct pmc_mdep *pmc_generic_cpu_initialize(void);
static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
/*
* Kernel tunables and sysctl(8) interface.
*/
SYSCTL_DECL(_kern_hwpmc);
static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
&pmc_callchaindepth, 0, "depth of call chain records");
#ifdef HWPMC_DEBUG
struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
char pmc_debugstr[PMC_DEBUG_STRSIZE];
TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
sizeof(pmc_debugstr));
SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
#endif
/*
* kern.hwpmc.hashrows -- determines the number of rows in the
* of the hash table used to look up threads
*/
static int pmc_hashsize = PMC_HASH_SIZE;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
&pmc_hashsize, 0, "rows in hash tables");
/*
* kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
*/
static int pmc_nsamples = PMC_NSAMPLES;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
&pmc_nsamples, 0, "number of PC samples per CPU");
/*
* kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
*/
static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
&pmc_mtxpool_size, 0, "size of spin mutex pool");
/*
* security.bsd.unprivileged_syspmcs -- allow non-root processes to
* allocate system-wide PMCs.
*
* Allowing unprivileged processes to allocate system PMCs is convenient
* if system-wide measurements need to be taken concurrently with other
* per-process measurements. This feature is turned off by default.
*/
static int pmc_unprivileged_syspmcs = 0;
SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
&pmc_unprivileged_syspmcs, 0,
"allow unprivileged process to allocate system PMCs");
/*
* Hash function. Discard the lower 2 bits of the pointer since
* these are always zero for our uses. The hash multiplier is
* round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
*/
#if LONG_BIT == 64
#define _PMC_HM 11400714819323198486u
#elif LONG_BIT == 32
#define _PMC_HM 2654435769u
#else
#error Must know the size of 'long' to compile
#endif
#define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
/*
* Syscall structures
*/
/* The `sysent' for the new syscall */
static struct sysent pmc_sysent = {
2, /* sy_narg */
pmc_syscall_handler /* sy_call */
};
static struct syscall_module_data pmc_syscall_mod = {
load,
NULL,
&pmc_syscall_num,
&pmc_sysent,
#if (__FreeBSD_version >= 1100000)
{ 0, NULL },
SY_THR_STATIC_KLD,
#else
{ 0, NULL }
#endif
};
static moduledata_t pmc_mod = {
PMC_MODULE_NAME,
syscall_module_handler,
&pmc_syscall_mod
};
DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
MODULE_VERSION(pmc, PMC_VERSION);
#ifdef HWPMC_DEBUG
enum pmc_dbgparse_state {
PMCDS_WS, /* in whitespace */
PMCDS_MAJOR, /* seen a major keyword */
PMCDS_MINOR
};
static int
pmc_debugflags_parse(char *newstr, char *fence)
{
char c, *p, *q;
struct pmc_debugflags *tmpflags;
int error, found, *newbits, tmp;
size_t kwlen;
tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO);
p = newstr;
error = 0;
for (; p < fence && (c = *p); p++) {
/* skip white space */
if (c == ' ' || c == '\t')
continue;
/* look for a keyword followed by "=" */
for (q = p; p < fence && (c = *p) && c != '='; p++)
;
if (c != '=') {
error = EINVAL;
goto done;
}
kwlen = p - q;
newbits = NULL;
/* lookup flag group name */
#define DBG_SET_FLAG_MAJ(S,F) \
if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
newbits = &tmpflags->pdb_ ## F;
DBG_SET_FLAG_MAJ("cpu", CPU);
DBG_SET_FLAG_MAJ("csw", CSW);
DBG_SET_FLAG_MAJ("logging", LOG);
DBG_SET_FLAG_MAJ("module", MOD);
DBG_SET_FLAG_MAJ("md", MDP);
DBG_SET_FLAG_MAJ("owner", OWN);
DBG_SET_FLAG_MAJ("pmc", PMC);
DBG_SET_FLAG_MAJ("process", PRC);
DBG_SET_FLAG_MAJ("sampling", SAM);
if (newbits == NULL) {
error = EINVAL;
goto done;
}
p++; /* skip the '=' */
/* Now parse the individual flags */
tmp = 0;
newflag:
for (q = p; p < fence && (c = *p); p++)
if (c == ' ' || c == '\t' || c == ',')
break;
/* p == fence or c == ws or c == "," or c == 0 */
if ((kwlen = p - q) == 0) {
*newbits = tmp;
continue;
}
found = 0;
#define DBG_SET_FLAG_MIN(S,F) \
if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
/* a '*' denotes all possible flags in the group */
if (kwlen == 1 && *q == '*')
tmp = found = ~0;
/* look for individual flag names */
DBG_SET_FLAG_MIN("allocaterow", ALR);
DBG_SET_FLAG_MIN("allocate", ALL);
DBG_SET_FLAG_MIN("attach", ATT);
DBG_SET_FLAG_MIN("bind", BND);
DBG_SET_FLAG_MIN("config", CFG);
DBG_SET_FLAG_MIN("exec", EXC);
DBG_SET_FLAG_MIN("exit", EXT);
DBG_SET_FLAG_MIN("find", FND);
DBG_SET_FLAG_MIN("flush", FLS);
DBG_SET_FLAG_MIN("fork", FRK);
DBG_SET_FLAG_MIN("getbuf", GTB);
DBG_SET_FLAG_MIN("hook", PMH);
DBG_SET_FLAG_MIN("init", INI);
DBG_SET_FLAG_MIN("intr", INT);
DBG_SET_FLAG_MIN("linktarget", TLK);
DBG_SET_FLAG_MIN("mayberemove", OMR);
DBG_SET_FLAG_MIN("ops", OPS);
DBG_SET_FLAG_MIN("read", REA);
DBG_SET_FLAG_MIN("register", REG);
DBG_SET_FLAG_MIN("release", REL);
DBG_SET_FLAG_MIN("remove", ORM);
DBG_SET_FLAG_MIN("sample", SAM);
DBG_SET_FLAG_MIN("scheduleio", SIO);
DBG_SET_FLAG_MIN("select", SEL);
DBG_SET_FLAG_MIN("signal", SIG);
DBG_SET_FLAG_MIN("swi", SWI);
DBG_SET_FLAG_MIN("swo", SWO);
DBG_SET_FLAG_MIN("start", STA);
DBG_SET_FLAG_MIN("stop", STO);
DBG_SET_FLAG_MIN("syscall", PMS);
DBG_SET_FLAG_MIN("unlinktarget", TUL);
DBG_SET_FLAG_MIN("write", WRI);
if (found == 0) {
/* unrecognized flag name */
error = EINVAL;
goto done;
}
if (c == 0 || c == ' ' || c == '\t') { /* end of flag group */
*newbits = tmp;
continue;
}
p++;
goto newflag;
}
/* save the new flag set */
bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
done:
free(tmpflags, M_PMC);
return error;
}
static int
pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
{
char *fence, *newstr;
int error;
unsigned int n;
(void) arg1; (void) arg2; /* unused parameters */
n = sizeof(pmc_debugstr);
newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO);
(void) strlcpy(newstr, pmc_debugstr, n);
error = sysctl_handle_string(oidp, newstr, n, req);
/* if there is a new string, parse and copy it */
if (error == 0 && req->newptr != NULL) {
fence = newstr + (n < req->newlen ? n : req->newlen + 1);
if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
(void) strlcpy(pmc_debugstr, newstr,
sizeof(pmc_debugstr));
}
free(newstr, M_PMC);
return error;
}
#endif
/*
* Map a row index to a classdep structure and return the adjusted row
* index for the PMC class index.
*/
static struct pmc_classdep *
pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri)
{
struct pmc_classdep *pcd;
(void) md;
KASSERT(ri >= 0 && ri < md->pmd_npmc,
("[pmc,%d] illegal row-index %d", __LINE__, ri));
pcd = pmc_rowindex_to_classdep[ri];
KASSERT(pcd != NULL,
("[pmc,%d] ri %d null pcd", __LINE__, ri));
*adjri = ri - pcd->pcd_ri;
KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num,
("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));
return (pcd);
}
/*
* Concurrency Control
*
* The driver manages the following data structures:
*
* - target process descriptors, one per target process
* - owner process descriptors (and attached lists), one per owner process
* - lookup hash tables for owner and target processes
* - PMC descriptors (and attached lists)
* - per-cpu hardware state
* - the 'hook' variable through which the kernel calls into
* this module
* - the machine hardware state (managed by the MD layer)
*
* These data structures are accessed from:
*
* - thread context-switch code
* - interrupt handlers (possibly on multiple cpus)
* - kernel threads on multiple cpus running on behalf of user
* processes doing system calls
* - this driver's private kernel threads
*
* = Locks and Locking strategy =
*
* The driver uses four locking strategies for its operation:
*
* - The global SX lock "pmc_sx" is used to protect internal
* data structures.
*
* Calls into the module by syscall() start with this lock being
* held in exclusive mode. Depending on the requested operation,
* the lock may be downgraded to 'shared' mode to allow more
* concurrent readers into the module. Calls into the module from
* other parts of the kernel acquire the lock in shared mode.
*
* This SX lock is held in exclusive mode for any operations that
* modify the linkages between the driver's internal data structures.
*
* The 'pmc_hook' function pointer is also protected by this lock.
* It is only examined with the sx lock held in exclusive mode. The
* kernel module is allowed to be unloaded only with the sx lock held
* in exclusive mode. In normal syscall handling, after acquiring the
* pmc_sx lock we first check that 'pmc_hook' is non-null before
* proceeding. This prevents races between the thread unloading the module
* and other threads seeking to use the module.
*
* - Lookups of target process structures and owner process structures
* cannot use the global "pmc_sx" SX lock because these lookups need
* to happen during context switches and in other critical sections
* where sleeping is not allowed. We protect these lookup tables
* with their own private spin-mutexes, "pmc_processhash_mtx" and
* "pmc_ownerhash_mtx".
*
* - Interrupt handlers work in a lock free manner. At interrupt
* time, handlers look at the PMC pointer (phw->phw_pmc) configured
* when the PMC was started. If this pointer is NULL, the interrupt
* is ignored after updating driver statistics. We ensure that this
* pointer is set (using an atomic operation if necessary) before the
* PMC hardware is started. Conversely, this pointer is unset atomically
* only after the PMC hardware is stopped.
*
* We ensure that everything needed for the operation of an
* interrupt handler is available without it needing to acquire any
* locks. We also ensure that a PMC's software state is destroyed only
* after the PMC is taken off hardware (on all CPUs).
*
* - Context-switch handling with process-private PMCs needs more
* care.
*
* A given process may be the target of multiple PMCs. For example,
* PMCATTACH and PMCDETACH may be requested by a process on one CPU
* while the target process is running on another. A PMC could also
* be getting released because its owner is exiting. We tackle
* these situations in the following manner:
*
* - each target process structure 'pmc_process' has an array
* of 'struct pmc *' pointers, one for each hardware PMC.
*
* - At context switch IN time, each "target" PMC in RUNNING state
* gets started on hardware and a pointer to each PMC is copied into
* the per-cpu phw array. The 'runcount' for the PMC is
* incremented.
*
* - At context switch OUT time, all process-virtual PMCs are stopped
* on hardware. The saved value is added to the PMCs value field
* only if the PMC is in a non-deleted state (the PMCs state could
* have changed during the current time slice).
*
* Note that since in-between a switch IN on a processor and a switch
* OUT, the PMC could have been released on another CPU. Therefore
* context switch OUT always looks at the hardware state to turn
* OFF PMCs and will update a PMC's saved value only if reachable
* from the target process record.
*
* - OP PMCRELEASE could be called on a PMC at any time (the PMC could
* be attached to many processes at the time of the call and could
* be active on multiple CPUs).
*
* We prevent further scheduling of the PMC by marking it as in
* state 'DELETED'. If the runcount of the PMC is non-zero then
* this PMC is currently running on a CPU somewhere. The thread
* doing the PMCRELEASE operation waits by repeatedly doing a
* pause() till the runcount comes to zero.
*
* The contents of a PMC descriptor (struct pmc) are protected using
* a spin-mutex. In order to save space, we use a mutex pool.
*
* In terms of lock types used by witness(4), we use:
* - Type "pmc-sx", used by the global SX lock.
* - Type "pmc-sleep", for sleep mutexes used by logger threads.
* - Type "pmc-per-proc", for protecting PMC owner descriptors.
* - Type "pmc-leaf", used for all other spin mutexes.
*/
/*
* save the cpu binding of the current kthread
*/
static void
pmc_save_cpu_binding(struct pmc_binding *pb)
{
- PMCDBG(CPU,BND,2, "%s", "save-cpu");
+ PMCDBG0(CPU,BND,2, "save-cpu");
thread_lock(curthread);
pb->pb_bound = sched_is_bound(curthread);
pb->pb_cpu = curthread->td_oncpu;
thread_unlock(curthread);
- PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
+ PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
}
/*
* restore the cpu binding of the current thread
*/
static void
pmc_restore_cpu_binding(struct pmc_binding *pb)
{
- PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
+ PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
curthread->td_oncpu, pb->pb_cpu);
thread_lock(curthread);
if (pb->pb_bound)
sched_bind(curthread, pb->pb_cpu);
else
sched_unbind(curthread);
thread_unlock(curthread);
- PMCDBG(CPU,BND,2, "%s", "restore-cpu done");
+ PMCDBG0(CPU,BND,2, "restore-cpu done");
}
/*
* move execution over the specified cpu and bind it there.
*/
static void
pmc_select_cpu(int cpu)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[pmc,%d] bad cpu number %d", __LINE__, cpu));
/* Never move to an inactive CPU. */
KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
"CPU %d", __LINE__, cpu));
- PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu);
+ PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
thread_lock(curthread);
sched_bind(curthread, cpu);
thread_unlock(curthread);
KASSERT(curthread->td_oncpu == cpu,
("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
cpu, curthread->td_oncpu));
- PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
+ PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
}
/*
* Force a context switch.
*
* We do this by pause'ing for 1 tick -- invoking mi_switch() is not
* guaranteed to force a context switch.
*/
static void
pmc_force_context_switch(void)
{
pause("pmcctx", 1);
}
/*
* Get the file name for an executable. This is a simple wrapper
* around vn_fullpath(9).
*/
static void
pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
{
*fullpath = "unknown";
*freepath = NULL;
vn_fullpath(curthread, v, fullpath, freepath);
}
/*
* remove an process owning PMCs
*/
void
pmc_remove_owner(struct pmc_owner *po)
{
struct pmc *pm, *tmp;
sx_assert(&pmc_sx, SX_XLOCKED);
- PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
+ PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);
/* Remove descriptor from the owner hash table */
LIST_REMOVE(po, po_next);
/* release all owned PMC descriptors */
LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
- PMCDBG(OWN,ORM,2, "pmc=%p", pm);
+ PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
KASSERT(pm->pm_owner == po,
("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
pmc_release_pmc_descriptor(pm); /* will unlink from the list */
pmc_destroy_pmc_descriptor(pm);
}
KASSERT(po->po_sscount == 0,
("[pmc,%d] SS count not zero", __LINE__));
KASSERT(LIST_EMPTY(&po->po_pmcs),
("[pmc,%d] PMC list not empty", __LINE__));
/* de-configure the log file if present */
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_deconfigure_log(po);
}
/*
* remove an owner process record if all conditions are met.
*/
static void
pmc_maybe_remove_owner(struct pmc_owner *po)
{
- PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po);
+ PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);
/*
* Remove owner record if
* - this process does not own any PMCs
* - this process has not allocated a system-wide sampling buffer
*/
if (LIST_EMPTY(&po->po_pmcs) &&
((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
pmc_remove_owner(po);
pmc_destroy_owner_descriptor(po);
}
}
/*
* Add an association between a target process and a PMC.
*/
static void
pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
{
int ri;
struct pmc_target *pt;
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pm != NULL && pp != NULL,
("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
__LINE__, pm, pp->pp_proc->p_pid));
KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
("[pmc,%d] Illegal reference count %d for process record %p",
__LINE__, pp->pp_refcnt, (void *) pp));
ri = PMC_TO_ROWINDEX(pm);
- PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
+ PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
pm, ri, pp);
#ifdef HWPMC_DEBUG
LIST_FOREACH(pt, &pm->pm_targets, pt_next)
if (pt->pt_process == pp)
KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
__LINE__, pp, pm));
#endif
pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO);
pt->pt_process = pp;
LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
(uintptr_t)pm);
if (pm->pm_owner->po_owner == pp->pp_proc)
pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
/*
* Initialize the per-process values at this row index.
*/
pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
pm->pm_sc.pm_reloadcount : 0;
pp->pp_refcnt++;
}
/*
* Removes the association between a target process and a PMC.
*/
static void
pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
{
int ri;
struct proc *p;
struct pmc_target *ptgt;
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pm != NULL && pp != NULL,
("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
("[pmc,%d] Illegal ref count %d on process record %p",
__LINE__, pp->pp_refcnt, (void *) pp));
ri = PMC_TO_ROWINDEX(pm);
- PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
+ PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
pm, ri, pp);
KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
ri, pm, pp->pp_pmcs[ri].pp_pmc));
pp->pp_pmcs[ri].pp_pmc = NULL;
pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
/* Remove owner-specific flags */
if (pm->pm_owner->po_owner == pp->pp_proc) {
pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
}
pp->pp_refcnt--;
/* Remove the target process from the PMC structure */
LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
if (ptgt->pt_process == pp)
break;
KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
"in pmc %p", __LINE__, pp->pp_proc, pp, pm));
LIST_REMOVE(ptgt, pt_next);
free(ptgt, M_PMC);
/* if the PMC now lacks targets, send the owner a SIGIO */
if (LIST_EMPTY(&pm->pm_targets)) {
p = pm->pm_owner->po_owner;
PROC_LOCK(p);
kern_psignal(p, SIGIO);
PROC_UNLOCK(p);
- PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
+ PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
SIGIO);
}
}
/*
* Check if PMC 'pm' may be attached to target process 't'.
*/
static int
pmc_can_attach(struct pmc *pm, struct proc *t)
{
struct proc *o; /* pmc owner */
struct ucred *oc, *tc; /* owner, target credentials */
int decline_attach, i;
/*
* A PMC's owner can always attach that PMC to itself.
*/
if ((o = pm->pm_owner->po_owner) == t)
return 0;
PROC_LOCK(o);
oc = o->p_ucred;
crhold(oc);
PROC_UNLOCK(o);
PROC_LOCK(t);
tc = t->p_ucred;
crhold(tc);
PROC_UNLOCK(t);
/*
* The effective uid of the PMC owner should match at least one
* of the {effective,real,saved} uids of the target process.
*/
decline_attach = oc->cr_uid != tc->cr_uid &&
oc->cr_uid != tc->cr_svuid &&
oc->cr_uid != tc->cr_ruid;
/*
* Every one of the target's group ids, must be in the owner's
* group list.
*/
for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
decline_attach = !groupmember(tc->cr_groups[i], oc);
/* check the read and saved gids too */
if (decline_attach == 0)
decline_attach = !groupmember(tc->cr_rgid, oc) ||
!groupmember(tc->cr_svgid, oc);
crfree(tc);
crfree(oc);
return !decline_attach;
}
/*
* Attach a process to a PMC.
*/
static int
pmc_attach_one_process(struct proc *p, struct pmc *pm)
{
int ri;
char *fullpath, *freepath;
struct pmc_process *pp;
sx_assert(&pmc_sx, SX_XLOCKED);
- PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
+ PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
/*
* Locate the process descriptor corresponding to process 'p',
* allocating space as needed.
*
* Verify that rowindex 'pm_rowindex' is free in the process
* descriptor.
*
* If not, allocate space for a descriptor and link the
* process descriptor and PMC.
*/
ri = PMC_TO_ROWINDEX(pm);
if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
return ENOMEM;
if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
return EEXIST;
if (pp->pp_pmcs[ri].pp_pmc != NULL)
return EBUSY;
pmc_link_target_process(pm, pp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
(pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
/* issue an attach event to a configured log file */
if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
pmc_getfilename(p->p_textvp, &fullpath, &freepath);
if (p->p_flag & P_KTHREAD) {
fullpath = kernelname;
freepath = NULL;
} else
pmclog_process_pmcattach(pm, p->p_pid, fullpath);
if (freepath)
free(freepath, M_TEMP);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pmc_log_process_mappings(pm->pm_owner, p);
}
/* mark process as using HWPMCs */
PROC_LOCK(p);
p->p_flag |= P_HWPMC;
PROC_UNLOCK(p);
return 0;
}
/*
* Attach a process and optionally its children
*/
static int
pmc_attach_process(struct proc *p, struct pmc *pm)
{
int error;
struct proc *top;
sx_assert(&pmc_sx, SX_XLOCKED);
- PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
+ PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
/*
* If this PMC successfully allowed a GETMSR operation
* in the past, disallow further ATTACHes.
*/
if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
return EPERM;
if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
return pmc_attach_one_process(p, pm);
/*
* Traverse all child processes, attaching them to
* this PMC.
*/
sx_slock(&proctree_lock);
top = p;
for (;;) {
if ((error = pmc_attach_one_process(p, pm)) != 0)
break;
if (!LIST_EMPTY(&p->p_children))
p = LIST_FIRST(&p->p_children);
else for (;;) {
if (p == top)
goto done;
if (LIST_NEXT(p, p_sibling)) {
p = LIST_NEXT(p, p_sibling);
break;
}
p = p->p_pptr;
}
}
if (error)
(void) pmc_detach_process(top, pm);
done:
sx_sunlock(&proctree_lock);
return error;
}
/*
* Detach a process from a PMC. If there are no other PMCs tracking
* this process, remove the process structure from its hash table. If
* 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
*/
static int
pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
{
int ri;
struct pmc_process *pp;
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pm != NULL,
("[pmc,%d] null pm pointer", __LINE__));
ri = PMC_TO_ROWINDEX(pm);
- PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
+ PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
pm, ri, p, p->p_pid, p->p_comm, flags);
if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
return ESRCH;
if (pp->pp_pmcs[ri].pp_pmc != pm)
return EINVAL;
pmc_unlink_target_process(pm, pp);
/* Issue a detach entry if a log file is configured */
if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_pmcdetach(pm, p->p_pid);
/*
* If there are no PMCs targetting this process, we remove its
* descriptor from the target hash table and unset the P_HWPMC
* flag in the struct proc.
*/
KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
("[pmc,%d] Illegal refcnt %d for process struct %p",
__LINE__, pp->pp_refcnt, pp));
if (pp->pp_refcnt != 0) /* still a target of some PMC */
return 0;
pmc_remove_process_descriptor(pp);
if (flags & PMC_FLAG_REMOVE)
free(pp, M_PMC);
PROC_LOCK(p);
p->p_flag &= ~P_HWPMC;
PROC_UNLOCK(p);
return 0;
}
/*
* Detach a process and optionally its descendants from a PMC.
*/
static int
pmc_detach_process(struct proc *p, struct pmc *pm)
{
struct proc *top;
sx_assert(&pmc_sx, SX_XLOCKED);
- PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
+ PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
/*
* Traverse all children, detaching them from this PMC. We
* ignore errors since we could be detaching a PMC from a
* partially attached proc tree.
*/
sx_slock(&proctree_lock);
top = p;
for (;;) {
(void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
if (!LIST_EMPTY(&p->p_children))
p = LIST_FIRST(&p->p_children);
else for (;;) {
if (p == top)
goto done;
if (LIST_NEXT(p, p_sibling)) {
p = LIST_NEXT(p, p_sibling);
break;
}
p = p->p_pptr;
}
}
done:
sx_sunlock(&proctree_lock);
if (LIST_EMPTY(&pm->pm_targets))
pm->pm_flags &= ~PMC_F_ATTACH_DONE;
return 0;
}
/*
* Thread context switch IN
*/
static void
pmc_process_csw_in(struct thread *td)
{
int cpu;
unsigned int adjri, ri;
struct pmc *pm;
struct proc *p;
struct pmc_cpu *pc;
struct pmc_hw *phw;
pmc_value_t newvalue;
struct pmc_process *pp;
struct pmc_classdep *pcd;
p = td->td_proc;
if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
return;
KASSERT(pp->pp_proc == td->td_proc,
("[pmc,%d] not my thread state", __LINE__));
critical_enter(); /* no preemption from this point */
cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
- PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
+ PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
p->p_pid, p->p_comm, pp);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[pmc,%d] wierd CPU id %d", __LINE__, cpu));
pc = pmc_pcpu[cpu];
for (ri = 0; ri < md->pmd_npmc; ri++) {
if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
continue;
KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
("[pmc,%d] Target PMC in non-virtual mode (%d)",
__LINE__, PMC_TO_MODE(pm)));
KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] Row index mismatch pmc %d != ri %d",
__LINE__, PMC_TO_ROWINDEX(pm), ri));
/*
* Only PMCs that are marked as 'RUNNING' need
* be placed on hardware.
*/
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* increment PMC runcount */
atomic_add_rel_int(&pm->pm_runcount, 1);
/* configure the HWPMC we are going to use. */
pcd = pmc_ri_to_classdep(md, ri, &adjri);
pcd->pcd_config_pmc(cpu, adjri, pm);
phw = pc->pc_hwpmcs[ri];
KASSERT(phw != NULL,
("[pmc,%d] null hw pointer", __LINE__));
KASSERT(phw->phw_pmc == pm,
("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
phw->phw_pmc, pm));
/*
* Write out saved value and start the PMC.
*
* Sampling PMCs use a per-process value, while
* counting mode PMCs use a per-pmc value that is
* inherited across descendants.
*/
if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
mtx_pool_lock_spin(pmc_mtxpool, pm);
newvalue = PMC_PCPU_SAVED(cpu,ri) =
pp->pp_pmcs[ri].pp_pmcval;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
} else {
KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
("[pmc,%d] illegal mode=%d", __LINE__,
PMC_TO_MODE(pm)));
mtx_pool_lock_spin(pmc_mtxpool, pm);
newvalue = PMC_PCPU_SAVED(cpu, ri) =
pm->pm_gv.pm_savedvalue;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
}
- PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
+ PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
pcd->pcd_write_pmc(cpu, adjri, newvalue);
pcd->pcd_start_pmc(cpu, adjri);
}
/*
* perform any other architecture/cpu dependent thread
* switch-in actions.
*/
(void) (*md->pmd_switch_in)(pc, pp);
critical_exit();
}
/*
* Thread context switch OUT.
*/
static void
pmc_process_csw_out(struct thread *td)
{
int cpu;
int64_t tmp;
struct pmc *pm;
struct proc *p;
enum pmc_mode mode;
struct pmc_cpu *pc;
pmc_value_t newvalue;
unsigned int adjri, ri;
struct pmc_process *pp;
struct pmc_classdep *pcd;
/*
* Locate our process descriptor; this may be NULL if
* this process is exiting and we have already removed
* the process from the target process table.
*
* Note that due to kernel preemption, multiple
* context switches may happen while the process is
* exiting.
*
* Note also that if the target process cannot be
* found we still need to deconfigure any PMCs that
* are currently running on hardware.
*/
p = td->td_proc;
pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
/*
* save PMCs
*/
critical_enter();
cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
- PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
+ PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
p->p_pid, p->p_comm, pp);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[pmc,%d wierd CPU id %d", __LINE__, cpu));
pc = pmc_pcpu[cpu];
/*
* When a PMC gets unlinked from a target PMC, it will
* be removed from the target's pp_pmc[] array.
*
* However, on a MP system, the target could have been
* executing on another CPU at the time of the unlink.
* So, at context switch OUT time, we need to look at
* the hardware to determine if a PMC is scheduled on
* it.
*/
for (ri = 0; ri < md->pmd_npmc; ri++) {
pcd = pmc_ri_to_classdep(md, ri, &adjri);
pm = NULL;
(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
if (pm == NULL) /* nothing at this row index */
continue;
mode = PMC_TO_MODE(pm);
if (!PMC_IS_VIRTUAL_MODE(mode))
continue; /* not a process virtual PMC */
KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
__LINE__, PMC_TO_ROWINDEX(pm), ri));
/* Stop hardware if not already stopped */
if (pm->pm_stalled == 0)
pcd->pcd_stop_pmc(cpu, adjri);
/* reduce this PMC's runcount */
atomic_subtract_rel_int(&pm->pm_runcount, 1);
/*
* If this PMC is associated with this process,
* save the reading.
*/
if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
pm, ri, pp->pp_pmcs[ri].pp_pmc));
KASSERT(pp->pp_refcnt > 0,
("[pmc,%d] pp refcnt = %d", __LINE__,
pp->pp_refcnt));
pcd->pcd_read_pmc(cpu, adjri, &newvalue);
tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
- PMCDBG(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
+ PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
tmp);
if (mode == PMC_MODE_TS) {
/*
* For sampling process-virtual PMCs,
* we expect the count to be
* decreasing as the 'value'
* programmed into the PMC is the
* number of events to be seen till
* the next sampling interrupt.
*/
if (tmp < 0)
tmp += pm->pm_sc.pm_reloadcount;
mtx_pool_lock_spin(pmc_mtxpool, pm);
pp->pp_pmcs[ri].pp_pmcval -= tmp;
if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
pp->pp_pmcs[ri].pp_pmcval +=
pm->pm_sc.pm_reloadcount;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
} else {
/*
* For counting process-virtual PMCs,
* we expect the count to be
* increasing monotonically, modulo a 64
* bit wraparound.
*/
KASSERT((int64_t) tmp >= 0,
("[pmc,%d] negative increment cpu=%d "
"ri=%d newvalue=%jx saved=%jx "
"incr=%jx", __LINE__, cpu, ri,
newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
mtx_pool_lock_spin(pmc_mtxpool, pm);
pm->pm_gv.pm_savedvalue += tmp;
pp->pp_pmcs[ri].pp_pmcval += tmp;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
if (pm->pm_flags & PMC_F_LOG_PROCCSW)
pmclog_process_proccsw(pm, pp, tmp);
}
}
/* mark hardware as free */
pcd->pcd_config_pmc(cpu, adjri, NULL);
}
/*
* perform any other architecture/cpu dependent thread
* switch out functions.
*/
(void) (*md->pmd_switch_out)(pc, pp);
critical_exit();
}
/*
* A mapping change for a process.
*/
static void
pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm)
{
int ri;
pid_t pid;
char *fullpath, *freepath;
const struct pmc *pm;
struct pmc_owner *po;
const struct pmc_process *pp;
freepath = fullpath = NULL;
pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);
pid = td->td_proc->p_pid;
/* Inform owners of all system-wide sampling PMCs. */
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);
if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
goto done;
/*
* Inform sampling PMC owners tracking this process.
*/
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pmclog_process_map_in(pm->pm_owner,
pid, pkm->pm_address, fullpath);
done:
if (freepath)
free(freepath, M_TEMP);
}
/*
* Log an munmap request.
*/
static void
pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm)
{
int ri;
pid_t pid;
struct pmc_owner *po;
const struct pmc *pm;
const struct pmc_process *pp;
pid = td->td_proc->p_pid;
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_map_out(po, pid, pkm->pm_address,
pkm->pm_address + pkm->pm_size);
if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
return;
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pmclog_process_map_out(pm->pm_owner, pid,
pkm->pm_address, pkm->pm_address + pkm->pm_size);
}
/*
* Log mapping information about the kernel.
*/
static void
pmc_log_kernel_mappings(struct pmc *pm)
{
struct pmc_owner *po;
struct pmckern_map_in *km, *kmbase;
sx_assert(&pmc_sx, SX_LOCKED);
KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
("[pmc,%d] non-sampling PMC (%p) desires mapping information",
__LINE__, (void *) pm));
po = pm->pm_owner;
if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
return;
/*
* Log the current set of kernel modules.
*/
kmbase = linker_hwpmc_list_objects();
for (km = kmbase; km->pm_file != NULL; km++) {
- PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file,
+ PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
(void *) km->pm_address);
pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
km->pm_file);
}
free(kmbase, M_LINKER);
po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
}
/*
* Log the mappings for a single process.
*/
static void
pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
{
vm_map_t map;
struct vnode *vp;
struct vmspace *vm;
vm_map_entry_t entry;
vm_offset_t last_end;
u_int last_timestamp;
struct vnode *last_vp;
vm_offset_t start_addr;
vm_object_t obj, lobj, tobj;
char *fullpath, *freepath;
last_vp = NULL;
last_end = (vm_offset_t) 0;
fullpath = freepath = NULL;
if ((vm = vmspace_acquire_ref(p)) == NULL)
return;
map = &vm->vm_map;
vm_map_lock_read(map);
for (entry = map->header.next; entry != &map->header; entry = entry->next) {
if (entry == NULL) {
- PMCDBG(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
+ PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
"NULL! pid=%d vm_map=%p\n", p->p_pid, map);
break;
}
/*
* We only care about executable map entries.
*/
if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) ||
!(entry->protection & VM_PROT_EXECUTE) ||
(entry->object.vm_object == NULL)) {
continue;
}
obj = entry->object.vm_object;
VM_OBJECT_RLOCK(obj);
/*
* Walk the backing_object list to find the base
* (non-shadowed) vm_object.
*/
for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
if (tobj != obj)
VM_OBJECT_RLOCK(tobj);
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
lobj = tobj;
}
/*
* At this point lobj is the base vm_object and it is locked.
*/
if (lobj == NULL) {
- PMCDBG(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
+ PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
"vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
VM_OBJECT_RUNLOCK(obj);
continue;
}
if (lobj->type != OBJT_VNODE || lobj->handle == NULL) {
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
VM_OBJECT_RUNLOCK(obj);
continue;
}
/*
* Skip contiguous regions that point to the same
* vnode, so we don't emit redundant MAP-IN
* directives.
*/
if (entry->start == last_end && lobj->handle == last_vp) {
last_end = entry->end;
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
VM_OBJECT_RUNLOCK(obj);
continue;
}
/*
* We don't want to keep the proc's vm_map or this
* vm_object locked while we walk the pathname, since
* vn_fullpath() can sleep. However, if we drop the
* lock, it's possible for concurrent activity to
* modify the vm_map list. To protect against this,
* we save the vm_map timestamp before we release the
* lock, and check it after we reacquire the lock
* below.
*/
start_addr = entry->start;
last_end = entry->end;
last_timestamp = map->timestamp;
vm_map_unlock_read(map);
vp = lobj->handle;
vref(vp);
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
VM_OBJECT_RUNLOCK(obj);
freepath = NULL;
pmc_getfilename(vp, &fullpath, &freepath);
last_vp = vp;
vrele(vp);
vp = NULL;
pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
if (freepath)
free(freepath, M_TEMP);
vm_map_lock_read(map);
/*
* If our saved timestamp doesn't match, this means
* that the vm_map was modified out from under us and
* we can't trust our current "entry" pointer. Do a
* new lookup for this entry. If there is no entry
* for this address range, vm_map_lookup_entry() will
* return the previous one, so we always want to go to
* entry->next on the next loop iteration.
*
* There is an edge condition here that can occur if
* there is no entry at or before this address. In
* this situation, vm_map_lookup_entry returns
* &map->header, which would cause our loop to abort
* without processing the rest of the map. However,
* in practice this will never happen for process
* vm_map. This is because the executable's text
* segment is the first mapping in the proc's address
* space, and this mapping is never removed until the
* process exits, so there will always be a non-header
* entry at or before the requested address for
* vm_map_lookup_entry to return.
*/
if (map->timestamp != last_timestamp)
vm_map_lookup_entry(map, last_end - 1, &entry);
}
vm_map_unlock_read(map);
vmspace_free(vm);
return;
}
/*
* Log mappings for all processes in the system.
*/
static void
pmc_log_all_process_mappings(struct pmc_owner *po)
{
struct proc *p, *top;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((p = pfind(1)) == NULL)
panic("[pmc,%d] Cannot find init", __LINE__);
PROC_UNLOCK(p);
sx_slock(&proctree_lock);
top = p;
for (;;) {
pmc_log_process_mappings(po, p);
if (!LIST_EMPTY(&p->p_children))
p = LIST_FIRST(&p->p_children);
else for (;;) {
if (p == top)
goto done;
if (LIST_NEXT(p, p_sibling)) {
p = LIST_NEXT(p, p_sibling);
break;
}
p = p->p_pptr;
}
}
done:
sx_sunlock(&proctree_lock);
}
/*
* The 'hook' invoked from the kernel proper
*/
#ifdef HWPMC_DEBUG
const char *pmc_hooknames[] = {
/* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
"",
"EXEC",
"CSW-IN",
"CSW-OUT",
"SAMPLE",
"UNUSED1",
"UNUSED2",
"MMAP",
"MUNMAP",
"CALLCHAIN-NMI",
"CALLCHAIN-SOFT",
"SOFTSAMPLING"
};
#endif
static int
pmc_hook_handler(struct thread *td, int function, void *arg)
{
- PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
+ PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
pmc_hooknames[function], arg);
switch (function)
{
/*
* Process exec()
*/
case PMC_FN_PROCESS_EXEC:
{
char *fullpath, *freepath;
unsigned int ri;
int is_using_hwpmcs;
struct pmc *pm;
struct proc *p;
struct pmc_owner *po;
struct pmc_process *pp;
struct pmckern_procexec *pk;
sx_assert(&pmc_sx, SX_XLOCKED);
p = td->td_proc;
pmc_getfilename(p->p_textvp, &fullpath, &freepath);
pk = (struct pmckern_procexec *) arg;
/* Inform owners of SS mode PMCs of the exec event. */
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_procexec(po, PMC_ID_INVALID,
p->p_pid, pk->pm_entryaddr, fullpath);
PROC_LOCK(p);
is_using_hwpmcs = p->p_flag & P_HWPMC;
PROC_UNLOCK(p);
if (!is_using_hwpmcs) {
if (freepath)
free(freepath, M_TEMP);
break;
}
/*
* PMCs are not inherited across an exec(): remove any
* PMCs that this process is the owner of.
*/
if ((po = pmc_find_owner_descriptor(p)) != NULL) {
pmc_remove_owner(po);
pmc_destroy_owner_descriptor(po);
}
/*
* If the process being exec'ed is not the target of any
* PMC, we are done.
*/
if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
if (freepath)
free(freepath, M_TEMP);
break;
}
/*
* Log the exec event to all monitoring owners. Skip
* owners who have already recieved the event because
* they had system sampling PMCs active.
*/
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
po = pm->pm_owner;
if (po->po_sscount == 0 &&
po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_procexec(po, pm->pm_id,
p->p_pid, pk->pm_entryaddr,
fullpath);
}
if (freepath)
free(freepath, M_TEMP);
- PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
+ PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
if (pk->pm_credentialschanged == 0) /* no change */
break;
/*
* If the newly exec()'ed process has a different credential
* than before, allow it to be the target of a PMC only if
* the PMC's owner has sufficient priviledge.
*/
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
if (pmc_can_attach(pm, td->td_proc) != 0)
pmc_detach_one_process(td->td_proc,
pm, PMC_FLAG_NONE);
KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
pp->pp_refcnt, pp));
/*
* If this process is no longer the target of any
* PMCs, we can remove the process entry and free
* up space.
*/
if (pp->pp_refcnt == 0) {
pmc_remove_process_descriptor(pp);
free(pp, M_PMC);
break;
}
}
break;
case PMC_FN_CSW_IN:
pmc_process_csw_in(td);
break;
case PMC_FN_CSW_OUT:
pmc_process_csw_out(td);
break;
/*
* Process accumulated PC samples.
*
* This function is expected to be called by hardclock() for
* each CPU that has accumulated PC samples.
*
* This function is to be executed on the CPU whose samples
* are being processed.
*/
case PMC_FN_DO_SAMPLES:
/*
* Clear the cpu specific bit in the CPU mask before
* do the rest of the processing. If the NMI handler
* gets invoked after the "atomic_clear_int()" call
* below but before "pmc_process_samples()" gets
* around to processing the interrupt, then we will
* come back here at the next hardclock() tick (and
* may find nothing to do if "pmc_process_samples()"
* had already processed the interrupt). We don't
* lose the interrupt sample.
*/
CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask);
pmc_process_samples(PCPU_GET(cpuid), PMC_HR);
pmc_process_samples(PCPU_GET(cpuid), PMC_SR);
break;
case PMC_FN_MMAP:
sx_assert(&pmc_sx, SX_LOCKED);
pmc_process_mmap(td, (struct pmckern_map_in *) arg);
break;
case PMC_FN_MUNMAP:
sx_assert(&pmc_sx, SX_LOCKED);
pmc_process_munmap(td, (struct pmckern_map_out *) arg);
break;
case PMC_FN_USER_CALLCHAIN:
/*
* Record a call chain.
*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));
pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
(struct trapframe *) arg);
td->td_pflags &= ~TDP_CALLCHAIN;
break;
case PMC_FN_USER_CALLCHAIN_SOFT:
/*
* Record a call chain.
*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));
pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR,
(struct trapframe *) arg);
td->td_pflags &= ~TDP_CALLCHAIN;
break;
case PMC_FN_SOFT_SAMPLING:
/*
* Call soft PMC sampling intr.
*/
pmc_soft_intr((struct pmckern_soft *) arg);
break;
default:
#ifdef HWPMC_DEBUG
KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
#endif
break;
}
return 0;
}
/*
* allocate a 'struct pmc_owner' descriptor in the owner hash table.
*/
static struct pmc_owner *
pmc_allocate_owner_descriptor(struct proc *p)
{
uint32_t hindex;
struct pmc_owner *po;
struct pmc_ownerhash *poh;
hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
poh = &pmc_ownerhash[hindex];
/* allocate space for N pointers and one descriptor struct */
po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO);
po->po_owner = p;
LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
TAILQ_INIT(&po->po_logbuffers);
mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);
- PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
+ PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
p, p->p_pid, p->p_comm, po);
return po;
}
static void
pmc_destroy_owner_descriptor(struct pmc_owner *po)
{
- PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
+ PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
mtx_destroy(&po->po_mtx);
free(po, M_PMC);
}
/*
* find the descriptor corresponding to process 'p', adding or removing it
* as specified by 'mode'.
*/
static struct pmc_process *
pmc_find_process_descriptor(struct proc *p, uint32_t mode)
{
uint32_t hindex;
struct pmc_process *pp, *ppnew;
struct pmc_processhash *pph;
hindex = PMC_HASH_PTR(p, pmc_processhashmask);
pph = &pmc_processhash[hindex];
ppnew = NULL;
/*
* Pre-allocate memory in the FIND_ALLOCATE case since we
* cannot call malloc(9) once we hold a spin lock.
*/
if (mode & PMC_FLAG_ALLOCATE)
ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO);
mtx_lock_spin(&pmc_processhash_mtx);
LIST_FOREACH(pp, pph, pp_next)
if (pp->pp_proc == p)
break;
if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
LIST_REMOVE(pp, pp_next);
if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
ppnew != NULL) {
ppnew->pp_proc = p;
LIST_INSERT_HEAD(pph, ppnew, pp_next);
pp = ppnew;
ppnew = NULL;
}
mtx_unlock_spin(&pmc_processhash_mtx);
if (pp != NULL && ppnew != NULL)
free(ppnew, M_PMC);
return pp;
}
/*
* remove a process descriptor from the process hash table.
*/
static void
pmc_remove_process_descriptor(struct pmc_process *pp)
{
KASSERT(pp->pp_refcnt == 0,
("[pmc,%d] Removing process descriptor %p with count %d",
__LINE__, pp, pp->pp_refcnt));
mtx_lock_spin(&pmc_processhash_mtx);
LIST_REMOVE(pp, pp_next);
mtx_unlock_spin(&pmc_processhash_mtx);
}
/*
* find an owner descriptor corresponding to proc 'p'
*/
static struct pmc_owner *
pmc_find_owner_descriptor(struct proc *p)
{
uint32_t hindex;
struct pmc_owner *po;
struct pmc_ownerhash *poh;
hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
poh = &pmc_ownerhash[hindex];
po = NULL;
LIST_FOREACH(po, poh, po_next)
if (po->po_owner == p)
break;
- PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
+ PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
"pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
return po;
}
/*
* pmc_allocate_pmc_descriptor
*
* Allocate a pmc descriptor and initialize its
* fields.
*/
static struct pmc *
pmc_allocate_pmc_descriptor(void)
{
struct pmc *pmc;
pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO);
- PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
+ PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
return pmc;
}
/*
* Destroy a pmc descriptor.
*/
static void
pmc_destroy_pmc_descriptor(struct pmc *pm)
{
KASSERT(pm->pm_state == PMC_STATE_DELETED ||
pm->pm_state == PMC_STATE_FREE,
("[pmc,%d] destroying non-deleted PMC", __LINE__));
KASSERT(LIST_EMPTY(&pm->pm_targets),
("[pmc,%d] destroying pmc with targets", __LINE__));
KASSERT(pm->pm_owner == NULL,
("[pmc,%d] destroying pmc attached to an owner", __LINE__));
KASSERT(pm->pm_runcount == 0,
("[pmc,%d] pmc has non-zero run count %d", __LINE__,
pm->pm_runcount));
free(pm, M_PMC);
}
static void
pmc_wait_for_pmc_idle(struct pmc *pm)
{
#ifdef HWPMC_DEBUG
volatile int maxloop;
maxloop = 100 * pmc_cpu_max();
#endif
/*
* Loop (with a forced context switch) till the PMC's runcount
* comes down to zero.
*/
while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
#ifdef HWPMC_DEBUG
maxloop--;
KASSERT(maxloop > 0,
("[pmc,%d] (ri%d, rc%d) waiting too long for "
"pmc to be free", __LINE__,
PMC_TO_ROWINDEX(pm), pm->pm_runcount));
#endif
pmc_force_context_switch();
}
}
/*
* This function does the following things:
*
* - detaches the PMC from hardware
* - unlinks all target threads that were attached to it
* - removes the PMC from its owner's list
* - destroys the PMC private mutex
*
* Once this function completes, the given pmc pointer can be freed by
* calling pmc_destroy_pmc_descriptor().
*/
static void
pmc_release_pmc_descriptor(struct pmc *pm)
{
enum pmc_mode mode;
struct pmc_hw *phw;
u_int adjri, ri, cpu;
struct pmc_owner *po;
struct pmc_binding pb;
struct pmc_process *pp;
struct pmc_classdep *pcd;
struct pmc_target *ptgt, *tmp;
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
mode = PMC_TO_MODE(pm);
- PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
+ PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
mode);
/*
* First, we take the PMC off hardware.
*/
cpu = 0;
if (PMC_IS_SYSTEM_MODE(mode)) {
/*
* A system mode PMC runs on a specific CPU. Switch
* to this CPU and turn hardware off.
*/
pmc_save_cpu_binding(&pb);
cpu = PMC_TO_CPU(pm);
pmc_select_cpu(cpu);
/* switch off non-stalled CPUs */
if (pm->pm_state == PMC_STATE_RUNNING &&
pm->pm_stalled == 0) {
phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
KASSERT(phw->phw_pmc == pm,
("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
__LINE__, ri, phw->phw_pmc, pm));
- PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
critical_enter();
pcd->pcd_stop_pmc(cpu, adjri);
critical_exit();
}
- PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
critical_enter();
pcd->pcd_config_pmc(cpu, adjri, NULL);
critical_exit();
/* adjust the global and process count of SS mode PMCs */
if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
po = pm->pm_owner;
po->po_sscount--;
if (po->po_sscount == 0) {
atomic_subtract_rel_int(&pmc_ss_count, 1);
LIST_REMOVE(po, po_ssnext);
}
}
pm->pm_state = PMC_STATE_DELETED;
pmc_restore_cpu_binding(&pb);
/*
* We could have references to this PMC structure in
* the per-cpu sample queues. Wait for the queue to
* drain.
*/
pmc_wait_for_pmc_idle(pm);
} else if (PMC_IS_VIRTUAL_MODE(mode)) {
/*
* A virtual PMC could be running on multiple CPUs at
* a given instant.
*
* By marking its state as DELETED, we ensure that
* this PMC is never further scheduled on hardware.
*
* Then we wait till all CPUs are done with this PMC.
*/
pm->pm_state = PMC_STATE_DELETED;
/* Wait for the PMCs runcount to come to zero. */
pmc_wait_for_pmc_idle(pm);
/*
* At this point the PMC is off all CPUs and cannot be
* freshly scheduled onto a CPU. It is now safe to
* unlink all targets from this PMC. If a
* process-record's refcount falls to zero, we remove
* it from the hash table. The module-wide SX lock
* protects us from races.
*/
LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
pp = ptgt->pt_process;
pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
- PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
+ PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
/*
* If the target process record shows that no
* PMCs are attached to it, reclaim its space.
*/
if (pp->pp_refcnt == 0) {
pmc_remove_process_descriptor(pp);
free(pp, M_PMC);
}
}
cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
}
/*
* Release any MD resources
*/
(void) pcd->pcd_release_pmc(cpu, adjri, pm);
/*
* Update row disposition
*/
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
PMC_UNMARK_ROW_STANDALONE(ri);
else
PMC_UNMARK_ROW_THREAD(ri);
/* unlink from the owner's list */
if (pm->pm_owner) {
LIST_REMOVE(pm, pm_next);
pm->pm_owner = NULL;
}
}
/*
* Register an owner and a pmc.
*/
static int
pmc_register_owner(struct proc *p, struct pmc *pmc)
{
struct pmc_owner *po;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((po = pmc_find_owner_descriptor(p)) == NULL)
if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
return ENOMEM;
KASSERT(pmc->pm_owner == NULL,
("[pmc,%d] attempting to own an initialized PMC", __LINE__));
pmc->pm_owner = po;
LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
PROC_LOCK(p);
p->p_flag |= P_HWPMC;
PROC_UNLOCK(p);
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_pmcallocate(pmc);
- PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
+ PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
po, pmc);
return 0;
}
/*
* Return the current row disposition:
* == 0 => FREE
* > 0 => PROCESS MODE
* < 0 => SYSTEM MODE
*/
int
pmc_getrowdisp(int ri)
{
return pmc_pmcdisp[ri];
}
/*
* Check if a PMC at row index 'ri' can be allocated to the current
* process.
*
* Allocation can fail if:
* - the current process is already being profiled by a PMC at index 'ri',
* attached to it via OP_PMCATTACH.
* - the current process has already allocated a PMC at index 'ri'
* via OP_ALLOCATE.
*/
static int
pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
{
enum pmc_mode mode;
struct pmc *pm;
struct pmc_owner *po;
struct pmc_process *pp;
- PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
+ PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
"cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
/*
* We shouldn't have already allocated a process-mode PMC at
* row index 'ri'.
*
* We shouldn't have allocated a system-wide PMC on the same
* CPU and same RI.
*/
if ((po = pmc_find_owner_descriptor(p)) != NULL)
LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
if (PMC_TO_ROWINDEX(pm) == ri) {
mode = PMC_TO_MODE(pm);
if (PMC_IS_VIRTUAL_MODE(mode))
return EEXIST;
if (PMC_IS_SYSTEM_MODE(mode) &&
(int) PMC_TO_CPU(pm) == cpu)
return EEXIST;
}
}
/*
* We also shouldn't be the target of any PMC at this index
* since otherwise a PMC_ATTACH to ourselves will fail.
*/
if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
if (pp->pp_pmcs[ri].pp_pmc)
return EEXIST;
- PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
+ PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
p, p->p_pid, p->p_comm, ri);
return 0;
}
/*
* Check if a given PMC at row index 'ri' can be currently used in
* mode 'mode'.
*/
static int
pmc_can_allocate_row(int ri, enum pmc_mode mode)
{
enum pmc_disp disp;
sx_assert(&pmc_sx, SX_XLOCKED);
- PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
+ PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
if (PMC_IS_SYSTEM_MODE(mode))
disp = PMC_DISP_STANDALONE;
else
disp = PMC_DISP_THREAD;
/*
* check disposition for PMC row 'ri':
*
* Expected disposition Row-disposition Result
*
* STANDALONE STANDALONE or FREE proceed
* STANDALONE THREAD fail
* THREAD THREAD or FREE proceed
* THREAD STANDALONE fail
*/
if (!PMC_ROW_DISP_IS_FREE(ri) &&
!(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
!(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
return EBUSY;
/*
* All OK
*/
- PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
+ PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
return 0;
}
/*
* Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
*/
static struct pmc *
pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
{
struct pmc *pm;
KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
LIST_FOREACH(pm, &po->po_pmcs, pm_next)
if (pm->pm_id == pmcid)
return pm;
return NULL;
}
static int
pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
{
struct pmc *pm;
struct pmc_owner *po;
- PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid);
+ PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL)
return ESRCH;
if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
return EINVAL;
- PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
+ PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
*pmc = pm;
return 0;
}
/*
* Start a PMC.
*/
static int
pmc_start(struct pmc *pm)
{
enum pmc_mode mode;
struct pmc_owner *po;
struct pmc_binding pb;
struct pmc_classdep *pcd;
int adjri, error, cpu, ri;
KASSERT(pm != NULL,
("[pmc,%d] null pm", __LINE__));
mode = PMC_TO_MODE(pm);
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
error = 0;
- PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
+ PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
po = pm->pm_owner;
/*
* Disallow PMCSTART if a logfile is required but has not been
* configured yet.
*/
if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
(po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
return (EDOOFUS); /* programming error */
/*
* If this is a sampling mode PMC, log mapping information for
* the kernel modules that are currently loaded.
*/
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pmc_log_kernel_mappings(pm);
if (PMC_IS_VIRTUAL_MODE(mode)) {
/*
* If a PMCATTACH has never been done on this PMC,
* attach it to its owner process.
*/
if (LIST_EMPTY(&pm->pm_targets))
error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
pmc_attach_process(po->po_owner, pm);
/*
* If the PMC is attached to its owner, then force a context
* switch to ensure that the MD state gets set correctly.
*/
if (error == 0) {
pm->pm_state = PMC_STATE_RUNNING;
if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
pmc_force_context_switch();
}
return (error);
}
/*
* A system-wide PMC.
*
* Add the owner to the global list if this is a system-wide
* sampling PMC.
*/
if (mode == PMC_MODE_SS) {
if (po->po_sscount == 0) {
LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
atomic_add_rel_int(&pmc_ss_count, 1);
- PMCDBG(PMC,OPS,1, "po=%p in global list", po);
+ PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
}
po->po_sscount++;
/*
* Log mapping information for all existing processes in the
* system. Subsequent mappings are logged as they happen;
* see pmc_process_mmap().
*/
if (po->po_logprocmaps == 0) {
pmc_log_all_process_mappings(po);
po->po_logprocmaps = 1;
}
}
/*
* Move to the CPU associated with this
* PMC, and start the hardware.
*/
pmc_save_cpu_binding(&pb);
cpu = PMC_TO_CPU(pm);
if (!pmc_cpu_is_active(cpu))
return (ENXIO);
pmc_select_cpu(cpu);
/*
* global PMCs are configured at allocation time
* so write out the initial value and start the PMC.
*/
pm->pm_state = PMC_STATE_RUNNING;
critical_enter();
if ((error = pcd->pcd_write_pmc(cpu, adjri,
PMC_IS_SAMPLING_MODE(mode) ?
pm->pm_sc.pm_reloadcount :
pm->pm_sc.pm_initial)) == 0)
error = pcd->pcd_start_pmc(cpu, adjri);
critical_exit();
pmc_restore_cpu_binding(&pb);
return (error);
}
/*
* Stop a PMC.
*/
static int
pmc_stop(struct pmc *pm)
{
struct pmc_owner *po;
struct pmc_binding pb;
struct pmc_classdep *pcd;
int adjri, cpu, error, ri;
KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
- PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
+ PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
pm->pm_state = PMC_STATE_STOPPED;
/*
* If the PMC is a virtual mode one, changing the state to
* non-RUNNING is enough to ensure that the PMC never gets
* scheduled.
*
* If this PMC is current running on a CPU, then it will
* handled correctly at the time its target process is context
* switched out.
*/
if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
return 0;
/*
* A system-mode PMC. Move to the CPU associated with
* this PMC, and stop the hardware. We update the
* 'initial count' so that a subsequent PMCSTART will
* resume counting from the current hardware count.
*/
pmc_save_cpu_binding(&pb);
cpu = PMC_TO_CPU(pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
if (!pmc_cpu_is_active(cpu))
return ENXIO;
pmc_select_cpu(cpu);
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
critical_enter();
if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
critical_exit();
pmc_restore_cpu_binding(&pb);
po = pm->pm_owner;
/* remove this owner from the global list of SS PMC owners */
if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
po->po_sscount--;
if (po->po_sscount == 0) {
atomic_subtract_rel_int(&pmc_ss_count, 1);
LIST_REMOVE(po, po_ssnext);
- PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
+ PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
}
}
return (error);
}
#ifdef HWPMC_DEBUG
static const char *pmc_op_to_name[] = {
#undef __PMC_OP
#define __PMC_OP(N, D) #N ,
__PMC_OPS()
NULL
};
#endif
/*
* The syscall interface
*/
#define PMC_GET_SX_XLOCK(...) do { \
sx_xlock(&pmc_sx); \
if (pmc_hook == NULL) { \
sx_xunlock(&pmc_sx); \
return __VA_ARGS__; \
} \
} while (0)
#define PMC_DOWNGRADE_SX() do { \
sx_downgrade(&pmc_sx); \
is_sx_downgraded = 1; \
} while (0)
static int
pmc_syscall_handler(struct thread *td, void *syscall_args)
{
int error, is_sx_downgraded, is_sx_locked, op;
struct pmc_syscall_args *c;
void *arg;
PMC_GET_SX_XLOCK(ENOSYS);
DROP_GIANT();
is_sx_downgraded = 0;
is_sx_locked = 1;
c = (struct pmc_syscall_args *) syscall_args;
op = c->pmop_code;
arg = c->pmop_data;
- PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
+ PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
pmc_op_to_name[op], arg);
error = 0;
atomic_add_int(&pmc_stats.pm_syscalls, 1);
switch(op)
{
/*
* Configure a log file.
*
* XXX This OP will be reworked.
*/
case PMC_OP_CONFIGURELOG:
{
struct proc *p;
struct pmc *pm;
struct pmc_owner *po;
struct pmc_op_configurelog cl;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
break;
/* mark this process as owning a log file */
p = td->td_proc;
if ((po = pmc_find_owner_descriptor(p)) == NULL)
if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
error = ENOMEM;
break;
}
/*
* If a valid fd was passed in, try to configure that,
* otherwise if 'fd' was less than zero and there was
* a log file configured, flush its buffers and
* de-configure it.
*/
if (cl.pm_logfd >= 0) {
sx_xunlock(&pmc_sx);
is_sx_locked = 0;
error = pmclog_configure_log(md, po, cl.pm_logfd);
} else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
pmclog_process_closelog(po);
error = pmclog_close(po);
if (error == 0) {
LIST_FOREACH(pm, &po->po_pmcs, pm_next)
if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
pm->pm_state == PMC_STATE_RUNNING)
pmc_stop(pm);
error = pmclog_deconfigure_log(po);
}
} else
error = EINVAL;
if (error)
break;
}
break;
/*
* Flush a log file.
*/
case PMC_OP_FLUSHLOG:
{
struct pmc_owner *po;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
error = EINVAL;
break;
}
error = pmclog_flush(po);
}
break;
/*
* Close a log file.
*/
case PMC_OP_CLOSELOG:
{
struct pmc_owner *po;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
error = EINVAL;
break;
}
error = pmclog_close(po);
}
break;
/*
* Retrieve hardware configuration.
*/
case PMC_OP_GETCPUINFO: /* CPU information */
{
struct pmc_op_getcpuinfo gci;
struct pmc_classinfo *pci;
struct pmc_classdep *pcd;
int cl;
gci.pm_cputype = md->pmd_cputype;
gci.pm_ncpu = pmc_cpu_max();
gci.pm_npmc = md->pmd_npmc;
gci.pm_nclass = md->pmd_nclass;
pci = gci.pm_classes;
pcd = md->pmd_classdep;
for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
pci->pm_caps = pcd->pcd_caps;
pci->pm_class = pcd->pcd_class;
pci->pm_width = pcd->pcd_width;
pci->pm_num = pcd->pcd_num;
}
error = copyout(&gci, arg, sizeof(gci));
}
break;
/*
* Retrieve soft events list.
*/
case PMC_OP_GETDYNEVENTINFO:
{
enum pmc_class cl;
enum pmc_event ev;
struct pmc_op_getdyneventinfo *gei;
struct pmc_dyn_event_descr dev;
struct pmc_soft *ps;
uint32_t nevent;
sx_assert(&pmc_sx, SX_LOCKED);
gei = (struct pmc_op_getdyneventinfo *) arg;
if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
break;
/* Only SOFT class is dynamic. */
if (cl != PMC_CLASS_SOFT) {
error = EINVAL;
break;
}
nevent = 0;
for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
ps = pmc_soft_ev_acquire(ev);
if (ps == NULL)
continue;
bcopy(&ps->ps_ev, &dev, sizeof(dev));
pmc_soft_ev_release(ps);
error = copyout(&dev,
&gei->pm_events[nevent],
sizeof(struct pmc_dyn_event_descr));
if (error != 0)
break;
nevent++;
}
if (error != 0)
break;
error = copyout(&nevent, &gei->pm_nevent,
sizeof(nevent));
}
break;
/*
* Get module statistics
*/
case PMC_OP_GETDRIVERSTATS:
{
struct pmc_op_getdriverstats gms;
bcopy(&pmc_stats, &gms, sizeof(gms));
error = copyout(&gms, arg, sizeof(gms));
}
break;
/*
* Retrieve module version number
*/
case PMC_OP_GETMODULEVERSION:
{
uint32_t cv, modv;
/* retrieve the client's idea of the ABI version */
if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
break;
/* don't service clients newer than our driver */
modv = PMC_VERSION;
if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
error = EPROGMISMATCH;
break;
}
error = copyout(&modv, arg, sizeof(int));
}
break;
/*
* Retrieve the state of all the PMCs on a given
* CPU.
*/
case PMC_OP_GETPMCINFO:
{
int ari;
struct pmc *pm;
size_t pmcinfo_size;
uint32_t cpu, n, npmc;
struct pmc_owner *po;
struct pmc_binding pb;
struct pmc_classdep *pcd;
struct pmc_info *p, *pmcinfo;
struct pmc_op_getpmcinfo *gpi;
PMC_DOWNGRADE_SX();
gpi = (struct pmc_op_getpmcinfo *) arg;
if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
break;
if (cpu >= pmc_cpu_max()) {
error = EINVAL;
break;
}
if (!pmc_cpu_is_active(cpu)) {
error = ENXIO;
break;
}
/* switch to CPU 'cpu' */
pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu);
npmc = md->pmd_npmc;
pmcinfo_size = npmc * sizeof(struct pmc_info);
pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK);
p = pmcinfo;
for (n = 0; n < md->pmd_npmc; n++, p++) {
pcd = pmc_ri_to_classdep(md, n, &ari);
KASSERT(pcd != NULL,
("[pmc,%d] null pcd ri=%d", __LINE__, n));
if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
break;
if (PMC_ROW_DISP_IS_STANDALONE(n))
p->pm_rowdisp = PMC_DISP_STANDALONE;
else if (PMC_ROW_DISP_IS_THREAD(n))
p->pm_rowdisp = PMC_DISP_THREAD;
else
p->pm_rowdisp = PMC_DISP_FREE;
p->pm_ownerpid = -1;
if (pm == NULL) /* no PMC associated */
continue;
po = pm->pm_owner;
KASSERT(po->po_owner != NULL,
("[pmc,%d] pmc_owner had a null proc pointer",
__LINE__));
p->pm_ownerpid = po->po_owner->p_pid;
p->pm_mode = PMC_TO_MODE(pm);
p->pm_event = pm->pm_event;
p->pm_flags = pm->pm_flags;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
p->pm_reloadcount =
pm->pm_sc.pm_reloadcount;
}
pmc_restore_cpu_binding(&pb);
/* now copy out the PMC info collected */
if (error == 0)
error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
free(pmcinfo, M_PMC);
}
break;
/*
* Set the administrative state of a PMC. I.e. whether
* the PMC is to be used or not.
*/
case PMC_OP_PMCADMIN:
{
int cpu, ri;
enum pmc_state request;
struct pmc_cpu *pc;
struct pmc_hw *phw;
struct pmc_op_pmcadmin pma;
struct pmc_binding pb;
sx_assert(&pmc_sx, SX_XLOCKED);
KASSERT(td == curthread,
("[pmc,%d] td != curthread", __LINE__));
error = priv_check(td, PRIV_PMC_MANAGE);
if (error)
break;
if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
break;
cpu = pma.pm_cpu;
if (cpu < 0 || cpu >= (int) pmc_cpu_max()) {
error = EINVAL;
break;
}
if (!pmc_cpu_is_active(cpu)) {
error = ENXIO;
break;
}
request = pma.pm_state;
if (request != PMC_STATE_DISABLED &&
request != PMC_STATE_FREE) {
error = EINVAL;
break;
}
ri = pma.pm_pmc; /* pmc id == row index */
if (ri < 0 || ri >= (int) md->pmd_npmc) {
error = EINVAL;
break;
}
/*
* We can't disable a PMC with a row-index allocated
* for process virtual PMCs.
*/
if (PMC_ROW_DISP_IS_THREAD(ri) &&
request == PMC_STATE_DISABLED) {
error = EBUSY;
break;
}
/*
* otherwise, this PMC on this CPU is either free or
* in system-wide mode.
*/
pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu);
pc = pmc_pcpu[cpu];
phw = pc->pc_hwpmcs[ri];
/*
* XXX do we need some kind of 'forced' disable?
*/
if (phw->phw_pmc == NULL) {
if (request == PMC_STATE_DISABLED &&
(phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
PMC_MARK_ROW_STANDALONE(ri);
} else if (request == PMC_STATE_FREE &&
(phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED;
PMC_UNMARK_ROW_STANDALONE(ri);
}
/* other cases are a no-op */
} else
error = EBUSY;
pmc_restore_cpu_binding(&pb);
}
break;
/*
* Allocate a PMC.
*/
case PMC_OP_PMCALLOCATE:
{
int adjri, n;
u_int cpu;
uint32_t caps;
struct pmc *pmc;
enum pmc_mode mode;
struct pmc_hw *phw;
struct pmc_binding pb;
struct pmc_classdep *pcd;
struct pmc_op_pmcallocate pa;
if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
break;
caps = pa.pm_caps;
mode = pa.pm_mode;
cpu = pa.pm_cpu;
if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC &&
mode != PMC_MODE_TS && mode != PMC_MODE_TC) ||
(cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
error = EINVAL;
break;
}
/*
* Virtual PMCs should only ask for a default CPU.
* System mode PMCs need to specify a non-default CPU.
*/
if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
(PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
error = EINVAL;
break;
}
/*
* Check that an inactive CPU is not being asked for.
*/
if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
error = ENXIO;
break;
}
/*
* Refuse an allocation for a system-wide PMC if this
* process has been jailed, or if this process lacks
* super-user credentials and the sysctl tunable
* 'security.bsd.unprivileged_syspmcs' is zero.
*/
if (PMC_IS_SYSTEM_MODE(mode)) {
if (jailed(curthread->td_ucred)) {
error = EPERM;
break;
}
if (!pmc_unprivileged_syspmcs) {
error = priv_check(curthread,
PRIV_PMC_SYSTEM);
if (error)
break;
}
}
/*
* Look for valid values for 'pm_flags'
*/
if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) {
error = EINVAL;
break;
}
/* process logging options are not allowed for system PMCs */
if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
(PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
error = EINVAL;
break;
}
/*
* All sampling mode PMCs need to be able to interrupt the
* CPU.
*/
if (PMC_IS_SAMPLING_MODE(mode))
caps |= PMC_CAP_INTERRUPT;
/* A valid class specifier should have been passed in. */
for (n = 0; n < md->pmd_nclass; n++)
if (md->pmd_classdep[n].pcd_class == pa.pm_class)
break;
if (n == md->pmd_nclass) {
error = EINVAL;
break;
}
/* The requested PMC capabilities should be feasible. */
if ((md->pmd_classdep[n].pcd_caps & caps) != caps) {
error = EOPNOTSUPP;
break;
}
- PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
+ PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
pa.pm_ev, caps, mode, cpu);
pmc = pmc_allocate_pmc_descriptor();
pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
PMC_ID_INVALID);
pmc->pm_event = pa.pm_ev;
pmc->pm_state = PMC_STATE_FREE;
pmc->pm_caps = caps;
pmc->pm_flags = pa.pm_flags;
/* switch thread to CPU 'cpu' */
pmc_save_cpu_binding(&pb);
#define PMC_IS_SHAREABLE_PMC(cpu, n) \
(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
PMC_PHW_FLAG_IS_SHAREABLE)
#define PMC_IS_UNALLOCATED(cpu, n) \
(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
if (PMC_IS_SYSTEM_MODE(mode)) {
pmc_select_cpu(cpu);
for (n = 0; n < (int) md->pmd_npmc; n++) {
pcd = pmc_ri_to_classdep(md, n, &adjri);
if (pmc_can_allocate_row(n, mode) == 0 &&
pmc_can_allocate_rowindex(
curthread->td_proc, n, cpu) == 0 &&
(PMC_IS_UNALLOCATED(cpu, n) ||
PMC_IS_SHAREABLE_PMC(cpu, n)) &&
pcd->pcd_allocate_pmc(cpu, adjri, pmc,
&pa) == 0)
break;
}
} else {
/* Process virtual mode */
for (n = 0; n < (int) md->pmd_npmc; n++) {
pcd = pmc_ri_to_classdep(md, n, &adjri);
if (pmc_can_allocate_row(n, mode) == 0 &&
pmc_can_allocate_rowindex(
curthread->td_proc, n,
PMC_CPU_ANY) == 0 &&
pcd->pcd_allocate_pmc(curthread->td_oncpu,
adjri, pmc, &pa) == 0)
break;
}
}
#undef PMC_IS_UNALLOCATED
#undef PMC_IS_SHAREABLE_PMC
pmc_restore_cpu_binding(&pb);
if (n == (int) md->pmd_npmc) {
pmc_destroy_pmc_descriptor(pmc);
pmc = NULL;
error = EINVAL;
break;
}
/* Fill in the correct value in the ID field */
pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
- PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
+ PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
/* Process mode PMCs with logging enabled need log files */
if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
/* All system mode sampling PMCs require a log file */
if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
/*
* Configure global pmc's immediately
*/
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu);
phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
pcd = pmc_ri_to_classdep(md, n, &adjri);
if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
(error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
(void) pcd->pcd_release_pmc(cpu, adjri, pmc);
pmc_destroy_pmc_descriptor(pmc);
pmc = NULL;
pmc_restore_cpu_binding(&pb);
error = EPERM;
break;
}
pmc_restore_cpu_binding(&pb);
}
pmc->pm_state = PMC_STATE_ALLOCATED;
/*
* mark row disposition
*/
if (PMC_IS_SYSTEM_MODE(mode))
PMC_MARK_ROW_STANDALONE(n);
else
PMC_MARK_ROW_THREAD(n);
/*
* Register this PMC with the current thread as its owner.
*/
if ((error =
pmc_register_owner(curthread->td_proc, pmc)) != 0) {
pmc_release_pmc_descriptor(pmc);
pmc_destroy_pmc_descriptor(pmc);
pmc = NULL;
break;
}
/*
* Return the allocated index.
*/
pa.pm_pmcid = pmc->pm_id;
error = copyout(&pa, arg, sizeof(pa));
}
break;
/*
* Attach a PMC to a process.
*/
case PMC_OP_PMCATTACH:
{
struct pmc *pm;
struct proc *p;
struct pmc_op_pmcattach a;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((error = copyin(arg, &a, sizeof(a))) != 0)
break;
if (a.pm_pid < 0) {
error = EINVAL;
break;
} else if (a.pm_pid == 0)
a.pm_pid = td->td_proc->p_pid;
if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
break;
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
error = EINVAL;
break;
}
/* PMCs may be (re)attached only when allocated or stopped */
if (pm->pm_state == PMC_STATE_RUNNING) {
error = EBUSY;
break;
} else if (pm->pm_state != PMC_STATE_ALLOCATED &&
pm->pm_state != PMC_STATE_STOPPED) {
error = EINVAL;
break;
}
/* lookup pid */
if ((p = pfind(a.pm_pid)) == NULL) {
error = ESRCH;
break;
}
/*
* Ignore processes that are working on exiting.
*/
if (p->p_flag & P_WEXIT) {
error = ESRCH;
PROC_UNLOCK(p); /* pfind() returns a locked process */
break;
}
/*
* we are allowed to attach a PMC to a process if
* we can debug it.
*/
error = p_candebug(curthread, p);
PROC_UNLOCK(p);
if (error == 0)
error = pmc_attach_process(p, pm);
}
break;
/*
* Detach an attached PMC from a process.
*/
case PMC_OP_PMCDETACH:
{
struct pmc *pm;
struct proc *p;
struct pmc_op_pmcattach a;
if ((error = copyin(arg, &a, sizeof(a))) != 0)
break;
if (a.pm_pid < 0) {
error = EINVAL;
break;
} else if (a.pm_pid == 0)
a.pm_pid = td->td_proc->p_pid;
if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
break;
if ((p = pfind(a.pm_pid)) == NULL) {
error = ESRCH;
break;
}
/*
* Treat processes that are in the process of exiting
* as if they were not present.
*/
if (p->p_flag & P_WEXIT)
error = ESRCH;
PROC_UNLOCK(p); /* pfind() returns a locked process */
if (error == 0)
error = pmc_detach_process(p, pm);
}
break;
/*
* Retrieve the MSR number associated with the counter
* 'pmc_id'. This allows processes to directly use RDPMC
* instructions to read their PMCs, without the overhead of a
* system call.
*/
case PMC_OP_PMCGETMSR:
{
int adjri, ri;
struct pmc *pm;
struct pmc_target *pt;
struct pmc_op_getmsr gm;
struct pmc_classdep *pcd;
PMC_DOWNGRADE_SX();
if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
break;
if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
break;
/*
* The allocated PMC has to be a process virtual PMC,
* i.e., of type MODE_T[CS]. Global PMCs can only be
* read using the PMCREAD operation since they may be
* allocated on a different CPU than the one we could
* be running on at the time of the RDPMC instruction.
*
* The GETMSR operation is not allowed for PMCs that
* are inherited across processes.
*/
if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
(pm->pm_flags & PMC_F_DESCENDANTS)) {
error = EINVAL;
break;
}
/*
* It only makes sense to use a RDPMC (or its
* equivalent instruction on non-x86 architectures) on
* a process that has allocated and attached a PMC to
* itself. Conversely the PMC is only allowed to have
* one process attached to it -- its owner.
*/
if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
LIST_NEXT(pt, pt_next) != NULL ||
pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
error = EINVAL;
break;
}
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
/* PMC class has no 'GETMSR' support */
if (pcd->pcd_get_msr == NULL) {
error = ENOSYS;
break;
}
if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
break;
if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
break;
/*
* Mark our process as using MSRs. Update machine
* state using a forced context switch.
*/
pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
pmc_force_context_switch();
}
break;
/*
* Release an allocated PMC
*/
case PMC_OP_PMCRELEASE:
{
pmc_id_t pmcid;
struct pmc *pm;
struct pmc_owner *po;
struct pmc_op_simple sp;
/*
* Find PMC pointer for the named PMC.
*
* Use pmc_release_pmc_descriptor() to switch off the
* PMC, remove all its target threads, and remove the
* PMC from its owner's list.
*
* Remove the owner record if this is the last PMC
* owned.
*
* Free up space.
*/
if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
break;
pmcid = sp.pm_pmcid;
if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
break;
po = pm->pm_owner;
pmc_release_pmc_descriptor(pm);
pmc_maybe_remove_owner(po);
pmc_destroy_pmc_descriptor(pm);
}
break;
/*
* Read and/or write a PMC.
*/
case PMC_OP_PMCRW:
{
int adjri;
struct pmc *pm;
uint32_t cpu, ri;
pmc_value_t oldvalue;
struct pmc_binding pb;
struct pmc_op_pmcrw prw;
struct pmc_classdep *pcd;
struct pmc_op_pmcrw *pprw;
PMC_DOWNGRADE_SX();
if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
break;
ri = 0;
- PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
+ PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
prw.pm_flags);
/* must have at least one flag set */
if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
error = EINVAL;
break;
}
/* locate pmc descriptor */
if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
break;
/* Can't read a PMC that hasn't been started. */
if (pm->pm_state != PMC_STATE_ALLOCATED &&
pm->pm_state != PMC_STATE_STOPPED &&
pm->pm_state != PMC_STATE_RUNNING) {
error = EINVAL;
break;
}
/* writing a new value is allowed only for 'STOPPED' pmcs */
if (pm->pm_state == PMC_STATE_RUNNING &&
(prw.pm_flags & PMC_F_NEWVALUE)) {
error = EBUSY;
break;
}
if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
/*
* If this PMC is attached to its owner (i.e.,
* the process requesting this operation) and
* is running, then attempt to get an
* upto-date reading from hardware for a READ.
* Writes are only allowed when the PMC is
* stopped, so only update the saved value
* field.
*
* If the PMC is not running, or is not
* attached to its owner, read/write to the
* savedvalue field.
*/
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
mtx_pool_lock_spin(pmc_mtxpool, pm);
cpu = curthread->td_oncpu;
if (prw.pm_flags & PMC_F_OLDVALUE) {
if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
(pm->pm_state == PMC_STATE_RUNNING))
error = (*pcd->pcd_read_pmc)(cpu, adjri,
&oldvalue);
else
oldvalue = pm->pm_gv.pm_savedvalue;
}
if (prw.pm_flags & PMC_F_NEWVALUE)
pm->pm_gv.pm_savedvalue = prw.pm_value;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
} else { /* System mode PMCs */
cpu = PMC_TO_CPU(pm);
ri = PMC_TO_ROWINDEX(pm);
pcd = pmc_ri_to_classdep(md, ri, &adjri);
if (!pmc_cpu_is_active(cpu)) {
error = ENXIO;
break;
}
/* move this thread to CPU 'cpu' */
pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu);
critical_enter();
/* save old value */
if (prw.pm_flags & PMC_F_OLDVALUE)
if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
&oldvalue)))
goto error;
/* write out new value */
if (prw.pm_flags & PMC_F_NEWVALUE)
error = (*pcd->pcd_write_pmc)(cpu, adjri,
prw.pm_value);
error:
critical_exit();
pmc_restore_cpu_binding(&pb);
if (error)
break;
}
pprw = (struct pmc_op_pmcrw *) arg;
#ifdef HWPMC_DEBUG
if (prw.pm_flags & PMC_F_NEWVALUE)
- PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
+ PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
ri, prw.pm_value, oldvalue);
else if (prw.pm_flags & PMC_F_OLDVALUE)
- PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
+ PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
#endif
/* return old value if requested */
if (prw.pm_flags & PMC_F_OLDVALUE)
if ((error = copyout(&oldvalue, &pprw->pm_value,
sizeof(prw.pm_value))))
break;
}
break;
/*
* Set the sampling rate for a sampling mode PMC and the
* initial count for a counting mode PMC.
*/
case PMC_OP_PMCSETCOUNT:
{
struct pmc *pm;
struct pmc_op_pmcsetcount sc;
PMC_DOWNGRADE_SX();
if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
break;
if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
break;
if (pm->pm_state == PMC_STATE_RUNNING) {
error = EBUSY;
break;
}
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pm->pm_sc.pm_reloadcount = sc.pm_count;
else
pm->pm_sc.pm_initial = sc.pm_count;
}
break;
/*
* Start a PMC.
*/
case PMC_OP_PMCSTART:
{
pmc_id_t pmcid;
struct pmc *pm;
struct pmc_op_simple sp;
sx_assert(&pmc_sx, SX_XLOCKED);
if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
break;
pmcid = sp.pm_pmcid;
if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
break;
KASSERT(pmcid == pm->pm_id,
("[pmc,%d] pmcid %x != id %x", __LINE__,
pm->pm_id, pmcid));
if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
break;
else if (pm->pm_state != PMC_STATE_STOPPED &&
pm->pm_state != PMC_STATE_ALLOCATED) {
error = EINVAL;
break;
}
error = pmc_start(pm);
}
break;
/*
* Stop a PMC.
*/
case PMC_OP_PMCSTOP:
{
pmc_id_t pmcid;
struct pmc *pm;
struct pmc_op_simple sp;
PMC_DOWNGRADE_SX();
if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
break;
pmcid = sp.pm_pmcid;
/*
* Mark the PMC as inactive and invoke the MD stop
* routines if needed.
*/
if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
break;
KASSERT(pmcid == pm->pm_id,
("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
pm->pm_id, pmcid));
if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
break;
else if (pm->pm_state != PMC_STATE_RUNNING) {
error = EINVAL;
break;
}
error = pmc_stop(pm);
}
break;
/*
* Write a user supplied value to the log file.
*/
case PMC_OP_WRITELOG:
{
struct pmc_op_writelog wl;
struct pmc_owner *po;
PMC_DOWNGRADE_SX();
if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
break;
if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
error = EINVAL;
break;
}
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
error = EINVAL;
break;
}
error = pmclog_process_userlog(po, &wl);
}
break;
default:
error = EINVAL;
break;
}
if (is_sx_locked != 0) {
if (is_sx_downgraded)
sx_sunlock(&pmc_sx);
else
sx_xunlock(&pmc_sx);
}
if (error)
atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
PICKUP_GIANT();
return error;
}
/*
* Helper functions
*/
/*
* Mark the thread as needing callchain capture and post an AST. The
* actual callchain capture will be done in a context where it is safe
* to take page faults.
*/
static void
pmc_post_callchain_callback(void)
{
struct thread *td;
td = curthread;
/*
* If there is multiple PMCs for the same interrupt ignore new post
*/
if (td->td_pflags & TDP_CALLCHAIN)
return;
/*
* Mark this thread as needing callchain capture.
* `td->td_pflags' will be safe to touch because this thread
* was in user space when it was interrupted.
*/
td->td_pflags |= TDP_CALLCHAIN;
/*
* Don't let this thread migrate between CPUs until callchain
* capture completes.
*/
sched_pin();
return;
}
/*
* Interrupt processing.
*
* Find a free slot in the per-cpu array of samples and capture the
* current callchain there. If a sample was successfully added, a bit
* is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
* needs to be invoked from the clock handler.
*
* This function is meant to be called from an NMI handler. It cannot
* use any of the locking primitives supplied by the OS.
*/
int
pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
int inuserspace)
{
int error, callchaindepth;
struct thread *td;
struct pmc_sample *ps;
struct pmc_samplebuffer *psb;
error = 0;
/*
* Allocate space for a sample buffer.
*/
psb = pmc_pcpu[cpu]->pc_sb[ring];
ps = psb->ps_write;
if (ps->ps_nsamples) { /* in use, reader hasn't caught up */
pm->pm_stalled = 1;
atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
- PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
+ PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
cpu, pm, (void *) tf, inuserspace,
(int) (psb->ps_write - psb->ps_samples),
(int) (psb->ps_read - psb->ps_samples));
error = ENOMEM;
goto done;
}
/* Fill in entry. */
- PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
+ PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
(void *) tf, inuserspace,
(int) (psb->ps_write - psb->ps_samples),
(int) (psb->ps_read - psb->ps_samples));
KASSERT(pm->pm_runcount >= 0,
("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
pm->pm_runcount));
atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */
ps->ps_pmc = pm;
if ((td = curthread) && td->td_proc)
ps->ps_pid = td->td_proc->p_pid;
else
ps->ps_pid = -1;
ps->ps_cpu = cpu;
ps->ps_td = td;
ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
pmc_callchaindepth : 1;
if (callchaindepth == 1)
ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
else {
/*
* Kernel stack traversals can be done immediately,
* while we defer to an AST for user space traversals.
*/
if (!inuserspace) {
callchaindepth =
pmc_save_kernel_callchain(ps->ps_pc,
callchaindepth, tf);
} else {
pmc_post_callchain_callback();
callchaindepth = PMC_SAMPLE_INUSE;
}
}
ps->ps_nsamples = callchaindepth; /* mark entry as in use */
/* increment write pointer, modulo ring buffer size */
ps++;
if (ps == psb->ps_fence)
psb->ps_write = psb->ps_samples;
else
psb->ps_write = ps;
done:
/* mark CPU as needing processing */
CPU_SET_ATOMIC(cpu, &pmc_cpumask);
return (error);
}
/*
* Capture a user call chain. This function will be called from ast()
* before control returns to userland and before the process gets
* rescheduled.
*/
static void
pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
{
int i;
struct pmc *pm;
struct thread *td;
struct pmc_sample *ps;
struct pmc_samplebuffer *psb;
#ifdef INVARIANTS
int ncallchains;
#endif
psb = pmc_pcpu[cpu]->pc_sb[ring];
td = curthread;
KASSERT(td->td_pflags & TDP_CALLCHAIN,
("[pmc,%d] Retrieving callchain for thread that doesn't want it",
__LINE__));
#ifdef INVARIANTS
ncallchains = 0;
#endif
/*
* Iterate through all deferred callchain requests.
*/
ps = psb->ps_samples;
for (i = 0; i < pmc_nsamples; i++, ps++) {
if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
continue;
if (ps->ps_td != td)
continue;
KASSERT(ps->ps_cpu == cpu,
("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
ps->ps_cpu, PCPU_GET(cpuid)));
pm = ps->ps_pmc;
KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
("[pmc,%d] Retrieving callchain for PMC that doesn't "
"want it", __LINE__));
KASSERT(pm->pm_runcount > 0,
("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount));
/*
* Retrieve the callchain and mark the sample buffer
* as 'processable' by the timer tick sweep code.
*/
ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
pmc_callchaindepth, tf);
#ifdef INVARIANTS
ncallchains++;
#endif
}
KASSERT(ncallchains > 0,
("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
cpu));
KASSERT(td->td_pinned == 1,
("[pmc,%d] invalid td_pinned value", __LINE__));
sched_unpin(); /* Can migrate safely now. */
return;
}
/*
* Process saved PC samples.
*/
static void
pmc_process_samples(int cpu, int ring)
{
struct pmc *pm;
int adjri, n;
struct thread *td;
struct pmc_owner *po;
struct pmc_sample *ps;
struct pmc_classdep *pcd;
struct pmc_samplebuffer *psb;
KASSERT(PCPU_GET(cpuid) == cpu,
("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
PCPU_GET(cpuid), cpu));
psb = pmc_pcpu[cpu]->pc_sb[ring];
for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
ps = psb->ps_read;
if (ps->ps_nsamples == PMC_SAMPLE_FREE)
break;
pm = ps->ps_pmc;
KASSERT(pm->pm_runcount > 0,
("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
pm->pm_runcount));
po = pm->pm_owner;
KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
pm, PMC_TO_MODE(pm)));
/* Ignore PMCs that have been switched off */
if (pm->pm_state != PMC_STATE_RUNNING)
goto entrydone;
/* If there is a pending AST wait for completion */
if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
/* Need a rescan at a later time. */
CPU_SET_ATOMIC(cpu, &pmc_cpumask);
break;
}
- PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
+ PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
pm, ps->ps_nsamples, ps->ps_flags,
(int) (psb->ps_write - psb->ps_samples),
(int) (psb->ps_read - psb->ps_samples));
/*
* If this is a process-mode PMC that is attached to
* its owner, and if the PC is in user mode, update
* profiling statistics like timer-based profiling
* would have done.
*/
if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
if (ps->ps_flags & PMC_CC_F_USERSPACE) {
td = FIRST_THREAD_IN_PROC(po->po_owner);
addupc_intr(td, ps->ps_pc[0], 1);
}
goto entrydone;
}
/*
* Otherwise, this is either a sampling mode PMC that
* is attached to a different process than its owner,
* or a system-wide sampling PMC. Dispatch a log
* entry to the PMC's owner process.
*/
pmclog_process_callchain(pm, ps);
entrydone:
ps->ps_nsamples = 0; /* mark entry as free */
atomic_subtract_rel_int(&pm->pm_runcount, 1);
/* increment read pointer, modulo sample size */
if (++ps == psb->ps_fence)
psb->ps_read = psb->ps_samples;
else
psb->ps_read = ps;
}
atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
/* Do not re-enable stalled PMCs if we failed to process any samples */
if (n == 0)
return;
/*
* Restart any stalled sampling PMCs on this CPU.
*
* If the NMI handler sets the pm_stalled field of a PMC after
* the check below, we'll end up processing the stalled PMC at
* the next hardclock tick.
*/
for (n = 0; n < md->pmd_npmc; n++) {
pcd = pmc_ri_to_classdep(md, n, &adjri);
KASSERT(pcd != NULL,
("[pmc,%d] null pcd ri=%d", __LINE__, n));
(void) (*pcd->pcd_get_config)(cpu,adjri,&pm);
if (pm == NULL || /* !cfg'ed */
pm->pm_state != PMC_STATE_RUNNING || /* !active */
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
pm->pm_stalled == 0) /* !stalled */
continue;
pm->pm_stalled = 0;
(*pcd->pcd_start_pmc)(cpu, adjri);
}
}
/*
* Event handlers.
*/
/*
* Handle a process exit.
*
* Remove this process from all hash tables. If this process
* owned any PMCs, turn off those PMCs and deallocate them,
* removing any associations with target processes.
*
* This function will be called by the last 'thread' of a
* process.
*
* XXX This eventhandler gets called early in the exit process.
* Consider using a 'hook' invocation from thread_exit() or equivalent
* spot. Another negative is that kse_exit doesn't seem to call
* exit1() [??].
*
*/
static void
pmc_process_exit(void *arg __unused, struct proc *p)
{
struct pmc *pm;
int adjri, cpu;
unsigned int ri;
int is_using_hwpmcs;
struct pmc_owner *po;
struct pmc_process *pp;
struct pmc_classdep *pcd;
pmc_value_t newvalue, tmp;
PROC_LOCK(p);
is_using_hwpmcs = p->p_flag & P_HWPMC;
PROC_UNLOCK(p);
/*
* Log a sysexit event to all SS PMC owners.
*/
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_sysexit(po, p->p_pid);
if (!is_using_hwpmcs)
return;
PMC_GET_SX_XLOCK();
- PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
+ PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
p->p_comm);
/*
* Since this code is invoked by the last thread in an exiting
* process, we would have context switched IN at some prior
* point. However, with PREEMPTION, kernel mode context
* switches may happen any time, so we want to disable a
* context switch OUT till we get any PMCs targetting this
* process off the hardware.
*
* We also need to atomically remove this process'
* entry from our target process hash table, using
* PMC_FLAG_REMOVE.
*/
- PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
+ PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
p->p_comm);
critical_enter(); /* no preemption */
cpu = curthread->td_oncpu;
if ((pp = pmc_find_process_descriptor(p,
PMC_FLAG_REMOVE)) != NULL) {
- PMCDBG(PRC,EXT,2,
+ PMCDBG2(PRC,EXT,2,
"process-exit proc=%p pmc-process=%p", p, pp);
/*
* The exiting process could the target of
* some PMCs which will be running on
* currently executing CPU.
*
* We need to turn these PMCs off like we
* would do at context switch OUT time.
*/
for (ri = 0; ri < md->pmd_npmc; ri++) {
/*
* Pick up the pmc pointer from hardware
* state similar to the CSW_OUT code.
*/
pm = NULL;
pcd = pmc_ri_to_classdep(md, ri, &adjri);
(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);
- PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
+ PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
if (pm == NULL ||
!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
continue;
- PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
+ PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
"state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
pm, pm->pm_state);
KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
__LINE__, PMC_TO_ROWINDEX(pm), ri));
KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
("[pmc,%d] pm %p != pp_pmcs[%d] %p",
__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
(void) pcd->pcd_stop_pmc(cpu, adjri);
KASSERT(pm->pm_runcount > 0,
("[pmc,%d] bad runcount ri %d rc %d",
__LINE__, ri, pm->pm_runcount));
/* Stop hardware only if it is actually running */
if (pm->pm_state == PMC_STATE_RUNNING &&
pm->pm_stalled == 0) {
pcd->pcd_read_pmc(cpu, adjri, &newvalue);
tmp = newvalue -
PMC_PCPU_SAVED(cpu,ri);
mtx_pool_lock_spin(pmc_mtxpool, pm);
pm->pm_gv.pm_savedvalue += tmp;
pp->pp_pmcs[ri].pp_pmcval += tmp;
mtx_pool_unlock_spin(pmc_mtxpool, pm);
}
atomic_subtract_rel_int(&pm->pm_runcount,1);
KASSERT((int) pm->pm_runcount >= 0,
("[pmc,%d] runcount is %d", __LINE__, ri));
(void) pcd->pcd_config_pmc(cpu, adjri, NULL);
}
/*
* Inform the MD layer of this pseudo "context switch
* out"
*/
(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
critical_exit(); /* ok to be pre-empted now */
/*
* Unlink this process from the PMCs that are
* targetting it. This will send a signal to
* all PMC owner's whose PMCs are orphaned.
*
* Log PMC value at exit time if requested.
*/
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
pmclog_process_procexit(pm, pp);
pmc_unlink_target_process(pm, pp);
}
free(pp, M_PMC);
} else
critical_exit(); /* pp == NULL */
/*
* If the process owned PMCs, free them up and free up
* memory.
*/
if ((po = pmc_find_owner_descriptor(p)) != NULL) {
pmc_remove_owner(po);
pmc_destroy_owner_descriptor(po);
}
sx_xunlock(&pmc_sx);
}
/*
* Handle a process fork.
*
* If the parent process 'p1' is under HWPMC monitoring, then copy
* over any attached PMCs that have 'do_descendants' semantics.
*/
static void
pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
int flags)
{
int is_using_hwpmcs;
unsigned int ri;
uint32_t do_descendants;
struct pmc *pm;
struct pmc_owner *po;
struct pmc_process *ppnew, *ppold;
(void) flags; /* unused parameter */
PROC_LOCK(p1);
is_using_hwpmcs = p1->p_flag & P_HWPMC;
PROC_UNLOCK(p1);
/*
* If there are system-wide sampling PMCs active, we need to
* log all fork events to their owner's logs.
*/
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
if (!is_using_hwpmcs)
return;
PMC_GET_SX_XLOCK();
- PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
+ PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
p1->p_pid, p1->p_comm, newproc);
/*
* If the parent process (curthread->td_proc) is a
* target of any PMCs, look for PMCs that are to be
* inherited, and link these into the new process
* descriptor.
*/
if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
PMC_FLAG_NONE)) == NULL)
goto done; /* nothing to do */
do_descendants = 0;
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
if (do_descendants == 0) /* nothing to do */
goto done;
/* allocate a descriptor for the new process */
if ((ppnew = pmc_find_process_descriptor(newproc,
PMC_FLAG_ALLOCATE)) == NULL)
goto done;
/*
* Run through all PMCs that were targeting the old process
* and which specified F_DESCENDANTS and attach them to the
* new process.
*
* Log the fork event to all owners of PMCs attached to this
* process, if not already logged.
*/
for (ri = 0; ri < md->pmd_npmc; ri++)
if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
(pm->pm_flags & PMC_F_DESCENDANTS)) {
pmc_link_target_process(pm, ppnew);
po = pm->pm_owner;
if (po->po_sscount == 0 &&
po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_procfork(po, p1->p_pid,
newproc->p_pid);
}
/*
* Now mark the new process as being tracked by this driver.
*/
PROC_LOCK(newproc);
newproc->p_flag |= P_HWPMC;
PROC_UNLOCK(newproc);
done:
sx_xunlock(&pmc_sx);
}
static void
pmc_kld_load(void *arg __unused, linker_file_t lf)
{
struct pmc_owner *po;
sx_slock(&pmc_sx);
/*
* Notify owners of system sampling PMCs about KLD operations.
*/
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_map_in(po, (pid_t) -1,
(uintfptr_t) lf->address, lf->filename);
/*
* TODO: Notify owners of (all) process-sampling PMCs too.
*/
sx_sunlock(&pmc_sx);
}
static void
pmc_kld_unload(void *arg __unused, const char *filename __unused,
caddr_t address, size_t size)
{
struct pmc_owner *po;
sx_slock(&pmc_sx);
LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmclog_process_map_out(po, (pid_t) -1,
(uintfptr_t) address, (uintfptr_t) address + size);
/*
* TODO: Notify owners of process-sampling PMCs.
*/
sx_sunlock(&pmc_sx);
}
/*
* initialization
*/
static const char *pmc_name_of_pmcclass[] = {
#undef __PMC_CLASS
#define __PMC_CLASS(N) #N ,
__PMC_CLASSES()
};
/*
* Base class initializer: allocate structure and set default classes.
*/
struct pmc_mdep *
pmc_mdep_alloc(int nclasses)
{
struct pmc_mdep *md;
int n;
/* SOFT + md classes */
n = 1 + nclasses;
md = malloc(sizeof(struct pmc_mdep) + n *
sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
md->pmd_nclass = n;
/* Add base class. */
pmc_soft_initialize(md);
return md;
}
void
pmc_mdep_free(struct pmc_mdep *md)
{
pmc_soft_finalize(md);
free(md, M_PMC);
}
static int
generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc; (void) pp;
return (0);
}
static int
generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc; (void) pp;
return (0);
}
static struct pmc_mdep *
pmc_generic_cpu_initialize(void)
{
struct pmc_mdep *md;
md = pmc_mdep_alloc(0);
md->pmd_cputype = PMC_CPU_GENERIC;
md->pmd_pcpu_init = NULL;
md->pmd_pcpu_fini = NULL;
md->pmd_switch_in = generic_switch_in;
md->pmd_switch_out = generic_switch_out;
return (md);
}
static void
pmc_generic_cpu_finalize(struct pmc_mdep *md)
{
(void) md;
}
static int
pmc_initialize(void)
{
int c, cpu, error, n, ri;
unsigned int maxcpu;
struct pmc_binding pb;
struct pmc_sample *ps;
struct pmc_classdep *pcd;
struct pmc_samplebuffer *sb;
md = NULL;
error = 0;
#ifdef HWPMC_DEBUG
/* parse debug flags first */
if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
pmc_debugstr, sizeof(pmc_debugstr)))
pmc_debugflags_parse(pmc_debugstr,
pmc_debugstr+strlen(pmc_debugstr));
#endif
- PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
+ PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
/* check kernel version */
if (pmc_kernel_version != PMC_VERSION) {
if (pmc_kernel_version == 0)
printf("hwpmc: this kernel has not been compiled with "
"'options HWPMC_HOOKS'.\n");
else
printf("hwpmc: kernel version (0x%x) does not match "
"module version (0x%x).\n", pmc_kernel_version,
PMC_VERSION);
return EPROGMISMATCH;
}
/*
* check sysctl parameters
*/
if (pmc_hashsize <= 0) {
(void) printf("hwpmc: tunable \"hashsize\"=%d must be "
"greater than zero.\n", pmc_hashsize);
pmc_hashsize = PMC_HASH_SIZE;
}
if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
(void) printf("hwpmc: tunable \"nsamples\"=%d out of "
"range.\n", pmc_nsamples);
pmc_nsamples = PMC_NSAMPLES;
}
if (pmc_callchaindepth <= 0 ||
pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
(void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
"range - using %d.\n", pmc_callchaindepth,
PMC_CALLCHAIN_DEPTH_MAX);
pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
}
md = pmc_md_initialize();
if (md == NULL) {
/* Default to generic CPU. */
md = pmc_generic_cpu_initialize();
if (md == NULL)
return (ENOSYS);
}
KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
("[pmc,%d] no classes or pmcs", __LINE__));
/* Compute the map from row-indices to classdep pointers. */
pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) *
md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO);
for (n = 0; n < md->pmd_npmc; n++)
pmc_rowindex_to_classdep[n] = NULL;
for (ri = c = 0; c < md->pmd_nclass; c++) {
pcd = &md->pmd_classdep[c];
for (n = 0; n < pcd->pcd_num; n++, ri++)
pmc_rowindex_to_classdep[ri] = pcd;
}
KASSERT(ri == md->pmd_npmc,
("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
ri, md->pmd_npmc));
maxcpu = pmc_cpu_max();
/* allocate space for the per-cpu array */
pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
M_WAITOK|M_ZERO);
/* per-cpu 'saved values' for managing process-mode PMCs */
pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
M_PMC, M_WAITOK);
/* Perform CPU-dependent initialization. */
pmc_save_cpu_binding(&pb);
error = 0;
for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
if (!pmc_cpu_is_active(cpu))
continue;
pmc_select_cpu(cpu);
pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
M_WAITOK|M_ZERO);
if (md->pmd_pcpu_init)
error = md->pmd_pcpu_init(md, cpu);
for (n = 0; error == 0 && n < md->pmd_nclass; n++)
error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
}
pmc_restore_cpu_binding(&pb);
if (error)
return (error);
/* allocate space for the sample array */
for (cpu = 0; cpu < maxcpu; cpu++) {
if (!pmc_cpu_is_active(cpu))
continue;
sb = malloc(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO);
sb->ps_read = sb->ps_write = sb->ps_samples;
sb->ps_fence = sb->ps_samples + pmc_nsamples;
KASSERT(pmc_pcpu[cpu] != NULL,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
sb = malloc(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO);
sb->ps_read = sb->ps_write = sb->ps_samples;
sb->ps_fence = sb->ps_samples + pmc_nsamples;
KASSERT(pmc_pcpu[cpu] != NULL,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
}
/* allocate space for the row disposition array */
pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
M_PMC, M_WAITOK|M_ZERO);
/* mark all PMCs as available */
for (n = 0; n < (int) md->pmd_npmc; n++)
PMC_MARK_ROW_FREE(n);
/* allocate thread hash tables */
pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
&pmc_ownerhashmask);
pmc_processhash = hashinit(pmc_hashsize, M_PMC,
&pmc_processhashmask);
mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
MTX_SPIN);
LIST_INIT(&pmc_ss_owners);
pmc_ss_count = 0;
/* allocate a pool of spin mutexes */
pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
MTX_SPIN);
- PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
+ PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
"targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
pmc_processhash, pmc_processhashmask);
/* register process {exit,fork,exec} handlers */
pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
/* register kld event handlers */
pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
NULL, EVENTHANDLER_PRI_ANY);
pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
NULL, EVENTHANDLER_PRI_ANY);
/* initialize logging */
pmclog_initialize();
/* set hook functions */
pmc_intr = md->pmd_intr;
pmc_hook = pmc_hook_handler;
if (error == 0) {
printf(PMC_MODULE_NAME ":");
for (n = 0; n < (int) md->pmd_nclass; n++) {
pcd = &md->pmd_classdep[n];
printf(" %s/%d/%d/0x%b",
pmc_name_of_pmcclass[pcd->pcd_class],
pcd->pcd_num,
pcd->pcd_width,
pcd->pcd_caps,
"\20"
"\1INT\2USR\3SYS\4EDG\5THR"
"\6REA\7WRI\10INV\11QUA\12PRC"
"\13TAG\14CSC");
}
printf("\n");
}
return (error);
}
/* prepare to be unloaded */
static void
pmc_cleanup(void)
{
int c, cpu;
unsigned int maxcpu;
struct pmc_ownerhash *ph;
struct pmc_owner *po, *tmp;
struct pmc_binding pb;
#ifdef HWPMC_DEBUG
struct pmc_processhash *prh;
#endif
- PMCDBG(MOD,INI,0, "%s", "cleanup");
+ PMCDBG0(MOD,INI,0, "cleanup");
/* switch off sampling */
CPU_ZERO(&pmc_cpumask);
pmc_intr = NULL;
sx_xlock(&pmc_sx);
if (pmc_hook == NULL) { /* being unloaded already */
sx_xunlock(&pmc_sx);
return;
}
pmc_hook = NULL; /* prevent new threads from entering module */
/* deregister event handlers */
EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
/* send SIGBUS to all owner threads, free up allocations */
if (pmc_ownerhash)
for (ph = pmc_ownerhash;
ph <= &pmc_ownerhash[pmc_ownerhashmask];
ph++) {
LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
pmc_remove_owner(po);
/* send SIGBUS to owner processes */
- PMCDBG(MOD,INI,2, "cleanup signal proc=%p "
+ PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
"(%d, %s)", po->po_owner,
po->po_owner->p_pid,
po->po_owner->p_comm);
PROC_LOCK(po->po_owner);
kern_psignal(po->po_owner, SIGBUS);
PROC_UNLOCK(po->po_owner);
pmc_destroy_owner_descriptor(po);
}
}
/* reclaim allocated data structures */
if (pmc_mtxpool)
mtx_pool_destroy(&pmc_mtxpool);
mtx_destroy(&pmc_processhash_mtx);
if (pmc_processhash) {
#ifdef HWPMC_DEBUG
struct pmc_process *pp;
- PMCDBG(MOD,INI,3, "%s", "destroy process hash");
+ PMCDBG0(MOD,INI,3, "destroy process hash");
for (prh = pmc_processhash;
prh <= &pmc_processhash[pmc_processhashmask];
prh++)
LIST_FOREACH(pp, prh, pp_next)
- PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
+ PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
#endif
hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
pmc_processhash = NULL;
}
if (pmc_ownerhash) {
- PMCDBG(MOD,INI,3, "%s", "destroy owner hash");
+ PMCDBG0(MOD,INI,3, "destroy owner hash");
hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
pmc_ownerhash = NULL;
}
KASSERT(LIST_EMPTY(&pmc_ss_owners),
("[pmc,%d] Global SS owner list not empty", __LINE__));
KASSERT(pmc_ss_count == 0,
("[pmc,%d] Global SS count not empty", __LINE__));
/* do processor and pmc-class dependent cleanup */
maxcpu = pmc_cpu_max();
- PMCDBG(MOD,INI,3, "%s", "md cleanup");
+ PMCDBG0(MOD,INI,3, "md cleanup");
if (md) {
pmc_save_cpu_binding(&pb);
for (cpu = 0; cpu < maxcpu; cpu++) {
- PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
+ PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
cpu, pmc_pcpu[cpu]);
if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL)
continue;
pmc_select_cpu(cpu);
for (c = 0; c < md->pmd_nclass; c++)
md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
if (md->pmd_pcpu_fini)
md->pmd_pcpu_fini(md, cpu);
}
if (md->pmd_cputype == PMC_CPU_GENERIC)
pmc_generic_cpu_finalize(md);
else
pmc_md_finalize(md);
pmc_mdep_free(md);
md = NULL;
pmc_restore_cpu_binding(&pb);
}
/* Free per-cpu descriptors. */
for (cpu = 0; cpu < maxcpu; cpu++) {
if (!pmc_cpu_is_active(cpu))
continue;
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
cpu));
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
cpu));
free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
free(pmc_pcpu[cpu], M_PMC);
}
free(pmc_pcpu, M_PMC);
pmc_pcpu = NULL;
free(pmc_pcpu_saved, M_PMC);
pmc_pcpu_saved = NULL;
if (pmc_pmcdisp) {
free(pmc_pmcdisp, M_PMC);
pmc_pmcdisp = NULL;
}
if (pmc_rowindex_to_classdep) {
free(pmc_rowindex_to_classdep, M_PMC);
pmc_rowindex_to_classdep = NULL;
}
pmclog_shutdown();
sx_xunlock(&pmc_sx); /* we are done */
}
/*
* The function called at load/unload.
*/
static int
load (struct module *module __unused, int cmd, void *arg __unused)
{
int error;
error = 0;
switch (cmd) {
case MOD_LOAD :
/* initialize the subsystem */
error = pmc_initialize();
if (error != 0)
break;
- PMCDBG(MOD,INI,1, "syscall=%d maxcpu=%d",
+ PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
pmc_syscall_num, pmc_cpu_max());
break;
case MOD_UNLOAD :
case MOD_SHUTDOWN:
pmc_cleanup();
- PMCDBG(MOD,INI,1, "%s", "unloaded");
+ PMCDBG0(MOD,INI,1, "unloaded");
break;
default :
error = EINVAL; /* XXX should panic(9) */
break;
}
return error;
}
/* memory pool */
MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");
Index: head/sys/dev/hwpmc/hwpmc_mpc7xxx.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_mpc7xxx.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_mpc7xxx.c (revision 282658)
@@ -1,755 +1,755 @@
/*-
* Copyright (c) 2011 Justin Hibbits
* Copyright (c) 2005, Joseph Koshy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/pmc_mdep.h>
#include <machine/spr.h>
#include <machine/cpu.h>
#include "hwpmc_powerpc.h"
#define POWERPC_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \
PMC_CAP_SYSTEM | PMC_CAP_EDGE | \
PMC_CAP_THRESHOLD | PMC_CAP_READ | \
PMC_CAP_WRITE | PMC_CAP_INVERT | \
PMC_CAP_QUALIFIER)
#define PPC_SET_PMC1SEL(r, x) ((r & ~(SPR_MMCR0_PMC1SEL(0x3f))) | SPR_MMCR0_PMC1SEL(x))
#define PPC_SET_PMC2SEL(r, x) ((r & ~(SPR_MMCR0_PMC2SEL(0x3f))) | SPR_MMCR0_PMC2SEL(x))
#define PPC_SET_PMC3SEL(r, x) ((r & ~(SPR_MMCR1_PMC3SEL(0x1f))) | SPR_MMCR1_PMC3SEL(x))
#define PPC_SET_PMC4SEL(r, x) ((r & ~(SPR_MMCR1_PMC4SEL(0x1f))) | SPR_MMCR1_PMC4SEL(x))
#define PPC_SET_PMC5SEL(r, x) ((r & ~(SPR_MMCR1_PMC5SEL(0x1f))) | SPR_MMCR1_PMC5SEL(x))
#define PPC_SET_PMC6SEL(r, x) ((r & ~(SPR_MMCR1_PMC6SEL(0x3f))) | SPR_MMCR1_PMC6SEL(x))
/* Change this when we support more than just the 7450. */
#define MPC7XXX_MAX_PMCS 6
#define MPC7XXX_PMC_HAS_OVERFLOWED(x) (mpc7xxx_pmcn_read(x) & (0x1 << 31))
/*
* Things to improve on this:
* - It stops (clears to 0) the PMC and resets it at every context switch
* currently.
*/
/*
* This should work for every 32-bit PowerPC implementation I know of (G3 and G4
* specifically).
*/
struct mpc7xxx_event_code_map {
enum pmc_event pe_ev; /* enum value */
uint8_t pe_counter_mask; /* Which counter this can be counted in. */
uint8_t pe_code; /* numeric code */
};
#define PPC_PMC_MASK1 0
#define PPC_PMC_MASK2 1
#define PPC_PMC_MASK3 2
#define PPC_PMC_MASK4 3
#define PPC_PMC_MASK5 4
#define PPC_PMC_MASK6 5
#define PPC_PMC_MASK_ALL 0x3f
#define PMC_POWERPC_EVENT(id, mask, number) \
{ .pe_ev = PMC_EV_PPC7450_##id, .pe_counter_mask = mask, .pe_code = number }
static struct mpc7xxx_event_code_map mpc7xxx_event_codes[] = {
PMC_POWERPC_EVENT(CYCLE,PPC_PMC_MASK_ALL, 1),
PMC_POWERPC_EVENT(INSTR_COMPLETED, 0x0f, 2),
PMC_POWERPC_EVENT(TLB_BIT_TRANSITIONS, 0x0f, 3),
PMC_POWERPC_EVENT(INSTR_DISPATCHED, 0x0f, 4),
PMC_POWERPC_EVENT(PMON_EXCEPT, 0x0f, 5),
PMC_POWERPC_EVENT(PMON_SIG, 0x0f, 7),
PMC_POWERPC_EVENT(VPU_INSTR_COMPLETED, 0x03, 8),
PMC_POWERPC_EVENT(VFPU_INSTR_COMPLETED, 0x03, 9),
PMC_POWERPC_EVENT(VIU1_INSTR_COMPLETED, 0x03, 10),
PMC_POWERPC_EVENT(VIU2_INSTR_COMPLETED, 0x03, 11),
PMC_POWERPC_EVENT(MTVSCR_INSTR_COMPLETED, 0x03, 12),
PMC_POWERPC_EVENT(MTVRSAVE_INSTR_COMPLETED, 0x03, 13),
PMC_POWERPC_EVENT(VPU_INSTR_WAIT_CYCLES, 0x03, 14),
PMC_POWERPC_EVENT(VFPU_INSTR_WAIT_CYCLES, 0x03, 15),
PMC_POWERPC_EVENT(VIU1_INSTR_WAIT_CYCLES, 0x03, 16),
PMC_POWERPC_EVENT(VIU2_INSTR_WAIT_CYCLES, 0x03, 17),
PMC_POWERPC_EVENT(MFVSCR_SYNC_CYCLES, 0x03, 18),
PMC_POWERPC_EVENT(VSCR_SAT_SET, 0x03, 19),
PMC_POWERPC_EVENT(STORE_INSTR_COMPLETED, 0x03, 20),
PMC_POWERPC_EVENT(L1_INSTR_CACHE_MISSES, 0x03, 21),
PMC_POWERPC_EVENT(L1_DATA_SNOOPS, 0x03, 22),
PMC_POWERPC_EVENT(UNRESOLVED_BRANCHES, 0x01, 23),
PMC_POWERPC_EVENT(SPEC_BUFFER_CYCLES, 0x01, 24),
PMC_POWERPC_EVENT(BRANCH_UNIT_STALL_CYCLES, 0x01, 25),
PMC_POWERPC_EVENT(TRUE_BRANCH_TARGET_HITS, 0x01, 26),
PMC_POWERPC_EVENT(BRANCH_LINK_STAC_PREDICTED, 0x01, 27),
PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_DISPATCHES, 0x01, 28),
PMC_POWERPC_EVENT(CYCLES_THREE_INSTR_DISPATCHED, 0x01, 29),
PMC_POWERPC_EVENT(THRESHOLD_INSTR_QUEUE_ENTRIES_CYCLES, 0x01, 30),
PMC_POWERPC_EVENT(THRESHOLD_VEC_INSTR_QUEUE_ENTRIES_CYCLES, 0x01, 31),
PMC_POWERPC_EVENT(CYCLES_NO_COMPLETED_INSTRS, 0x01, 32),
PMC_POWERPC_EVENT(IU2_INSTR_COMPLETED, 0x01, 33),
PMC_POWERPC_EVENT(BRANCHES_COMPLETED, 0x01, 34),
PMC_POWERPC_EVENT(EIEIO_INSTR_COMPLETED, 0x01, 35),
PMC_POWERPC_EVENT(MTSPR_INSTR_COMPLETED, 0x01, 36),
PMC_POWERPC_EVENT(SC_INSTR_COMPLETED, 0x01, 37),
PMC_POWERPC_EVENT(LS_LM_COMPLETED, 0x01, 38),
PMC_POWERPC_EVENT(ITLB_HW_TABLE_SEARCH_CYCLES, 0x01, 39),
PMC_POWERPC_EVENT(DTLB_HW_SEARCH_CYCLES_OVER_THRESHOLD, 0x01, 40),
PMC_POWERPC_EVENT(L1_INSTR_CACHE_ACCESSES, 0x01, 41),
PMC_POWERPC_EVENT(INSTR_BKPT_MATCHES, 0x01, 42),
PMC_POWERPC_EVENT(L1_DATA_CACHE_LOAD_MISS_CYCLES_OVER_THRESHOLD, 0x01, 43),
PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_ON_MODIFIED, 0x01, 44),
PMC_POWERPC_EVENT(LOAD_MISS_ALIAS, 0x01, 45),
PMC_POWERPC_EVENT(LOAD_MISS_ALIAS_ON_TOUCH, 0x01, 46),
PMC_POWERPC_EVENT(TOUCH_ALIAS, 0x01, 47),
PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_CASTOUT_QUEUE, 0x01, 48),
PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_CASTOUT, 0x01, 49),
PMC_POWERPC_EVENT(L1_DATA_SNOOP_HITS, 0x01, 50),
PMC_POWERPC_EVENT(WRITE_THROUGH_STORES, 0x01, 51),
PMC_POWERPC_EVENT(CACHE_INHIBITED_STORES, 0x01, 52),
PMC_POWERPC_EVENT(L1_DATA_LOAD_HIT, 0x01, 53),
PMC_POWERPC_EVENT(L1_DATA_TOUCH_HIT, 0x01, 54),
PMC_POWERPC_EVENT(L1_DATA_STORE_HIT, 0x01, 55),
PMC_POWERPC_EVENT(L1_DATA_TOTAL_HITS, 0x01, 56),
PMC_POWERPC_EVENT(DST_INSTR_DISPATCHED, 0x01, 57),
PMC_POWERPC_EVENT(REFRESHED_DSTS, 0x01, 58),
PMC_POWERPC_EVENT(SUCCESSFUL_DST_TABLE_SEARCHES, 0x01, 59),
PMC_POWERPC_EVENT(DSS_INSTR_COMPLETED, 0x01, 60),
PMC_POWERPC_EVENT(DST_STREAM_0_CACHE_LINE_FETCHES, 0x01, 61),
PMC_POWERPC_EVENT(VTQ_SUSPENDS_DUE_TO_CTX_CHANGE, 0x01, 62),
PMC_POWERPC_EVENT(VTQ_LINE_FETCH_HIT, 0x01, 63),
PMC_POWERPC_EVENT(VEC_LOAD_INSTR_COMPLETED, 0x01, 64),
PMC_POWERPC_EVENT(FP_STORE_INSTR_COMPLETED_IN_LSU, 0x01, 65),
PMC_POWERPC_EVENT(FPU_RENORMALIZATION, 0x01, 66),
PMC_POWERPC_EVENT(FPU_DENORMALIZATION, 0x01, 67),
PMC_POWERPC_EVENT(FP_STORE_CAUSES_STALL_IN_LSU, 0x01, 68),
PMC_POWERPC_EVENT(LD_ST_TRUE_ALIAS_STALL, 0x01, 70),
PMC_POWERPC_EVENT(LSU_INDEXED_ALIAS_STALL, 0x01, 71),
PMC_POWERPC_EVENT(LSU_ALIAS_VS_FSQ_WB0_WB1, 0x01, 72),
PMC_POWERPC_EVENT(LSU_ALIAS_VS_CSQ, 0x01, 73),
PMC_POWERPC_EVENT(LSU_LOAD_HIT_LINE_ALIAS_VS_CSQ0, 0x01, 74),
PMC_POWERPC_EVENT(LSU_LOAD_MISS_LINE_ALIAS_VS_CSQ0, 0x01, 75),
PMC_POWERPC_EVENT(LSU_TOUCH_LINE_ALIAS_VS_FSQ_WB0_WB1, 0x01, 76),
PMC_POWERPC_EVENT(LSU_TOUCH_ALIAS_VS_CSQ, 0x01, 77),
PMC_POWERPC_EVENT(LSU_LMQ_FULL_STALL, 0x01, 78),
PMC_POWERPC_EVENT(FP_LOAD_INSTR_COMPLETED_IN_LSU, 0x01, 79),
PMC_POWERPC_EVENT(FP_LOAD_SINGLE_INSTR_COMPLETED_IN_LSU, 0x01, 80),
PMC_POWERPC_EVENT(FP_LOAD_DOUBLE_COMPLETED_IN_LSU, 0x01, 81),
PMC_POWERPC_EVENT(LSU_RA_LATCH_STALL, 0x01, 82),
PMC_POWERPC_EVENT(LSU_LOAD_VS_STORE_QUEUE_ALIAS_STALL, 0x01, 83),
PMC_POWERPC_EVENT(LSU_LMQ_INDEX_ALIAS, 0x01, 84),
PMC_POWERPC_EVENT(LSU_STORE_QUEUE_INDEX_ALIAS, 0x01, 85),
PMC_POWERPC_EVENT(LSU_CSQ_FORWARDING, 0x01, 86),
PMC_POWERPC_EVENT(LSU_MISALIGNED_LOAD_FINISH, 0x01, 87),
PMC_POWERPC_EVENT(LSU_MISALIGN_STORE_COMPLETED, 0x01, 88),
PMC_POWERPC_EVENT(LSU_MISALIGN_STALL, 0x01, 89),
PMC_POWERPC_EVENT(FP_ONE_QUARTER_FPSCR_RENAMES_BUSY, 0x01, 90),
PMC_POWERPC_EVENT(FP_ONE_HALF_FPSCR_RENAMES_BUSY, 0x01, 91),
PMC_POWERPC_EVENT(FP_THREE_QUARTERS_FPSCR_RENAMES_BUSY, 0x01, 92),
PMC_POWERPC_EVENT(FP_ALL_FPSCR_RENAMES_BUSY, 0x01, 93),
PMC_POWERPC_EVENT(FP_DENORMALIZED_RESULT, 0x01, 94),
PMC_POWERPC_EVENT(L1_DATA_TOTAL_MISSES, 0x02, 23),
PMC_POWERPC_EVENT(DISPATCHES_TO_FPR_ISSUE_QUEUE, 0x02, 24),
PMC_POWERPC_EVENT(LSU_INSTR_COMPLETED, 0x02, 25),
PMC_POWERPC_EVENT(LOAD_INSTR_COMPLETED, 0x02, 26),
PMC_POWERPC_EVENT(SS_SM_INSTR_COMPLETED, 0x02, 27),
PMC_POWERPC_EVENT(TLBIE_INSTR_COMPLETED, 0x02, 28),
PMC_POWERPC_EVENT(LWARX_INSTR_COMPLETED, 0x02, 29),
PMC_POWERPC_EVENT(MFSPR_INSTR_COMPLETED, 0x02, 30),
PMC_POWERPC_EVENT(REFETCH_SERIALIZATION, 0x02, 31),
PMC_POWERPC_EVENT(COMPLETION_QUEUE_ENTRIES_OVER_THRESHOLD, 0x02, 32),
PMC_POWERPC_EVENT(CYCLES_ONE_INSTR_DISPATCHED, 0x02, 33),
PMC_POWERPC_EVENT(CYCLES_TWO_INSTR_COMPLETED, 0x02, 34),
PMC_POWERPC_EVENT(ITLB_NON_SPECULATIVE_MISSES, 0x02, 35),
PMC_POWERPC_EVENT(CYCLES_WAITING_FROM_L1_INSTR_CACHE_MISS, 0x02, 36),
PMC_POWERPC_EVENT(L1_DATA_LOAD_ACCESS_MISS, 0x02, 37),
PMC_POWERPC_EVENT(L1_DATA_TOUCH_MISS, 0x02, 38),
PMC_POWERPC_EVENT(L1_DATA_STORE_MISS, 0x02, 39),
PMC_POWERPC_EVENT(L1_DATA_TOUCH_MISS_CYCLES, 0x02, 40),
PMC_POWERPC_EVENT(L1_DATA_CYCLES_USED, 0x02, 41),
PMC_POWERPC_EVENT(DST_STREAM_1_CACHE_LINE_FETCHES, 0x02, 42),
PMC_POWERPC_EVENT(VTQ_STREAM_CANCELED_PREMATURELY, 0x02, 43),
PMC_POWERPC_EVENT(VTQ_RESUMES_DUE_TO_CTX_CHANGE, 0x02, 44),
PMC_POWERPC_EVENT(VTQ_LINE_FETCH_MISS, 0x02, 45),
PMC_POWERPC_EVENT(VTQ_LINE_FETCH, 0x02, 46),
PMC_POWERPC_EVENT(TLBIE_SNOOPS, 0x02, 47),
PMC_POWERPC_EVENT(L1_INSTR_CACHE_RELOADS, 0x02, 48),
PMC_POWERPC_EVENT(L1_DATA_CACHE_RELOADS, 0x02, 49),
PMC_POWERPC_EVENT(L1_DATA_CACHE_CASTOUTS_TO_L2, 0x02, 50),
PMC_POWERPC_EVENT(STORE_MERGE_GATHER, 0x02, 51),
PMC_POWERPC_EVENT(CACHEABLE_STORE_MERGE_TO_32_BYTES, 0x02, 52),
PMC_POWERPC_EVENT(DATA_BKPT_MATCHES, 0x02, 53),
PMC_POWERPC_EVENT(FALL_THROUGH_BRANCHES_PROCESSED, 0x02, 54),
PMC_POWERPC_EVENT(FIRST_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x02, 55),
PMC_POWERPC_EVENT(SECOND_SPECULATION_BUFFER_ACTIVE, 0x02, 56),
PMC_POWERPC_EVENT(BPU_STALL_ON_LR_DEPENDENCY, 0x02, 57),
PMC_POWERPC_EVENT(BTIC_MISS, 0x02, 58),
PMC_POWERPC_EVENT(BRANCH_LINK_STACK_CORRECTLY_RESOLVED, 0x02, 59),
PMC_POWERPC_EVENT(FPR_ISSUE_STALLED, 0x02, 60),
PMC_POWERPC_EVENT(SWITCHES_BETWEEN_PRIV_USER, 0x02, 61),
PMC_POWERPC_EVENT(LSU_COMPLETES_FP_STORE_SINGLE, 0x02, 62),
PMC_POWERPC_EVENT(CYCLES_TWO_INSTR_COMPLETED, 0x04, 8),
PMC_POWERPC_EVENT(CYCLES_ONE_INSTR_DISPATCHED, 0x04, 9),
PMC_POWERPC_EVENT(VR_ISSUE_QUEUE_DISPATCHES, 0x04, 10),
PMC_POWERPC_EVENT(VR_STALLS, 0x04, 11),
PMC_POWERPC_EVENT(GPR_RENAME_BUFFER_ENTRIES_OVER_THRESHOLD, 0x04, 12),
PMC_POWERPC_EVENT(FPR_ISSUE_QUEUE_ENTRIES, 0x04, 13),
PMC_POWERPC_EVENT(FPU_INSTR_COMPLETED, 0x04, 14),
PMC_POWERPC_EVENT(STWCX_INSTR_COMPLETED, 0x04, 15),
PMC_POWERPC_EVENT(LS_LM_INSTR_PIECES, 0x04, 16),
PMC_POWERPC_EVENT(ITLB_HW_SEARCH_CYCLES_OVER_THRESHOLD, 0x04, 17),
PMC_POWERPC_EVENT(DTLB_MISSES, 0x04, 18),
PMC_POWERPC_EVENT(CANCELLED_L1_INSTR_CACHE_MISSES, 0x04, 19),
PMC_POWERPC_EVENT(L1_DATA_CACHE_OP_HIT, 0x04, 20),
PMC_POWERPC_EVENT(L1_DATA_LOAD_MISS_CYCLES, 0x04, 21),
PMC_POWERPC_EVENT(L1_DATA_PUSHES, 0x04, 22),
PMC_POWERPC_EVENT(L1_DATA_TOTAL_MISS, 0x04, 23),
PMC_POWERPC_EVENT(VT2_FETCHES, 0x04, 24),
PMC_POWERPC_EVENT(TAKEN_BRANCHES_PROCESSED, 0x04, 25),
PMC_POWERPC_EVENT(BRANCH_FLUSHES, 0x04, 26),
PMC_POWERPC_EVENT(SECOND_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x04, 27),
PMC_POWERPC_EVENT(THIRD_SPECULATION_BUFFER_ACTIVE, 0x04, 28),
PMC_POWERPC_EVENT(BRANCH_UNIT_STALL_ON_CTR_DEPENDENCY, 0x04, 29),
PMC_POWERPC_EVENT(FAST_BTIC_HIT, 0x04, 30),
PMC_POWERPC_EVENT(BRANCH_LINK_STACK_MISPREDICTED, 0x04, 31),
PMC_POWERPC_EVENT(CYCLES_THREE_INSTR_COMPLETED, 0x08, 14),
PMC_POWERPC_EVENT(CYCLES_NO_INSTR_DISPATCHED, 0x08, 15),
PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_ENTRIES_OVER_THRESHOLD, 0x08, 16),
PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_STALLED, 0x08, 17),
PMC_POWERPC_EVENT(IU1_INSTR_COMPLETED, 0x08, 18),
PMC_POWERPC_EVENT(DSSALL_INSTR_COMPLETED, 0x08, 19),
PMC_POWERPC_EVENT(TLBSYNC_INSTR_COMPLETED, 0x08, 20),
PMC_POWERPC_EVENT(SYNC_INSTR_COMPLETED, 0x08, 21),
PMC_POWERPC_EVENT(SS_SM_INSTR_PIECES, 0x08, 22),
PMC_POWERPC_EVENT(DTLB_HW_SEARCH_CYCLES, 0x08, 23),
PMC_POWERPC_EVENT(SNOOP_RETRIES, 0x08, 24),
PMC_POWERPC_EVENT(SUCCESSFUL_STWCX, 0x08, 25),
PMC_POWERPC_EVENT(DST_STREAM_3_CACHE_LINE_FETCHES, 0x08, 26),
PMC_POWERPC_EVENT(THIRD_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x08, 27),
PMC_POWERPC_EVENT(MISPREDICTED_BRANCHES, 0x08, 28),
PMC_POWERPC_EVENT(FOLDED_BRANCHES, 0x08, 29),
PMC_POWERPC_EVENT(FP_STORE_DOUBLE_COMPLETES_IN_LSU, 0x08, 30),
PMC_POWERPC_EVENT(L2_CACHE_HITS, 0x30, 2),
PMC_POWERPC_EVENT(L3_CACHE_HITS, 0x30, 3),
PMC_POWERPC_EVENT(L2_INSTR_CACHE_MISSES, 0x30, 4),
PMC_POWERPC_EVENT(L3_INSTR_CACHE_MISSES, 0x30, 5),
PMC_POWERPC_EVENT(L2_DATA_CACHE_MISSES, 0x30, 6),
PMC_POWERPC_EVENT(L3_DATA_CACHE_MISSES, 0x30, 7),
PMC_POWERPC_EVENT(L2_LOAD_HITS, 0x10, 8),
PMC_POWERPC_EVENT(L2_STORE_HITS, 0x10, 9),
PMC_POWERPC_EVENT(L3_LOAD_HITS, 0x10, 10),
PMC_POWERPC_EVENT(L3_STORE_HITS, 0x10, 11),
PMC_POWERPC_EVENT(L2_TOUCH_HITS, 0x30, 13),
PMC_POWERPC_EVENT(L3_TOUCH_HITS, 0x30, 14),
PMC_POWERPC_EVENT(SNOOP_RETRIES, 0x30, 15),
PMC_POWERPC_EVENT(SNOOP_MODIFIED, 0x10, 16),
PMC_POWERPC_EVENT(SNOOP_VALID, 0x10, 17),
PMC_POWERPC_EVENT(INTERVENTION, 0x30, 18),
PMC_POWERPC_EVENT(L2_CACHE_MISSES, 0x10, 19),
PMC_POWERPC_EVENT(L3_CACHE_MISSES, 0x10, 20),
PMC_POWERPC_EVENT(L2_CACHE_CASTOUTS, 0x20, 8),
PMC_POWERPC_EVENT(L3_CACHE_CASTOUTS, 0x20, 9),
PMC_POWERPC_EVENT(L2SQ_FULL_CYCLES, 0x20, 10),
PMC_POWERPC_EVENT(L3SQ_FULL_CYCLES, 0x20, 11),
PMC_POWERPC_EVENT(RAQ_FULL_CYCLES, 0x20, 16),
PMC_POWERPC_EVENT(WAQ_FULL_CYCLES, 0x20, 17),
PMC_POWERPC_EVENT(L1_EXTERNAL_INTERVENTIONS, 0x20, 19),
PMC_POWERPC_EVENT(L2_EXTERNAL_INTERVENTIONS, 0x20, 20),
PMC_POWERPC_EVENT(L3_EXTERNAL_INTERVENTIONS, 0x20, 21),
PMC_POWERPC_EVENT(EXTERNAL_INTERVENTIONS, 0x20, 22),
PMC_POWERPC_EVENT(EXTERNAL_PUSHES, 0x20, 23),
PMC_POWERPC_EVENT(EXTERNAL_SNOOP_RETRY, 0x20, 24),
PMC_POWERPC_EVENT(DTQ_FULL_CYCLES, 0x20, 25),
PMC_POWERPC_EVENT(BUS_RETRY, 0x20, 26),
PMC_POWERPC_EVENT(L2_VALID_REQUEST, 0x20, 27),
PMC_POWERPC_EVENT(BORDQ_FULL, 0x20, 28),
PMC_POWERPC_EVENT(BUS_TAS_FOR_READS, 0x20, 42),
PMC_POWERPC_EVENT(BUS_TAS_FOR_WRITES, 0x20, 43),
PMC_POWERPC_EVENT(BUS_READS_NOT_RETRIED, 0x20, 44),
PMC_POWERPC_EVENT(BUS_WRITES_NOT_RETRIED, 0x20, 45),
PMC_POWERPC_EVENT(BUS_READS_WRITES_NOT_RETRIED, 0x20, 46),
PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_L1_RETRY, 0x20, 47),
PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_PREVIOUS_ADJACENT, 0x20, 48),
PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_COLLISION, 0x20, 49),
PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_INTERVENTION_ORDERING, 0x20, 50),
PMC_POWERPC_EVENT(SNOOP_REQUESTS, 0x20, 51),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_REQUEST, 0x20, 52),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_LOAD, 0x20, 53),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_STORE, 0x20, 54),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_INSTR_FETCH, 0x20, 55),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_LOAD_STORE_INSTR_FETCH, 0x20, 56),
PMC_POWERPC_EVENT(PREFETCH_ENGINE_FULL, 0x20, 57)
};
const size_t mpc7xxx_event_codes_size =
sizeof(mpc7xxx_event_codes) / sizeof(mpc7xxx_event_codes[0]);
static pmc_value_t
mpc7xxx_pmcn_read(unsigned int pmc)
{
switch (pmc) {
case 0:
return mfspr(SPR_PMC1);
break;
case 1:
return mfspr(SPR_PMC2);
break;
case 2:
return mfspr(SPR_PMC3);
break;
case 3:
return mfspr(SPR_PMC4);
break;
case 4:
return mfspr(SPR_PMC5);
break;
case 5:
return mfspr(SPR_PMC6);
default:
panic("Invalid PMC number: %d\n", pmc);
}
}
static void
mpc7xxx_pmcn_write(unsigned int pmc, uint32_t val)
{
switch (pmc) {
case 0:
mtspr(SPR_PMC1, val);
break;
case 1:
mtspr(SPR_PMC2, val);
break;
case 2:
mtspr(SPR_PMC3, val);
break;
case 3:
mtspr(SPR_PMC4, val);
break;
case 4:
mtspr(SPR_PMC5, val);
break;
case 5:
mtspr(SPR_PMC6, val);
break;
default:
panic("Invalid PMC number: %d\n", pmc);
}
}
static int
mpc7xxx_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = mpc7xxx_pmcn_read(ri);
- PMCDBG(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp);
+ PMCDBG2(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
return 0;
}
static int
mpc7xxx_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
- PMCDBG(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v);
mpc7xxx_pmcn_write(ri, v);
return 0;
}
static int
mpc7xxx_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
static int
mpc7xxx_start_pmc(int cpu, int ri)
{
uint32_t config;
struct pmc *pm;
struct pmc_hw *phw;
register_t pmc_mmcr;
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
pm = phw->phw_pmc;
config = pm->pm_md.pm_powerpc.pm_powerpc_evsel & ~POWERPC_PMC_ENABLE;
/* Enable the PMC. */
switch (ri) {
case 0:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC1SEL(pmc_mmcr, config);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 1:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC2SEL(pmc_mmcr, config);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 2:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC3SEL(pmc_mmcr, config);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
case 3:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC4SEL(pmc_mmcr, config);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 4:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC5SEL(pmc_mmcr, config);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
case 5:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC6SEL(pmc_mmcr, config);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
default:
break;
}
/* The mask is inverted (enable is 1) compared to the flags in MMCR0, which
* are Freeze flags.
*/
config = ~pm->pm_md.pm_powerpc.pm_powerpc_evsel & POWERPC_PMC_ENABLE;
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr &= ~SPR_MMCR0_FC;
pmc_mmcr |= config;
mtspr(SPR_MMCR0, pmc_mmcr);
return 0;
}
static int
mpc7xxx_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct pmc_hw *phw;
register_t pmc_mmcr;
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
pm = phw->phw_pmc;
/*
* Disable the PMCs.
*/
switch (ri) {
case 0:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC1SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 1:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC2SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 2:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC3SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
case 3:
pmc_mmcr = mfspr(SPR_MMCR0);
pmc_mmcr = PPC_SET_PMC4SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR0, pmc_mmcr);
break;
case 4:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC5SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
case 5:
pmc_mmcr = mfspr(SPR_MMCR1);
pmc_mmcr = PPC_SET_PMC6SEL(pmc_mmcr, 0);
mtspr(SPR_MMCR1, pmc_mmcr);
break;
default:
break;
}
return 0;
}
static int
mpc7xxx_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct pmc_cpu *pc;
struct powerpc_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] wrong cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"powerpc-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu);
powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * MPC7XXX_MAX_PMCS,
M_PMC, M_WAITOK|M_ZERO);
pac->pc_class = PMC_CLASS_PPC7450;
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri;
KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_ppcpmcs; i < MPC7XXX_MAX_PMCS; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/* Clear the MMCRs, and set FC, to disable all PMCs. */
mtspr(SPR_MMCR0, SPR_MMCR0_FC | SPR_MMCR0_PMXE |
SPR_MMCR0_FCECE | SPR_MMCR0_PMC1CE | SPR_MMCR0_PMCNCE);
mtspr(SPR_MMCR1, 0);
return 0;
}
static int
mpc7xxx_pcpu_fini(struct pmc_mdep *md, int cpu)
{
uint32_t mmcr0 = mfspr(SPR_MMCR0);
mtmsr(mfmsr() & ~PSL_PMM);
mmcr0 |= SPR_MMCR0_FC;
mtspr(SPR_MMCR0, mmcr0);
free(powerpc_pcpu[cpu]->pc_ppcpmcs, M_PMC);
free(powerpc_pcpu[cpu], M_PMC);
return 0;
}
static int
mpc7xxx_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event pe;
uint32_t caps, config, counter;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
caps = a->pm_caps;
pe = a->pm_ev;
for (i = 0; i < mpc7xxx_event_codes_size; i++) {
if (mpc7xxx_event_codes[i].pe_ev == pe) {
config = mpc7xxx_event_codes[i].pe_code;
counter = mpc7xxx_event_codes[i].pe_counter_mask;
break;
}
}
if (i == mpc7xxx_event_codes_size)
return (EINVAL);
if ((counter & (1 << ri)) == 0)
return (EINVAL);
if (caps & PMC_CAP_SYSTEM)
config |= POWERPC_PMC_KERNEL_ENABLE;
if (caps & PMC_CAP_USER)
config |= POWERPC_PMC_USER_ENABLE;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
config |= POWERPC_PMC_ENABLE;
pm->pm_md.pm_powerpc.pm_powerpc_evsel = config;
- PMCDBG(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
mpc7xxx_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
KASSERT(phw->phw_pmc == NULL,
("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
return 0;
}
static int
mpc7xxx_intr(int cpu, struct trapframe *tf)
{
int i, error, retval;
uint32_t config;
struct pmc *pm;
struct powerpc_cpu *pac;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
- PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC;
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
for (i = 0; i < MPC7XXX_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!MPC7XXX_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the counter if logging fails. */
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error != 0)
mpc7xxx_stop_pmc(cpu, i);
/* reload count. */
mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
int
pmc_mpc7xxx_initialize(struct pmc_mdep *pmc_mdep)
{
struct pmc_classdep *pcd;
pmc_mdep->pmd_cputype = PMC_CPU_PPC_7450;
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC];
pcd->pcd_caps = POWERPC_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_PPC7450;
pcd->pcd_num = MPC7XXX_MAX_PMCS;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 32; /* All PMCs, even in ppc970, are 32-bit */
pcd->pcd_allocate_pmc = mpc7xxx_allocate_pmc;
pcd->pcd_config_pmc = mpc7xxx_config_pmc;
pcd->pcd_pcpu_fini = mpc7xxx_pcpu_fini;
pcd->pcd_pcpu_init = mpc7xxx_pcpu_init;
pcd->pcd_describe = powerpc_describe;
pcd->pcd_get_config = powerpc_get_config;
pcd->pcd_read_pmc = mpc7xxx_read_pmc;
pcd->pcd_release_pmc = mpc7xxx_release_pmc;
pcd->pcd_start_pmc = mpc7xxx_start_pmc;
pcd->pcd_stop_pmc = mpc7xxx_stop_pmc;
pcd->pcd_write_pmc = mpc7xxx_write_pmc;
pmc_mdep->pmd_npmc += MPC7XXX_MAX_PMCS;
pmc_mdep->pmd_intr = mpc7xxx_intr;
return (0);
}
Index: head/sys/dev/hwpmc/hwpmc_octeon.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_octeon.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_octeon.c (revision 282658)
@@ -1,195 +1,196 @@
/*-
* Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/pmc_mdep.h>
#include <contrib/octeon-sdk/cvmx.h>
#include <contrib/octeon-sdk/cvmx-core.h>
#define OCTEON_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \
PMC_CAP_SYSTEM | PMC_CAP_EDGE | \
PMC_CAP_THRESHOLD | PMC_CAP_READ | \
PMC_CAP_WRITE | PMC_CAP_INVERT | \
PMC_CAP_QUALIFIER)
const struct mips_event_code_map mips_event_codes[] =
{
{ PMC_EV_OCTEON_CLK, MIPS_CTR_ALL, CVMX_CORE_PERF_CLK },
{ PMC_EV_OCTEON_ISSUE, MIPS_CTR_ALL, CVMX_CORE_PERF_ISSUE },
{ PMC_EV_OCTEON_RET, MIPS_CTR_ALL, CVMX_CORE_PERF_RET },
{ PMC_EV_OCTEON_NISSUE, MIPS_CTR_ALL, CVMX_CORE_PERF_NISSUE },
{ PMC_EV_OCTEON_SISSUE, MIPS_CTR_ALL, CVMX_CORE_PERF_SISSUE },
{ PMC_EV_OCTEON_DISSUE, MIPS_CTR_ALL, CVMX_CORE_PERF_DISSUE },
{ PMC_EV_OCTEON_IFI, MIPS_CTR_ALL, CVMX_CORE_PERF_IFI },
{ PMC_EV_OCTEON_BR, MIPS_CTR_ALL, CVMX_CORE_PERF_BR },
{ PMC_EV_OCTEON_BRMIS, MIPS_CTR_ALL, CVMX_CORE_PERF_BRMIS },
{ PMC_EV_OCTEON_J, MIPS_CTR_ALL, CVMX_CORE_PERF_J },
{ PMC_EV_OCTEON_JMIS, MIPS_CTR_ALL, CVMX_CORE_PERF_JMIS },
{ PMC_EV_OCTEON_REPLAY, MIPS_CTR_ALL, CVMX_CORE_PERF_REPLAY },
{ PMC_EV_OCTEON_IUNA, MIPS_CTR_ALL, CVMX_CORE_PERF_IUNA },
{ PMC_EV_OCTEON_TRAP, MIPS_CTR_ALL, CVMX_CORE_PERF_TRAP },
{ PMC_EV_OCTEON_UULOAD, MIPS_CTR_ALL, CVMX_CORE_PERF_UULOAD },
{ PMC_EV_OCTEON_UUSTORE, MIPS_CTR_ALL, CVMX_CORE_PERF_UUSTORE },
{ PMC_EV_OCTEON_ULOAD, MIPS_CTR_ALL, CVMX_CORE_PERF_ULOAD },
{ PMC_EV_OCTEON_USTORE, MIPS_CTR_ALL, CVMX_CORE_PERF_USTORE },
{ PMC_EV_OCTEON_EC, MIPS_CTR_ALL, CVMX_CORE_PERF_EC },
{ PMC_EV_OCTEON_MC, MIPS_CTR_ALL, CVMX_CORE_PERF_MC },
{ PMC_EV_OCTEON_CC, MIPS_CTR_ALL, CVMX_CORE_PERF_CC },
{ PMC_EV_OCTEON_CSRC, MIPS_CTR_ALL, CVMX_CORE_PERF_CSRC },
{ PMC_EV_OCTEON_CFETCH, MIPS_CTR_ALL, CVMX_CORE_PERF_CFETCH },
{ PMC_EV_OCTEON_CPREF, MIPS_CTR_ALL, CVMX_CORE_PERF_CPREF },
{ PMC_EV_OCTEON_ICA, MIPS_CTR_ALL, CVMX_CORE_PERF_ICA },
{ PMC_EV_OCTEON_II, MIPS_CTR_ALL, CVMX_CORE_PERF_II },
{ PMC_EV_OCTEON_IP, MIPS_CTR_ALL, CVMX_CORE_PERF_IP },
{ PMC_EV_OCTEON_CIMISS, MIPS_CTR_ALL, CVMX_CORE_PERF_CIMISS },
{ PMC_EV_OCTEON_WBUF, MIPS_CTR_ALL, CVMX_CORE_PERF_WBUF },
{ PMC_EV_OCTEON_WDAT, MIPS_CTR_ALL, CVMX_CORE_PERF_WDAT },
{ PMC_EV_OCTEON_WBUFLD, MIPS_CTR_ALL, CVMX_CORE_PERF_WBUFLD },
{ PMC_EV_OCTEON_WBUFFL, MIPS_CTR_ALL, CVMX_CORE_PERF_WBUFFL },
{ PMC_EV_OCTEON_WBUFTR, MIPS_CTR_ALL, CVMX_CORE_PERF_WBUFTR },
{ PMC_EV_OCTEON_BADD, MIPS_CTR_ALL, CVMX_CORE_PERF_BADD },
{ PMC_EV_OCTEON_BADDL2, MIPS_CTR_ALL, CVMX_CORE_PERF_BADDL2 },
{ PMC_EV_OCTEON_BFILL, MIPS_CTR_ALL, CVMX_CORE_PERF_BFILL },
{ PMC_EV_OCTEON_DDIDS, MIPS_CTR_ALL, CVMX_CORE_PERF_DDIDS },
{ PMC_EV_OCTEON_IDIDS, MIPS_CTR_ALL, CVMX_CORE_PERF_IDIDS },
{ PMC_EV_OCTEON_DIDNA, MIPS_CTR_ALL, CVMX_CORE_PERF_DIDNA },
{ PMC_EV_OCTEON_LDS, MIPS_CTR_ALL, CVMX_CORE_PERF_LDS },
{ PMC_EV_OCTEON_LMLDS, MIPS_CTR_ALL, CVMX_CORE_PERF_LMLDS },
{ PMC_EV_OCTEON_IOLDS, MIPS_CTR_ALL, CVMX_CORE_PERF_IOLDS },
{ PMC_EV_OCTEON_DMLDS, MIPS_CTR_ALL, CVMX_CORE_PERF_DMLDS },
{ PMC_EV_OCTEON_STS, MIPS_CTR_ALL, CVMX_CORE_PERF_STS },
{ PMC_EV_OCTEON_LMSTS, MIPS_CTR_ALL, CVMX_CORE_PERF_LMSTS },
{ PMC_EV_OCTEON_IOSTS, MIPS_CTR_ALL, CVMX_CORE_PERF_IOSTS },
{ PMC_EV_OCTEON_IOBDMA, MIPS_CTR_ALL, CVMX_CORE_PERF_IOBDMA },
{ PMC_EV_OCTEON_DTLB, MIPS_CTR_ALL, CVMX_CORE_PERF_DTLB },
{ PMC_EV_OCTEON_DTLBAD, MIPS_CTR_ALL, CVMX_CORE_PERF_DTLBAD },
{ PMC_EV_OCTEON_ITLB, MIPS_CTR_ALL, CVMX_CORE_PERF_ITLB },
{ PMC_EV_OCTEON_SYNC, MIPS_CTR_ALL, CVMX_CORE_PERF_SYNC },
{ PMC_EV_OCTEON_SYNCIOB, MIPS_CTR_ALL, CVMX_CORE_PERF_SYNCIOB },
{ PMC_EV_OCTEON_SYNCW, MIPS_CTR_ALL, CVMX_CORE_PERF_SYNCW },
};
const int mips_event_codes_size =
sizeof(mips_event_codes) / sizeof(mips_event_codes[0]);
struct mips_pmc_spec mips_pmc_spec = {
.ps_cpuclass = PMC_CLASS_OCTEON,
.ps_cputype = PMC_CPU_MIPS_OCTEON,
.ps_capabilities = OCTEON_PMC_CAPS,
.ps_counter_width = 64
};
/*
* Performance Count Register N
*/
uint64_t
mips_pmcn_read(unsigned int pmc)
{
uint64_t reg = 0;
KASSERT(pmc < mips_npmcs, ("[mips,%d] illegal PMC number %d",
__LINE__, pmc));
/* The counter value is the next value after the control register. */
switch (pmc) {
case 0:
CVMX_MF_COP0(reg, COP0_PERFVALUE0);
break;
case 1:
CVMX_MF_COP0(reg, COP0_PERFVALUE1);
break;
default:
return 0;
}
return (reg);
}
uint64_t
mips_pmcn_write(unsigned int pmc, uint64_t reg)
{
KASSERT(pmc < mips_npmcs, ("[mips,%d] illegal PMC number %d",
__LINE__, pmc));
switch (pmc) {
case 0:
CVMX_MT_COP0(reg, COP0_PERFVALUE0);
break;
case 1:
CVMX_MT_COP0(reg, COP0_PERFVALUE1);
break;
default:
return 0;
}
return (reg);
}
uint32_t
mips_get_perfctl(int cpu, int ri, uint32_t event, uint32_t caps)
{
cvmx_core_perf_control_t control;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < mips_npmcs,
("[mips,%d] illegal row index %d", __LINE__, ri));
control.s.event = event;
if (caps & PMC_CAP_SYSTEM) {
control.s.k = 1;
control.s.s = 1;
control.s.ex = 1;
}
if (caps & PMC_CAP_USER)
control.s.u = 1;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) {
control.s.k = 1;
control.s.s = 1;
control.s.u = 1;
control.s.ex = 1;
}
if (caps & PMC_CAP_INTERRUPT)
control.s.ie = 1;
- PMCDBG(MDP,ALL,2,"mips-allocate ri=%d -> config=0x%x", ri, control.u32);
+ PMCDBG2(MDP,ALL,2,"mips-allocate ri=%d -> config=0x%x", ri,
+ control.u32);
return (control.u32);
}
Index: head/sys/dev/hwpmc/hwpmc_piv.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_piv.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_piv.c (revision 282658)
@@ -1,1700 +1,1701 @@
/*-
* Copyright (c) 2003-2007 Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <machine/intr_machdep.h>
#if (__FreeBSD_version >= 1100000)
#include <x86/apicvar.h>
#else
#include <machine/apicvar.h>
#endif
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/cputypes.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
/*
* PENTIUM 4 SUPPORT
*
* The P4 has 18 PMCs, divided into 4 groups with 4,4,4 and 6 PMCs
* respectively. Each PMC comprises of two model specific registers:
* a counter configuration control register (CCCR) and a counter
* register that holds the actual event counts.
*
* Configuring an event requires the use of one of 45 event selection
* control registers (ESCR). Events are associated with specific
* ESCRs. Each PMC group has a set of ESCRs it can use.
*
* - The BPU counter group (4 PMCs) can use the 16 ESCRs:
* BPU_ESCR{0,1}, IS_ESCR{0,1}, MOB_ESCR{0,1}, ITLB_ESCR{0,1},
* PMH_ESCR{0,1}, IX_ESCR{0,1}, FSB_ESCR{0,}, BSU_ESCR{0,1}.
*
* - The MS counter group (4 PMCs) can use the 6 ESCRs: MS_ESCR{0,1},
* TC_ESCR{0,1}, TBPU_ESCR{0,1}.
*
* - The FLAME counter group (4 PMCs) can use the 10 ESCRs:
* FLAME_ESCR{0,1}, FIRM_ESCR{0,1}, SAAT_ESCR{0,1}, U2L_ESCR{0,1},
* DAC_ESCR{0,1}.
*
* - The IQ counter group (6 PMCs) can use the 13 ESCRs: IQ_ESCR{0,1},
* ALF_ESCR{0,1}, RAT_ESCR{0,1}, SSU_ESCR0, CRU_ESCR{0,1,2,3,4,5}.
*
* Even-numbered ESCRs can be used with counters 0, 1 and 4 (if
* present) of a counter group. Odd-numbers ESCRs can be used with
* counters 2, 3 and 5 (if present) of a counter group. The
* 'p4_escrs[]' table describes these restrictions in a form that
* function 'p4_allocate()' uses for making allocation decisions.
*
* SYSTEM-MODE AND THREAD-MODE ALLOCATION
*
* In addition to remembering the state of PMC rows
* ('FREE','STANDALONE', or 'THREAD'), we similar need to track the
* state of ESCR rows. If an ESCR is allocated to a system-mode PMC
* on a CPU we cannot allocate this to a thread-mode PMC. On a
* multi-cpu (multiple physical CPUs) system, ESCR allocation on each
* CPU is tracked by the pc_escrs[] array.
*
* Each system-mode PMC that is using an ESCR records its row-index in
* the appropriate entry and system-mode allocation attempts check
* that an ESCR is available using this array. Process-mode PMCs do
* not use the pc_escrs[] array, since ESCR row itself would have been
* marked as in 'THREAD' mode.
*
* HYPERTHREADING SUPPORT
*
* When HTT is enabled, the FreeBSD kernel treats the two 'logical'
* cpus as independent CPUs and can schedule kernel threads on them
* independently. However, the two logical CPUs share the same set of
* PMC resources. We need to ensure that:
* - PMCs that use the PMC_F_DESCENDANTS semantics are handled correctly,
* and,
* - Threads of multi-threaded processes that get scheduled on the same
* physical CPU are handled correctly.
*
* HTT Detection
*
* Not all HTT capable systems will have HTT enabled. We detect the
* presence of HTT by detecting if 'p4_init()' was called for a secondary
* CPU in a HTT pair.
*
* Note that hwpmc(4) cannot currently deal with a change in HTT status once
* loaded.
*
* Handling HTT READ / WRITE / START / STOP
*
* PMC resources are shared across the CPUs in an HTT pair. We
* designate the lower numbered CPU in a HTT pair as the 'primary'
* CPU. In each primary CPU's state we keep track of a 'runcount'
* which reflects the number of PMC-using processes that have been
* scheduled on its secondary CPU. Process-mode PMC operations will
* actually 'start' or 'stop' hardware only if these are the first or
* last processes respectively to use the hardware. PMC values
* written by a 'write' operation are saved and are transferred to
* hardware at PMC 'start' time if the runcount is 0. If the runcount
* is greater than 0 at the time of a 'start' operation, we keep track
* of the actual hardware value at the time of the 'start' operation
* and use this to adjust the final readings at PMC 'stop' or 'read'
* time.
*
* Execution sequences:
*
* Case 1: CPUx +...- (no overlap)
* CPUy +...-
* RC 0 1 0 1 0
*
* Case 2: CPUx +........- (partial overlap)
* CPUy +........-
* RC 0 1 2 1 0
*
* Case 3: CPUx +..............- (fully overlapped)
* CPUy +.....-
* RC 0 1 2 1 0
*
* Key:
* 'CPU[xy]' : one of the two logical processors on a HTT CPU.
* 'RC' : run count (#threads per physical core).
* '+' : point in time when a thread is put on a CPU.
* '-' : point in time where a thread is taken off a CPU.
*
* Handling HTT CONFIG
*
* Different processes attached to the same PMC may get scheduled on
* the two logical processors in the package. We keep track of config
* and de-config operations using the CFGFLAGS fields of the per-physical
* cpu state.
*/
#define P4_PMCS() \
P4_PMC(BPU_COUNTER0) \
P4_PMC(BPU_COUNTER1) \
P4_PMC(BPU_COUNTER2) \
P4_PMC(BPU_COUNTER3) \
P4_PMC(MS_COUNTER0) \
P4_PMC(MS_COUNTER1) \
P4_PMC(MS_COUNTER2) \
P4_PMC(MS_COUNTER3) \
P4_PMC(FLAME_COUNTER0) \
P4_PMC(FLAME_COUNTER1) \
P4_PMC(FLAME_COUNTER2) \
P4_PMC(FLAME_COUNTER3) \
P4_PMC(IQ_COUNTER0) \
P4_PMC(IQ_COUNTER1) \
P4_PMC(IQ_COUNTER2) \
P4_PMC(IQ_COUNTER3) \
P4_PMC(IQ_COUNTER4) \
P4_PMC(IQ_COUNTER5) \
P4_PMC(NONE)
enum pmc_p4pmc {
#undef P4_PMC
#define P4_PMC(N) P4_PMC_##N ,
P4_PMCS()
};
/*
* P4 ESCR descriptors
*/
#define P4_ESCRS() \
P4_ESCR(BSU_ESCR0, 0x3A0, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(BSU_ESCR1, 0x3A1, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(FSB_ESCR0, 0x3A2, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(FSB_ESCR1, 0x3A3, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(FIRM_ESCR0, 0x3A4, FLAME_COUNTER0, FLAME_COUNTER1, NONE) \
P4_ESCR(FIRM_ESCR1, 0x3A5, FLAME_COUNTER2, FLAME_COUNTER3, NONE) \
P4_ESCR(FLAME_ESCR0, 0x3A6, FLAME_COUNTER0, FLAME_COUNTER1, NONE) \
P4_ESCR(FLAME_ESCR1, 0x3A7, FLAME_COUNTER2, FLAME_COUNTER3, NONE) \
P4_ESCR(DAC_ESCR0, 0x3A8, FLAME_COUNTER0, FLAME_COUNTER1, NONE) \
P4_ESCR(DAC_ESCR1, 0x3A9, FLAME_COUNTER2, FLAME_COUNTER3, NONE) \
P4_ESCR(MOB_ESCR0, 0x3AA, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(MOB_ESCR1, 0x3AB, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(PMH_ESCR0, 0x3AC, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(PMH_ESCR1, 0x3AD, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(SAAT_ESCR0, 0x3AE, FLAME_COUNTER0, FLAME_COUNTER1, NONE) \
P4_ESCR(SAAT_ESCR1, 0x3AF, FLAME_COUNTER2, FLAME_COUNTER3, NONE) \
P4_ESCR(U2L_ESCR0, 0x3B0, FLAME_COUNTER0, FLAME_COUNTER1, NONE) \
P4_ESCR(U2L_ESCR1, 0x3B1, FLAME_COUNTER2, FLAME_COUNTER3, NONE) \
P4_ESCR(BPU_ESCR0, 0x3B2, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(BPU_ESCR1, 0x3B3, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(IS_ESCR0, 0x3B4, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(IS_ESCR1, 0x3B5, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(ITLB_ESCR0, 0x3B6, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(ITLB_ESCR1, 0x3B7, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(CRU_ESCR0, 0x3B8, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(CRU_ESCR1, 0x3B9, IQ_COUNTER2, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(IQ_ESCR0, 0x3BA, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(IQ_ESCR1, 0x3BB, IQ_COUNTER1, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(RAT_ESCR0, 0x3BC, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(RAT_ESCR1, 0x3BD, IQ_COUNTER2, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(SSU_ESCR0, 0x3BE, IQ_COUNTER0, IQ_COUNTER2, IQ_COUNTER4) \
P4_ESCR(MS_ESCR0, 0x3C0, MS_COUNTER0, MS_COUNTER1, NONE) \
P4_ESCR(MS_ESCR1, 0x3C1, MS_COUNTER2, MS_COUNTER3, NONE) \
P4_ESCR(TBPU_ESCR0, 0x3C2, MS_COUNTER0, MS_COUNTER1, NONE) \
P4_ESCR(TBPU_ESCR1, 0x3C3, MS_COUNTER2, MS_COUNTER3, NONE) \
P4_ESCR(TC_ESCR0, 0x3C4, MS_COUNTER0, MS_COUNTER1, NONE) \
P4_ESCR(TC_ESCR1, 0x3C5, MS_COUNTER2, MS_COUNTER3, NONE) \
P4_ESCR(IX_ESCR0, 0x3C8, BPU_COUNTER0, BPU_COUNTER1, NONE) \
P4_ESCR(IX_ESCR1, 0x3C9, BPU_COUNTER2, BPU_COUNTER3, NONE) \
P4_ESCR(ALF_ESCR0, 0x3CA, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(ALF_ESCR1, 0x3CB, IQ_COUNTER2, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(CRU_ESCR2, 0x3CC, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(CRU_ESCR3, 0x3CD, IQ_COUNTER2, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(CRU_ESCR4, 0x3E0, IQ_COUNTER0, IQ_COUNTER1, IQ_COUNTER4) \
P4_ESCR(CRU_ESCR5, 0x3E1, IQ_COUNTER2, IQ_COUNTER3, IQ_COUNTER5) \
P4_ESCR(NONE, ~0, NONE, NONE, NONE)
enum pmc_p4escr {
#define P4_ESCR(N, MSR, P1, P2, P3) P4_ESCR_##N ,
P4_ESCRS()
#undef P4_ESCR
};
struct pmc_p4escr_descr {
const char pm_escrname[PMC_NAME_MAX];
u_short pm_escr_msr;
const enum pmc_p4pmc pm_pmcs[P4_MAX_PMC_PER_ESCR];
};
static struct pmc_p4escr_descr p4_escrs[] =
{
#define P4_ESCR(N, MSR, P1, P2, P3) \
{ \
.pm_escrname = #N, \
.pm_escr_msr = (MSR), \
.pm_pmcs = \
{ \
P4_PMC_##P1, \
P4_PMC_##P2, \
P4_PMC_##P3 \
} \
} ,
P4_ESCRS()
#undef P4_ESCR
};
/*
* P4 Event descriptor
*/
struct p4_event_descr {
const enum pmc_event pm_event;
const uint32_t pm_escr_eventselect;
const uint32_t pm_cccr_select;
const char pm_is_ti_event;
enum pmc_p4escr pm_escrs[P4_MAX_ESCR_PER_EVENT];
};
static struct p4_event_descr p4_events[] = {
#define P4_EVDESCR(NAME, ESCREVENTSEL, CCCRSEL, TI_EVENT, ESCR0, ESCR1) \
{ \
.pm_event = PMC_EV_P4_##NAME, \
.pm_escr_eventselect = (ESCREVENTSEL), \
.pm_cccr_select = (CCCRSEL), \
.pm_is_ti_event = (TI_EVENT), \
.pm_escrs = \
{ \
P4_ESCR_##ESCR0, \
P4_ESCR_##ESCR1 \
} \
}
P4_EVDESCR(TC_DELIVER_MODE, 0x01, 0x01, TRUE, TC_ESCR0, TC_ESCR1),
P4_EVDESCR(BPU_FETCH_REQUEST, 0x03, 0x00, FALSE, BPU_ESCR0, BPU_ESCR1),
P4_EVDESCR(ITLB_REFERENCE, 0x18, 0x03, FALSE, ITLB_ESCR0, ITLB_ESCR1),
P4_EVDESCR(MEMORY_CANCEL, 0x02, 0x05, FALSE, DAC_ESCR0, DAC_ESCR1),
P4_EVDESCR(MEMORY_COMPLETE, 0x08, 0x02, FALSE, SAAT_ESCR0, SAAT_ESCR1),
P4_EVDESCR(LOAD_PORT_REPLAY, 0x04, 0x02, FALSE, SAAT_ESCR0, SAAT_ESCR1),
P4_EVDESCR(STORE_PORT_REPLAY, 0x05, 0x02, FALSE, SAAT_ESCR0, SAAT_ESCR1),
P4_EVDESCR(MOB_LOAD_REPLAY, 0x03, 0x02, FALSE, MOB_ESCR0, MOB_ESCR1),
P4_EVDESCR(PAGE_WALK_TYPE, 0x01, 0x04, TRUE, PMH_ESCR0, PMH_ESCR1),
P4_EVDESCR(BSQ_CACHE_REFERENCE, 0x0C, 0x07, FALSE, BSU_ESCR0, BSU_ESCR1),
P4_EVDESCR(IOQ_ALLOCATION, 0x03, 0x06, FALSE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(IOQ_ACTIVE_ENTRIES, 0x1A, 0x06, FALSE, FSB_ESCR1, NONE),
P4_EVDESCR(FSB_DATA_ACTIVITY, 0x17, 0x06, TRUE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(BSQ_ALLOCATION, 0x05, 0x07, FALSE, BSU_ESCR0, NONE),
P4_EVDESCR(BSQ_ACTIVE_ENTRIES, 0x06, 0x07, FALSE, BSU_ESCR1, NONE),
/* BSQ_ACTIVE_ENTRIES inherits CPU specificity from BSQ_ALLOCATION */
P4_EVDESCR(SSE_INPUT_ASSIST, 0x34, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(PACKED_SP_UOP, 0x08, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(PACKED_DP_UOP, 0x0C, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(SCALAR_SP_UOP, 0x0A, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(SCALAR_DP_UOP, 0x0E, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(64BIT_MMX_UOP, 0x02, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(128BIT_MMX_UOP, 0x1A, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(X87_FP_UOP, 0x04, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(X87_SIMD_MOVES_UOP, 0x2E, 0x01, TRUE, FIRM_ESCR0, FIRM_ESCR1),
P4_EVDESCR(GLOBAL_POWER_EVENTS, 0x13, 0x06, FALSE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(TC_MS_XFER, 0x05, 0x00, FALSE, MS_ESCR0, MS_ESCR1),
P4_EVDESCR(UOP_QUEUE_WRITES, 0x09, 0x00, FALSE, MS_ESCR0, MS_ESCR1),
P4_EVDESCR(RETIRED_MISPRED_BRANCH_TYPE,
0x05, 0x02, FALSE, TBPU_ESCR0, TBPU_ESCR1),
P4_EVDESCR(RETIRED_BRANCH_TYPE, 0x04, 0x02, FALSE, TBPU_ESCR0, TBPU_ESCR1),
P4_EVDESCR(RESOURCE_STALL, 0x01, 0x01, FALSE, ALF_ESCR0, ALF_ESCR1),
P4_EVDESCR(WC_BUFFER, 0x05, 0x05, TRUE, DAC_ESCR0, DAC_ESCR1),
P4_EVDESCR(B2B_CYCLES, 0x16, 0x03, TRUE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(BNR, 0x08, 0x03, TRUE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(SNOOP, 0x06, 0x03, TRUE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(RESPONSE, 0x04, 0x03, TRUE, FSB_ESCR0, FSB_ESCR1),
P4_EVDESCR(FRONT_END_EVENT, 0x08, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3),
P4_EVDESCR(EXECUTION_EVENT, 0x0C, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3),
P4_EVDESCR(REPLAY_EVENT, 0x09, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3),
P4_EVDESCR(INSTR_RETIRED, 0x02, 0x04, FALSE, CRU_ESCR0, CRU_ESCR1),
P4_EVDESCR(UOPS_RETIRED, 0x01, 0x04, FALSE, CRU_ESCR0, CRU_ESCR1),
P4_EVDESCR(UOP_TYPE, 0x02, 0x02, FALSE, RAT_ESCR0, RAT_ESCR1),
P4_EVDESCR(BRANCH_RETIRED, 0x06, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3),
P4_EVDESCR(MISPRED_BRANCH_RETIRED, 0x03, 0x04, FALSE, CRU_ESCR0, CRU_ESCR1),
P4_EVDESCR(X87_ASSIST, 0x03, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3),
P4_EVDESCR(MACHINE_CLEAR, 0x02, 0x05, FALSE, CRU_ESCR2, CRU_ESCR3)
#undef P4_EVDESCR
};
#define P4_EVENT_IS_TI(E) ((E)->pm_is_ti_event == TRUE)
#define P4_NEVENTS (PMC_EV_P4_LAST - PMC_EV_P4_FIRST + 1)
/*
* P4 PMC descriptors
*/
struct p4pmc_descr {
struct pmc_descr pm_descr; /* common information */
enum pmc_p4pmc pm_pmcnum; /* PMC number */
uint32_t pm_pmc_msr; /* PERFCTR MSR address */
uint32_t pm_cccr_msr; /* CCCR MSR address */
};
static struct p4pmc_descr p4_pmcdesc[P4_NPMCS] = {
#define P4_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE | \
PMC_CAP_TAGGING | PMC_CAP_CASCADE)
#define P4_PMCDESCR(N, PMC, CCCR) \
{ \
.pm_descr = \
{ \
.pd_name = #N, \
.pd_class = PMC_CLASS_P4, \
.pd_caps = P4_PMC_CAPS, \
.pd_width = 40 \
}, \
.pm_pmcnum = P4_PMC_##N, \
.pm_cccr_msr = (CCCR), \
.pm_pmc_msr = (PMC) \
}
P4_PMCDESCR(BPU_COUNTER0, 0x300, 0x360),
P4_PMCDESCR(BPU_COUNTER1, 0x301, 0x361),
P4_PMCDESCR(BPU_COUNTER2, 0x302, 0x362),
P4_PMCDESCR(BPU_COUNTER3, 0x303, 0x363),
P4_PMCDESCR(MS_COUNTER0, 0x304, 0x364),
P4_PMCDESCR(MS_COUNTER1, 0x305, 0x365),
P4_PMCDESCR(MS_COUNTER2, 0x306, 0x366),
P4_PMCDESCR(MS_COUNTER3, 0x307, 0x367),
P4_PMCDESCR(FLAME_COUNTER0, 0x308, 0x368),
P4_PMCDESCR(FLAME_COUNTER1, 0x309, 0x369),
P4_PMCDESCR(FLAME_COUNTER2, 0x30A, 0x36A),
P4_PMCDESCR(FLAME_COUNTER3, 0x30B, 0x36B),
P4_PMCDESCR(IQ_COUNTER0, 0x30C, 0x36C),
P4_PMCDESCR(IQ_COUNTER1, 0x30D, 0x36D),
P4_PMCDESCR(IQ_COUNTER2, 0x30E, 0x36E),
P4_PMCDESCR(IQ_COUNTER3, 0x30F, 0x36F),
P4_PMCDESCR(IQ_COUNTER4, 0x310, 0x370),
P4_PMCDESCR(IQ_COUNTER5, 0x311, 0x371),
#undef P4_PMCDESCR
};
/* HTT support */
#define P4_NHTT 2 /* logical processors/chip */
static int p4_system_has_htt;
/*
* Per-CPU data structure for P4 class CPUs
*
* [19 struct pmc_hw structures]
* [45 ESCRs status bytes]
* [per-cpu spin mutex]
* [19 flag fields for holding config flags and a runcount]
* [19*2 hw value fields] (Thread mode PMC support)
* or
* [19*2 EIP values] (Sampling mode PMCs)
* [19*2 pmc value fields] (Thread mode PMC support))
*/
struct p4_cpu {
struct pmc_hw pc_p4pmcs[P4_NPMCS];
char pc_escrs[P4_NESCR];
struct mtx pc_mtx; /* spin lock */
uint32_t pc_intrflag; /* NMI handler flags */
unsigned int pc_intrlock; /* NMI handler spin lock */
unsigned char pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */
union {
pmc_value_t pc_hw[P4_NPMCS * P4_NHTT];
uintptr_t pc_ip[P4_NPMCS * P4_NHTT];
} pc_si;
pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT];
};
static struct p4_cpu **p4_pcpu;
#define P4_PCPU_PMC_VALUE(PC,RI,CPU) (PC)->pc_pmc_values[(RI)*((CPU) & 1)]
#define P4_PCPU_HW_VALUE(PC,RI,CPU) (PC)->pc_si.pc_hw[(RI)*((CPU) & 1)]
#define P4_PCPU_SAVED_IP(PC,RI,CPU) (PC)->pc_si.pc_ip[(RI)*((CPU) & 1)]
#define P4_PCPU_GET_FLAGS(PC,RI,MASK) ((PC)->pc_flags[(RI)] & (MASK))
#define P4_PCPU_SET_FLAGS(PC,RI,MASK,VAL) do { \
char _tmp; \
_tmp = (PC)->pc_flags[(RI)]; \
_tmp &= ~(MASK); \
_tmp |= (VAL) & (MASK); \
(PC)->pc_flags[(RI)] = _tmp; \
} while (0)
#define P4_PCPU_GET_RUNCOUNT(PC,RI) P4_PCPU_GET_FLAGS(PC,RI,0x0F)
#define P4_PCPU_SET_RUNCOUNT(PC,RI,V) P4_PCPU_SET_FLAGS(PC,RI,0x0F,V)
#define P4_PCPU_GET_CFGFLAGS(PC,RI) (P4_PCPU_GET_FLAGS(PC,RI,0xF0) >> 4)
#define P4_PCPU_SET_CFGFLAGS(PC,RI,C) P4_PCPU_SET_FLAGS(PC,RI,0xF0,((C) <<4))
#define P4_CPU_TO_FLAG(C) (P4_CPU_IS_HTT_SECONDARY(cpu) ? 0x2 : 0x1)
#define P4_PCPU_GET_INTRFLAG(PC,I) ((PC)->pc_intrflag & (1 << (I)))
#define P4_PCPU_SET_INTRFLAG(PC,I,V) do { \
uint32_t __mask; \
__mask = 1 << (I); \
if ((V)) \
(PC)->pc_intrflag |= __mask; \
else \
(PC)->pc_intrflag &= ~__mask; \
} while (0)
/*
* A minimal spin lock implementation for use inside the NMI handler.
*
* We don't want to use a regular spin lock here, because curthread
* may not be consistent at the time the handler is invoked.
*/
#define P4_PCPU_ACQ_INTR_SPINLOCK(PC) do { \
while (!atomic_cmpset_acq_int(&pc->pc_intrlock, 0, 1)) \
ia32_pause(); \
} while (0)
#define P4_PCPU_REL_INTR_SPINLOCK(PC) \
atomic_store_rel_int(&pc->pc_intrlock, 0);
/* ESCR row disposition */
static int p4_escrdisp[P4_NESCR];
#define P4_ESCR_ROW_DISP_IS_THREAD(E) (p4_escrdisp[(E)] > 0)
#define P4_ESCR_ROW_DISP_IS_STANDALONE(E) (p4_escrdisp[(E)] < 0)
#define P4_ESCR_ROW_DISP_IS_FREE(E) (p4_escrdisp[(E)] == 0)
#define P4_ESCR_MARK_ROW_STANDALONE(E) do { \
KASSERT(p4_escrdisp[(E)] <= 0, ("[p4,%d] row disposition error",\
__LINE__)); \
atomic_add_int(&p4_escrdisp[(E)], -1); \
KASSERT(p4_escrdisp[(E)] >= (-pmc_cpu_max_active()), \
("[p4,%d] row disposition error", __LINE__)); \
} while (0)
#define P4_ESCR_UNMARK_ROW_STANDALONE(E) do { \
atomic_add_int(&p4_escrdisp[(E)], 1); \
KASSERT(p4_escrdisp[(E)] <= 0, ("[p4,%d] row disposition error",\
__LINE__)); \
} while (0)
#define P4_ESCR_MARK_ROW_THREAD(E) do { \
KASSERT(p4_escrdisp[(E)] >= 0, ("[p4,%d] row disposition error", \
__LINE__)); \
atomic_add_int(&p4_escrdisp[(E)], 1); \
} while (0)
#define P4_ESCR_UNMARK_ROW_THREAD(E) do { \
atomic_add_int(&p4_escrdisp[(E)], -1); \
KASSERT(p4_escrdisp[(E)] >= 0, ("[p4,%d] row disposition error", \
__LINE__)); \
} while (0)
#define P4_PMC_IS_STOPPED(cccr) ((rdmsr(cccr) & P4_CCCR_ENABLE) == 0)
#define P4_CPU_IS_HTT_SECONDARY(cpu) \
(p4_system_has_htt ? ((cpu) & 1) : 0)
#define P4_TO_HTT_PRIMARY(cpu) \
(p4_system_has_htt ? ((cpu) & ~1) : (cpu))
#define P4_CCCR_Tx_MASK (~(P4_CCCR_OVF_PMI_T0|P4_CCCR_OVF_PMI_T1| \
P4_CCCR_ENABLE|P4_CCCR_OVF))
#define P4_ESCR_Tx_MASK (~(P4_ESCR_T0_OS|P4_ESCR_T0_USR|P4_ESCR_T1_OS| \
P4_ESCR_T1_USR))
/*
* support routines
*/
static struct p4_event_descr *
p4_find_event(enum pmc_event ev)
{
int n;
for (n = 0; n < P4_NEVENTS; n++)
if (p4_events[n].pm_event == ev)
break;
if (n == P4_NEVENTS)
return (NULL);
return (&p4_events[n]);
}
/*
* Initialize per-cpu state
*/
static int
p4_pcpu_init(struct pmc_mdep *md, int cpu)
{
char *pescr;
int n, first_ri, phycpu;
struct pmc_hw *phw;
struct p4_cpu *p4c;
struct pmc_cpu *pc, *plc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] insane cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,
+ PMCDBG2(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,
pmc_cpu_is_primary(cpu) != 0);
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
/*
* The two CPUs in an HT pair share their per-cpu state.
*
* For HT capable CPUs, we assume that the two logical
* processors in the HT pair get two consecutive CPU ids
* starting with an even id #.
*
* The primary CPU (the even numbered CPU of the pair) would
* have been initialized prior to the initialization for the
* secondary.
*/
if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) {
p4_system_has_htt = 1;
phycpu = P4_TO_HTT_PRIMARY(cpu);
pc = pmc_pcpu[phycpu];
plc = pmc_pcpu[cpu];
KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
- PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
+ PMCDBG3(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
phycpu, pc);
KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
__LINE__, cpu, phycpu));
/* PMCs are shared with the physical CPU. */
for (n = 0; n < P4_NPMCS; n++)
plc->pc_hwpmcs[n + first_ri] =
pc->pc_hwpmcs[n + first_ri];
return (0);
}
p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
if (p4c == NULL)
return (ENOMEM);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
p4_pcpu[cpu] = p4c;
phw = p4c->pc_p4pmcs;
for (n = 0; n < P4_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
pescr = p4c->pc_escrs;
for (n = 0; n < P4_NESCR; n++)
*pescr++ = P4_INVALID_PMC_INDEX;
mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
return (0);
}
/*
* Destroy per-cpu state.
*/
static int
p4_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct p4_cpu *p4c;
struct pmc_cpu *pc;
- PMCDBG(MDP,INI,0, "p4-cleanup cpu=%d", cpu);
+ PMCDBG1(MDP,INI,0, "p4-cleanup cpu=%d", cpu);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
for (i = 0; i < P4_NPMCS; i++)
pc->pc_hwpmcs[i + first_ri] = NULL;
if (!pmc_cpu_is_primary(cpu) && (cpu & 1))
return (0);
p4c = p4_pcpu[cpu];
KASSERT(p4c != NULL, ("[p4,%d] NULL pcpu", __LINE__));
/* Turn off all PMCs on this CPU */
for (i = 0; i < P4_NPMCS - 1; i++)
wrmsr(P4_CCCR_MSR_FIRST + i,
rdmsr(P4_CCCR_MSR_FIRST + i) & ~P4_CCCR_ENABLE);
mtx_destroy(&p4c->pc_mtx);
free(p4c, M_PMC);
p4_pcpu[cpu] = NULL;
return (0);
}
/*
* Read a PMC
*/
static int
p4_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
struct p4_cpu *pc;
enum pmc_mode mode;
struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));
KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm),
("[p4,%d] class mismatch pd %d != id class %d", __LINE__,
pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
mode = PMC_TO_MODE(pm);
- PMCDBG(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
+ PMCDBG3(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
if (PMC_IS_VIRTUAL_MODE(mode)) {
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */
tmp += (P4_PERFCTR_MASK + 1) -
P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);
}
if (PMC_IS_SAMPLING_MODE(mode)) /* undo transformation */
*v = P4_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
- PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
+ PMCDBG1(MDP,REA,2, "p4-read -> %jx", *v);
return (0);
}
/*
* Write a PMC
*/
static int
p4_write_pmc(int cpu, int ri, pmc_value_t v)
{
enum pmc_mode mode;
struct pmc *pm;
struct p4_cpu *pc;
const struct pmc_hw *phw;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
phw = &pc->pc_p4pmcs[ri];
pm = phw->phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri));
mode = PMC_TO_MODE(pm);
- PMCDBG(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
+ PMCDBG4(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
mode, v);
/*
* write the PMC value to the register/saved value: for
* sampling mode PMCs, the value to be programmed into the PMC
* counter is -(C+1) where 'C' is the requested sample rate.
*/
if (PMC_IS_SAMPLING_MODE(mode))
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
if (PMC_IS_SYSTEM_MODE(mode))
wrmsr(pd->pm_pmc_msr, v);
else
P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;
return (0);
}
/*
* Configure a PMC 'pm' on the given CPU and row-index.
*
* 'pm' may be NULL to indicate de-configuration.
*
* On HTT systems, a PMC may get configured twice, once for each
* "logical" CPU. We track this using the CFGFLAGS field of the
* per-cpu state; this field is a bit mask with one bit each for
* logical CPUs 0 & 1.
*/
static int
p4_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
struct p4_cpu *pc;
int cfgflags, cpuflag;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
phw = &pc->pc_p4pmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL ||
(p4_system_has_htt && phw->phw_pmc == pm),
("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
mtx_lock_spin(&pc->pc_mtx);
cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal cfgflags cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
KASSERT(cfgflags == 0 || phw->phw_pmc,
("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
__LINE__, cpu, ri));
cpuflag = P4_CPU_TO_FLAG(cpu);
if (pm) { /* config */
if (cfgflags == 0)
phw->phw_pmc = pm;
KASSERT(phw->phw_pmc == pm,
("[p4,%d] cpu=%d ri=%d config %p != hw %p",
__LINE__, cpu, ri, pm, phw->phw_pmc));
cfgflags |= cpuflag;
} else { /* unconfig */
cfgflags &= ~cpuflag;
if (cfgflags == 0)
phw->phw_pmc = NULL;
}
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
mtx_unlock_spin(&pc->pc_mtx);
return (0);
}
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
p4_get_config(int cpu, int ri, struct pmc **ppm)
{
int cfgflags;
struct p4_cpu *pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
mtx_lock_spin(&pc->pc_mtx);
cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
mtx_unlock_spin(&pc->pc_mtx);
if (cfgflags & P4_CPU_TO_FLAG(cpu))
*ppm = pc->pc_p4pmcs[ri].phw_pmc; /* PMC config'ed on this CPU */
else
*ppm = NULL;
return 0;
}
/*
* Allocate a PMC.
*
* The allocation strategy differs between HTT and non-HTT systems.
*
* The non-HTT case:
* - Given the desired event and the PMC row-index, lookup the
* list of valid ESCRs for the event.
* - For each valid ESCR:
* - Check if the ESCR is free and the ESCR row is in a compatible
* mode (i.e., system or process))
* - Check if the ESCR is usable with a P4 PMC at the desired row-index.
* If everything matches, we determine the appropriate bit values for the
* ESCR and CCCR registers.
*
* The HTT case:
*
* - Process mode PMCs require special care. The FreeBSD scheduler could
* schedule any two processes on the same physical CPU. We need to ensure
* that a given PMC row-index is never allocated to two different
* PMCs owned by different user-processes.
* This is ensured by always allocating a PMC from a 'FREE' PMC row
* if the system has HTT active.
* - A similar check needs to be done for ESCRs; we do not want two PMCs
* using the same ESCR to be scheduled at the same time. Thus ESCR
* allocation is also restricted to FREE rows if the system has HTT
* enabled.
* - Thirdly, some events are 'thread-independent' terminology, i.e.,
* the PMC hardware cannot distinguish between events caused by
* different logical CPUs. This makes it impossible to assign events
* to a given thread of execution. If the system has HTT enabled,
* these events are not allowed for process-mode PMCs.
*/
static int
p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int found, n, m;
uint32_t caps, cccrvalue, escrvalue, tflags;
enum pmc_p4escr escr;
struct p4_cpu *pc;
struct p4_event_descr *pevent;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index value %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
- PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
+ PMCDBG4(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
"reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
pm->pm_caps);
/* check class */
if (pd->pm_descr.pd_class != a->pm_class)
return (EINVAL);
/* check requested capabilities */
caps = a->pm_caps;
if ((pd->pm_descr.pd_caps & caps) != caps)
return (EPERM);
/*
* If the system has HTT enabled, and the desired allocation
* mode is process-private, and the PMC row disposition is not
* FREE (0), decline the allocation.
*/
if (p4_system_has_htt &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
pmc_getrowdisp(ri) != 0)
return (EBUSY);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__,
pd->pm_descr.pd_class));
if (pm->pm_event < PMC_EV_P4_FIRST ||
pm->pm_event > PMC_EV_P4_LAST)
return (EINVAL);
if ((pevent = p4_find_event(pm->pm_event)) == NULL)
return (ESRCH);
- PMCDBG(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}",
+ PMCDBG4(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}",
pevent->pm_event, pevent->pm_escr_eventselect,
pevent->pm_cccr_select, pevent->pm_is_ti_event);
/*
* Some PMC events are 'thread independent'and therefore
* cannot be used for process-private modes if HTT is being
* used.
*/
if (P4_EVENT_IS_TI(pevent) &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
p4_system_has_htt)
return (EINVAL);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
found = 0;
/* look for a suitable ESCR for this event */
for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) {
if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE)
break; /* out of ESCRs */
/*
* Check ESCR row disposition.
*
* If the request is for a system-mode PMC, then the
* ESCR row should not be in process-virtual mode, and
* should also be free on the current CPU.
*/
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
continue;
}
/*
* If the request is for a process-virtual PMC, and if
* HTT is not enabled, we can use an ESCR row that is
* either FREE or already in process mode.
*
* If HTT is enabled, then we need to ensure that a
* given ESCR is never allocated to two PMCS that
* could run simultaneously on the two logical CPUs of
* a CPU package. We ensure this be only allocating
* ESCRs from rows marked as 'FREE'.
*/
if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
if (p4_system_has_htt) {
if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
continue;
} else
if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr))
continue;
}
/*
* We found a suitable ESCR for this event. Now check if
* this escr can work with the PMC at row-index 'ri'.
*/
for (m = 0; m < P4_MAX_PMC_PER_ESCR; m++)
if (p4_escrs[escr].pm_pmcs[m] == pd->pm_pmcnum) {
found = 1;
break;
}
}
if (found == 0)
return (ESRCH);
KASSERT((int) escr >= 0 && escr < P4_NESCR,
("[p4,%d] illegal ESCR value %d", __LINE__, escr));
/* mark ESCR row mode */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
P4_ESCR_MARK_ROW_STANDALONE(escr);
} else {
KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX,
("[p4,%d] escr[%d] already in use", __LINE__, escr));
P4_ESCR_MARK_ROW_THREAD(escr);
}
pm->pm_md.pm_p4.pm_p4_escrmsr = p4_escrs[escr].pm_escr_msr;
pm->pm_md.pm_p4.pm_p4_escr = escr;
cccrvalue = P4_CCCR_TO_ESCR_SELECT(pevent->pm_cccr_select);
escrvalue = P4_ESCR_TO_EVENT_SELECT(pevent->pm_escr_eventselect);
/* CCCR fields */
if (caps & PMC_CAP_THRESHOLD)
cccrvalue |= (a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_THRESHOLD_MASK) | P4_CCCR_COMPARE;
if (caps & PMC_CAP_EDGE)
cccrvalue |= P4_CCCR_EDGE;
if (caps & PMC_CAP_INVERT)
cccrvalue |= P4_CCCR_COMPLEMENT;
if (p4_system_has_htt)
cccrvalue |= a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_ACTIVE_THREAD_MASK;
else /* no HTT; thread field should be '11b' */
cccrvalue |= P4_CCCR_TO_ACTIVE_THREAD(0x3);
if (caps & PMC_CAP_CASCADE)
cccrvalue |= P4_CCCR_CASCADE;
/* On HTT systems the PMI T0 field may get moved to T1 at pmc start */
if (caps & PMC_CAP_INTERRUPT)
cccrvalue |= P4_CCCR_OVF_PMI_T0;
/* ESCR fields */
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK;
if (caps & PMC_CAP_TAGGING)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_TAG_VALUE_MASK) | P4_ESCR_TAG_ENABLE;
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK);
/* HTT: T0_{OS,USR} bits may get moved to T1 at pmc start */
tflags = 0;
if (caps & PMC_CAP_SYSTEM)
tflags |= P4_ESCR_T0_OS;
if (caps & PMC_CAP_USER)
tflags |= P4_ESCR_T0_USR;
if (tflags == 0)
tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
escrvalue |= tflags;
pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue;
pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
- PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
+ PMCDBG5(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
"escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select,
cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
return (0);
}
/*
* release a PMC.
*/
static int
p4_release_pmc(int cpu, int ri, struct pmc *pm)
{
enum pmc_p4escr escr;
struct p4_cpu *pc;
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
escr = pm->pm_md.pm_p4.pm_p4_escr;
- PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
+ PMCDBG3(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
KASSERT(pc->pc_p4pmcs[ri].phw_pmc == NULL,
("[p4,%d] releasing configured PMC ri=%d", __LINE__, ri));
P4_ESCR_UNMARK_ROW_STANDALONE(escr);
KASSERT(pc->pc_escrs[escr] == ri,
("[p4,%d] escr[%d] not allocated to ri %d", __LINE__,
escr, ri));
pc->pc_escrs[escr] = P4_INVALID_PMC_INDEX; /* mark as free */
} else
P4_ESCR_UNMARK_ROW_THREAD(escr);
return (0);
}
/*
* Start a PMC
*/
static int
p4_start_pmc(int cpu, int ri)
{
int rc;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));
- PMCDBG(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] wrong PMC class %d", __LINE__,
pd->pm_descr.pd_class));
/* retrieve the desired CCCR/ESCR values from the PMC */
cccrvalue = pm->pm_md.pm_p4.pm_p4_cccrvalue;
escrvalue = pm->pm_md.pm_p4.pm_p4_escrvalue;
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
/* extract and zero the logical processor selection bits */
cccrtbits = cccrvalue & P4_CCCR_OVF_PMI_T0;
escrtbits = escrvalue & (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
cccrvalue &= ~P4_CCCR_OVF_PMI_T0;
escrvalue &= ~(P4_ESCR_T0_OS|P4_ESCR_T0_USR);
if (P4_CPU_IS_HTT_SECONDARY(cpu)) { /* shift T0 bits to T1 position */
cccrtbits <<= 1;
escrtbits >>= 2;
}
/* start system mode PMCs directly */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(escrmsr, escrvalue | escrtbits);
wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
return 0;
}
/*
* Thread mode PMCs
*
* On HTT machines, the same PMC could be scheduled on the
* same physical CPU twice (once for each logical CPU), for
* example, if two threads of a multi-threaded process get
* scheduled on the same CPU.
*
*/
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 0 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
if (rc == 0) { /* 1st CPU and the non-HTT case */
KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
cpu, ri, pd->pm_cccr_msr));
/* write out the low 40 bits of the saved value to hardware */
wrmsr(pd->pm_pmc_msr,
P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
} else if (rc == 1) { /* 2nd CPU */
/*
* Stop the PMC and retrieve the CCCR and ESCR values
* from their MSRs, and turn on the additional T[0/1]
* bits for the 2nd CPU.
*/
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
/* check that the configuration bits read back match the PMC */
KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
("[p4,%d] Extra CCCR bits cpu=%d rc=%d ri=%d "
"cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,
cccrvalue & P4_CCCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
KASSERT(cccrvalue & P4_CCCR_ENABLE,
("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
__LINE__, rc, cpu, ri));
KASSERT((cccrvalue & cccrtbits) == 0,
("[p4,%d] CCCR T0/T1 mismatch rc=%d cpu=%d ri=%d"
"cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
cccrvalue, cccrtbits));
escrvalue = rdmsr(escrmsr);
KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK),
("[p4,%d] Extra ESCR bits cpu=%d rc=%d ri=%d "
"escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
escrvalue & P4_ESCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
KASSERT((escrvalue & escrtbits) == 0,
("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
"escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
rc, cpu, ri, escrmsr, escrvalue, escrtbits));
}
/* Enable the correct bits for this CPU. */
escrvalue |= escrtbits;
cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
/* Save HW value at the time of starting hardware */
P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);
/* Program the ESCR and CCCR and start the PMC */
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
++rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
mtx_unlock_spin(&pc->pc_mtx);
- PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
- "escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x v=%jx", cpu, rc,
- ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
+ PMCDBG6(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
+ "escrmsr=0x%x escrvalue=0x%x", cpu, rc,
+ ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue);
+ PMCDBG2(MDP,STA,2,"cccr_config=0x%x v=%jx",
cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));
return (0);
}
/*
* Stop a PMC.
*/
static int
p4_stop_pmc(int cpu, int ri)
{
int rc;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row index %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
KASSERT(pm != NULL,
("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));
- PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(pd->pm_cccr_msr,
pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
return (0);
}
/*
* Thread mode PMCs.
*
* On HTT machines, this PMC may be in use by two threads
* running on two logical CPUS. Thus we look at the
* 'runcount' field and only turn off the appropriate TO/T1
* bits (and keep the PMC running) if two logical CPUs were
* using the PMC.
*
*/
/* bits to mask */
cccrtbits = P4_CCCR_OVF_PMI_T0;
escrtbits = P4_ESCR_T0_OS | P4_ESCR_T0_USR;
if (P4_CPU_IS_HTT_SECONDARY(cpu)) {
cccrtbits <<= 1;
escrtbits >>= 2;
}
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 2 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
--rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
/* Stop this PMC */
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
escrvalue = rdmsr(escrmsr);
/* The current CPU should be running on this PMC */
KASSERT(escrvalue & escrtbits,
("[p4,%d] ESCR T0/T1 mismatch cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,
escrvalue, escrtbits));
KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)) ||
(cccrvalue & cccrtbits),
("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
"tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
/* get the current hardware reading */
tmp = rdmsr(pd->pm_pmc_msr);
if (rc == 1) { /* need to keep the PMC running */
escrvalue &= ~escrtbits;
cccrvalue &= ~cccrtbits;
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
}
mtx_unlock_spin(&pc->pc_mtx);
- PMCDBG(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
- "escrval=0x%x cccrval=0x%x v=%jx", cpu, rc, ri, escrmsr,
- escrvalue, cccrvalue, tmp);
+ PMCDBG5(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
+ "escrval=0x%x", cpu, rc, ri, escrmsr, escrvalue);
+ PMCDBG2(MDP,STO,2, "cccrval=0x%x v=%jx", cccrvalue, tmp);
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */
tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
return 0;
}
/*
* Handle an interrupt.
*
* The hardware sets the CCCR_OVF whenever a counter overflow occurs,
* so the handler examines all the 18 CCCR registers, processing the
* counters that have overflowed.
*
* On HTT machines, the CCCR register is shared and will interrupt
* both logical processors if so configured. Thus multiple logical
* CPUs could enter the NMI service routine at the same time. These
* will get serialized using a per-cpu spinlock dedicated for use in
* the NMI handler.
*/
static int
p4_intr(int cpu, struct trapframe *tf)
{
uint32_t cccrval, ovf_mask, ovf_partner;
int did_interrupt, error, ri;
struct p4_cpu *pc;
struct pmc *pm;
pmc_value_t v;
- PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
ovf_mask |= P4_CCCR_OVF;
if (p4_system_has_htt)
ovf_partner = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T0 : P4_CCCR_OVF_PMI_T1;
else
ovf_partner = 0;
did_interrupt = 0;
if (p4_system_has_htt)
P4_PCPU_ACQ_INTR_SPINLOCK(pc);
/*
* Loop through all CCCRs, looking for ones that have
* interrupted this CPU.
*/
for (ri = 0; ri < P4_NPMCS; ri++) {
/*
* Check if our partner logical CPU has already marked
* this PMC has having interrupted it. If so, reset
* the flag and process the interrupt, but leave the
* hardware alone.
*/
if (p4_system_has_htt && P4_PCPU_GET_INTRFLAG(pc,ri)) {
P4_PCPU_SET_INTRFLAG(pc,ri,0);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs.
* Ignore PMCs not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
continue;
}
/*
* Fresh interrupt. Look for the CCCR_OVF bit
* and the OVF_Tx bit for this logical
* processor being set.
*/
cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
if ((cccrval & ovf_mask) != ovf_mask)
continue;
/*
* If the other logical CPU would also have been
* interrupted due to the PMC being shared, record
* this fact in the per-cpu saved interrupt flag
* bitmask.
*/
if (p4_system_has_htt && (cccrval & ovf_partner))
P4_PCPU_SET_INTRFLAG(pc, ri, 1);
v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
- PMCDBG(MDP,INT, 2, "ri=%d v=%jx", ri, v);
+ PMCDBG2(MDP,INT, 2, "ri=%d v=%jx", ri, v);
/* Stop the counter, and reset the overflow bit */
cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs. Ignore PMCs
* not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
/*
* Process the interrupt. Re-enable the PMC if
* processing was successful.
*/
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
/*
* Only the first processor executing the NMI handler
* in a HTT pair will restart a PMC, and that too
* only if there were no errors.
*/
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
pm->pm_sc.pm_reloadcount);
wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
if (error == 0)
wrmsr(P4_CCCR_MSR_FIRST + ri,
cccrval | P4_CCCR_ENABLE);
}
/* allow the other CPU to proceed */
if (p4_system_has_htt)
P4_PCPU_REL_INTR_SPINLOCK(pc);
/*
* On Intel P4 CPUs, the PMC 'pcint' entry in the LAPIC gets
* masked when a PMC interrupts the CPU. We need to unmask
* the interrupt source explicitly.
*/
if (did_interrupt)
lapic_reenable_pmc();
atomic_add_int(did_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (did_interrupt);
}
/*
* Describe a CPU's PMC state.
*/
static int
p4_describe(int cpu, int ri, struct pmc_info *pi,
struct pmc **ppmc)
{
int error;
size_t copied;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] row-index %d out of range", __LINE__, ri));
- PMCDBG(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri);
if (P4_CPU_IS_HTT_SECONDARY(cpu))
return (EINVAL);
pd = &p4_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
/*
* Get MSR# for use with RDPMC.
*/
static int
p4_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] ri %d out of range", __LINE__, ri));
*msr = p4_pmcdesc[ri].pm_pmc_msr - P4_PERFCTR_MSR_FIRST;
- PMCDBG(MDP,OPS, 1, "ri=%d getmsr=0x%x", ri, *msr);
+ PMCDBG2(MDP,OPS, 1, "ri=%d getmsr=0x%x", ri, *msr);
return 0;
}
int
pmc_p4_initialize(struct pmc_mdep *md, int ncpus)
{
struct pmc_classdep *pcd;
struct p4_event_descr *pe;
KASSERT(md != NULL, ("[p4,%d] md is NULL", __LINE__));
KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
("[p4,%d] Initializing non-intel processor", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "p4-initialize");
+ PMCDBG0(MDP,INI,1, "p4-initialize");
/* Allocate space for pointers to per-cpu descriptors. */
p4_pcpu = malloc(sizeof(*p4_pcpu) * ncpus, M_PMC, M_ZERO | M_WAITOK);
/* Fill in the class dependent descriptor. */
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4];
switch (md->pmd_cputype) {
case PMC_CPU_INTEL_PIV:
pcd->pcd_caps = P4_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_P4;
pcd->pcd_num = P4_NPMCS;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = 40;
pcd->pcd_allocate_pmc = p4_allocate_pmc;
pcd->pcd_config_pmc = p4_config_pmc;
pcd->pcd_describe = p4_describe;
pcd->pcd_get_config = p4_get_config;
pcd->pcd_get_msr = p4_get_msr;
pcd->pcd_pcpu_fini = p4_pcpu_fini;
pcd->pcd_pcpu_init = p4_pcpu_init;
pcd->pcd_read_pmc = p4_read_pmc;
pcd->pcd_release_pmc = p4_release_pmc;
pcd->pcd_start_pmc = p4_start_pmc;
pcd->pcd_stop_pmc = p4_stop_pmc;
pcd->pcd_write_pmc = p4_write_pmc;
md->pmd_pcpu_fini = NULL;
md->pmd_pcpu_init = NULL;
md->pmd_intr = p4_intr;
md->pmd_npmc += P4_NPMCS;
/* model specific configuration */
if ((cpu_id & 0xFFF) < 0xF27) {
/*
* On P4 and Xeon with CPUID < (Family 15,
* Model 2, Stepping 7), only one ESCR is
* available for the IOQ_ALLOCATION event.
*/
pe = p4_find_event(PMC_EV_P4_IOQ_ALLOCATION);
pe->pm_escrs[1] = P4_ESCR_NONE;
}
break;
default:
KASSERT(0,("[p4,%d] Unknown CPU type", __LINE__));
return ENOSYS;
}
return (0);
}
void
pmc_p4_finalize(struct pmc_mdep *md)
{
#if defined(INVARIANTS)
int i, ncpus;
#endif
KASSERT(p4_pcpu != NULL,
("[p4,%d] NULL p4_pcpu", __LINE__));
#if defined(INVARIANTS)
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(p4_pcpu[i] == NULL, ("[p4,%d] non-null pcpu %d",
__LINE__, i));
#endif
free(p4_pcpu, M_PMC);
p4_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_ppc970.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_ppc970.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_ppc970.c (revision 282658)
@@ -1,687 +1,687 @@
/*-
* Copyright (c) 2013 Justin Hibbits
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/pmc_mdep.h>
#include <machine/spr.h>
#include <machine/cpu.h>
#include "hwpmc_powerpc.h"
#define PPC970_MAX_PMCS 8
/* MMCR0, PMC1 is 8 bytes in, PMC2 is 1 byte in. */
#define PPC970_SET_MMCR0_PMCSEL(r, x, i) \
((r & ~(0x1f << (7 * (1 - i) + 1))) | (x << (7 * (1 - i) + 1)))
/* MMCR1 has 6 PMC*SEL items (PMC3->PMC8), in sequence. */
#define PPC970_SET_MMCR1_PMCSEL(r, x, i) \
((r & ~(0x1f << (5 * (7 - i) + 2))) | (x << (5 * (7 - i) + 2)))
#define PPC970_PMC_HAS_OVERFLOWED(x) (ppc970_pmcn_read(x) & (0x1 << 31))
/* How PMC works on PPC970:
*
* Any PMC can count a direct event. Indirect events are handled specially.
* Direct events: As published.
*
* Encoding 00 000 -- Add byte lane bit counters
* MMCR1[24:31] -- select bit matching PMC being an adder.
* Bus events:
* PMCxSEL: 1x -- select from byte lane: 10 == lower lane (0/1), 11 == upper
* lane (2/3).
* PMCxSEL[2:4] -- bit in the byte lane selected.
*
* PMC[1,2,5,6] == lane 0/lane 2
* PMC[3,4,7,8] == lane 1,3
*
*
* Lanes:
* Lane 0 -- TTM0(FPU,ISU,IFU,VPU)
* TTM1(IDU,ISU,STS)
* LSU0 byte 0
* LSU1 byte 0
* Lane 1 -- TTM0
* TTM1
* LSU0 byte 1
* LSU1 byte 1
* Lane 2 -- TTM0
* TTM1
* LSU0 byte 2
* LSU1 byte 2 or byte 6
* Lane 3 -- TTM0
* TTM1
* LSU0 byte 3
* LSU1 byte 3 or byte 7
*
* Adders:
* Add byte lane for PMC (above), bit 0+4, 1+5, 2+6, 3+7
*/
struct pmc_ppc970_event {
enum pmc_event pe_event;
uint32_t pe_flags;
#define PMC_PPC970_FLAG_PMCS 0x000000ff
#define PMC_PPC970_FLAG_PMC1 0x01
#define PMC_PPC970_FLAG_PMC2 0x02
#define PMC_PPC970_FLAG_PMC3 0x04
#define PMC_PPC970_FLAG_PMC4 0x08
#define PMC_PPC970_FLAG_PMC5 0x10
#define PMC_PPC970_FLAG_PMC6 0x20
#define PMC_PPC970_FLAG_PMC7 0x40
#define PMC_PPC970_FLAG_PMC8 0x80
uint32_t pe_code;
};
static struct pmc_ppc970_event ppc970_event_codes[] = {
{PMC_EV_PPC970_INSTR_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMCS,
.pe_code = 0x09
},
{PMC_EV_PPC970_MARKED_GROUP_DISPATCH,
.pe_flags = PMC_PPC970_FLAG_PMC1,
.pe_code = 0x2
},
{PMC_EV_PPC970_MARKED_STORE_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC1,
.pe_code = 0x03
},
{PMC_EV_PPC970_GCT_EMPTY,
.pe_flags = PMC_PPC970_FLAG_PMC1,
.pe_code = 0x04
},
{PMC_EV_PPC970_RUN_CYCLES,
.pe_flags = PMC_PPC970_FLAG_PMC1,
.pe_code = 0x05
},
{PMC_EV_PPC970_OVERFLOW,
.pe_flags = PMC_PPC970_FLAG_PMCS,
.pe_code = 0x0a
},
{PMC_EV_PPC970_CYCLES,
.pe_flags = PMC_PPC970_FLAG_PMCS,
.pe_code = 0x0f
},
{PMC_EV_PPC970_THRESHOLD_TIMEOUT,
.pe_flags = PMC_PPC970_FLAG_PMC2,
.pe_code = 0x3
},
{PMC_EV_PPC970_GROUP_DISPATCH,
.pe_flags = PMC_PPC970_FLAG_PMC2,
.pe_code = 0x4
},
{PMC_EV_PPC970_BR_MARKED_INSTR_FINISH,
.pe_flags = PMC_PPC970_FLAG_PMC2,
.pe_code = 0x5
},
{PMC_EV_PPC970_GCT_EMPTY_BY_SRQ_FULL,
.pe_flags = PMC_PPC970_FLAG_PMC2,
.pe_code = 0xb
},
{PMC_EV_PPC970_STOP_COMPLETION,
.pe_flags = PMC_PPC970_FLAG_PMC3,
.pe_code = 0x1
},
{PMC_EV_PPC970_LSU_EMPTY,
.pe_flags = PMC_PPC970_FLAG_PMC3,
.pe_code = 0x2
},
{PMC_EV_PPC970_MARKED_STORE_WITH_INTR,
.pe_flags = PMC_PPC970_FLAG_PMC3,
.pe_code = 0x3
},
{PMC_EV_PPC970_CYCLES_IN_SUPER,
.pe_flags = PMC_PPC970_FLAG_PMC3,
.pe_code = 0x4
},
{PMC_EV_PPC970_VPU_MARKED_INSTR_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC3,
.pe_code = 0x5
},
{PMC_EV_PPC970_FXU0_IDLE_FXU1_BUSY,
.pe_flags = PMC_PPC970_FLAG_PMC4,
.pe_code = 0x2
},
{PMC_EV_PPC970_SRQ_EMPTY,
.pe_flags = PMC_PPC970_FLAG_PMC4,
.pe_code = 0x3
},
{PMC_EV_PPC970_MARKED_GROUP_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC4,
.pe_code = 0x4
},
{PMC_EV_PPC970_CR_MARKED_INSTR_FINISH,
.pe_flags = PMC_PPC970_FLAG_PMC4,
.pe_code = 0x5
},
{PMC_EV_PPC970_DISPATCH_SUCCESS,
.pe_flags = PMC_PPC970_FLAG_PMC5,
.pe_code = 0x1
},
{PMC_EV_PPC970_FXU0_IDLE_FXU1_IDLE,
.pe_flags = PMC_PPC970_FLAG_PMC5,
.pe_code = 0x2
},
{PMC_EV_PPC970_ONE_PLUS_INSTR_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC5,
.pe_code = 0x3
},
{PMC_EV_PPC970_GROUP_MARKED_IDU,
.pe_flags = PMC_PPC970_FLAG_PMC5,
.pe_code = 0x4
},
{PMC_EV_PPC970_MARKED_GROUP_COMPLETE_TIMEOUT,
.pe_flags = PMC_PPC970_FLAG_PMC5,
.pe_code = 0x5
},
{PMC_EV_PPC970_FXU0_BUSY_FXU1_BUSY,
.pe_flags = PMC_PPC970_FLAG_PMC6,
.pe_code = 0x2
},
{PMC_EV_PPC970_MARKED_STORE_SENT_TO_STS,
.pe_flags = PMC_PPC970_FLAG_PMC6,
.pe_code = 0x3
},
{PMC_EV_PPC970_FXU_MARKED_INSTR_FINISHED,
.pe_flags = PMC_PPC970_FLAG_PMC6,
.pe_code = 0x4
},
{PMC_EV_PPC970_MARKED_GROUP_ISSUED,
.pe_flags = PMC_PPC970_FLAG_PMC6,
.pe_code = 0x5
},
{PMC_EV_PPC970_FXU0_BUSY_FXU1_IDLE,
.pe_flags = PMC_PPC970_FLAG_PMC7,
.pe_code = 0x2
},
{PMC_EV_PPC970_GROUP_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC7,
.pe_code = 0x3
},
{PMC_EV_PPC970_FPU_MARKED_INSTR_COMPLETED,
.pe_flags = PMC_PPC970_FLAG_PMC7,
.pe_code = 0x4
},
{PMC_EV_PPC970_MARKED_INSTR_FINISH_ANY_UNIT,
.pe_flags = PMC_PPC970_FLAG_PMC7,
.pe_code = 0x5
},
{PMC_EV_PPC970_EXTERNAL_INTERRUPT,
.pe_flags = PMC_PPC970_FLAG_PMC8,
.pe_code = 0x2
},
{PMC_EV_PPC970_GROUP_DISPATCH_REJECT,
.pe_flags = PMC_PPC970_FLAG_PMC8,
.pe_code = 0x3
},
{PMC_EV_PPC970_LSU_MARKED_INSTR_FINISH,
.pe_flags = PMC_PPC970_FLAG_PMC8,
.pe_code = 0x4
},
{PMC_EV_PPC970_TIMEBASE_EVENT,
.pe_flags = PMC_PPC970_FLAG_PMC8,
.pe_code = 0x5
},
#if 0
{PMC_EV_PPC970_LSU_COMPLETION_STALL, },
{PMC_EV_PPC970_FXU_COMPLETION_STALL, },
{PMC_EV_PPC970_DCACHE_MISS_COMPLETION_STALL, },
{PMC_EV_PPC970_FPU_COMPLETION_STALL, },
{PMC_EV_PPC970_FXU_LONG_INSTR_COMPLETION_STALL, },
{PMC_EV_PPC970_REJECT_COMPLETION_STALL, },
{PMC_EV_PPC970_FPU_LONG_INSTR_COMPLETION_STALL, },
{PMC_EV_PPC970_GCT_EMPTY_BY_ICACHE_MISS, },
{PMC_EV_PPC970_REJECT_COMPLETION_STALL_ERAT_MISS, },
{PMC_EV_PPC970_GCT_EMPTY_BY_BRANCH_MISS_PREDICT, },
#endif
};
static size_t ppc970_event_codes_size = nitems(ppc970_event_codes);
static pmc_value_t
ppc970_pmcn_read(unsigned int pmc)
{
pmc_value_t val;
switch (pmc) {
case 0:
val = mfspr(SPR_970PMC1);
break;
case 1:
val = mfspr(SPR_970PMC2);
break;
case 2:
val = mfspr(SPR_970PMC3);
break;
case 3:
val = mfspr(SPR_970PMC4);
break;
case 4:
val = mfspr(SPR_970PMC5);
break;
case 5:
val = mfspr(SPR_970PMC6);
break;
case 6:
val = mfspr(SPR_970PMC7);
break;
case 7:
val = mfspr(SPR_970PMC8);
break;
default:
panic("Invalid PMC number: %d\n", pmc);
}
return (val);
}
static void
ppc970_pmcn_write(unsigned int pmc, uint32_t val)
{
switch (pmc) {
case 0:
mtspr(SPR_970PMC1, val);
break;
case 1:
mtspr(SPR_970PMC2, val);
break;
case 2:
mtspr(SPR_970PMC3, val);
break;
case 3:
mtspr(SPR_970PMC4, val);
break;
case 4:
mtspr(SPR_970PMC5, val);
break;
case 5:
mtspr(SPR_970PMC6, val);
break;
case 6:
mtspr(SPR_970PMC7, val);
break;
case 7:
mtspr(SPR_970PMC8, val);
break;
default:
panic("Invalid PMC number: %d\n", pmc);
}
}
static int
ppc970_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < PPC970_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
static int
ppc970_set_pmc(int cpu, int ri, int config)
{
struct pmc *pm;
struct pmc_hw *phw;
register_t pmc_mmcr;
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
pm = phw->phw_pmc;
/*
* Disable the PMCs.
*/
switch (ri) {
case 0:
case 1:
pmc_mmcr = mfspr(SPR_970MMCR0);
pmc_mmcr = PPC970_SET_MMCR0_PMCSEL(pmc_mmcr, config, ri);
mtspr(SPR_970MMCR0, pmc_mmcr);
break;
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
pmc_mmcr = mfspr(SPR_970MMCR1);
pmc_mmcr = PPC970_SET_MMCR1_PMCSEL(pmc_mmcr, config, ri);
mtspr(SPR_970MMCR1, pmc_mmcr);
break;
}
return 0;
}
static int
ppc970_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct pmc_hw *phw;
register_t pmc_mmcr;
uint32_t config;
int error;
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
pm = phw->phw_pmc;
config = pm->pm_md.pm_powerpc.pm_powerpc_evsel & ~POWERPC_PMC_ENABLE;
error = ppc970_set_pmc(cpu, ri, config);
/* The mask is inverted (enable is 1) compared to the flags in MMCR0, which
* are Freeze flags.
*/
config = ~pm->pm_md.pm_powerpc.pm_powerpc_evsel & POWERPC_PMC_ENABLE;
pmc_mmcr = mfspr(SPR_970MMCR0);
pmc_mmcr &= ~SPR_MMCR0_FC;
pmc_mmcr |= config;
mtspr(SPR_970MMCR0, pmc_mmcr);
return 0;
}
static int
ppc970_stop_pmc(int cpu, int ri)
{
return ppc970_set_pmc(cpu, ri, PMC970N_NONE);
}
static int
ppc970_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < PPC970_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = ppc970_pmcn_read(ri);
- PMCDBG(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp);
+ PMCDBG2(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
return 0;
}
static int
ppc970_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < PPC970_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
- PMCDBG(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v);
ppc970_pmcn_write(ri, v);
return 0;
}
static int
ppc970_intr(int cpu, struct trapframe *tf)
{
struct pmc *pm;
struct powerpc_cpu *pac;
uint32_t config;
int i, error, retval;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
- PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
+ PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
config = mfspr(SPR_970MMCR0) & ~SPR_MMCR0_FC;
for (i = 0; i < PPC970_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!PPC970_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error != 0)
ppc970_stop_pmc(cpu, i);
/* reload sampling count. */
ppc970_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_970MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
static int
ppc970_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct powerpc_cpu *pac;
struct pmc_hw *phw;
int first_ri, i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] wrong cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"powerpc-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu);
powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * PPC970_MAX_PMCS,
M_PMC, M_WAITOK|M_ZERO);
pac->pc_class = PMC_CLASS_PPC970;
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri;
KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_ppcpmcs; i < PPC970_MAX_PMCS; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/* Clear the MMCRs, and set FC, to disable all PMCs. */
/* 970 PMC is not counted when set to 0x08 */
mtspr(SPR_970MMCR0, SPR_MMCR0_FC | SPR_MMCR0_PMXE |
SPR_MMCR0_FCECE | SPR_MMCR0_PMC1CE | SPR_MMCR0_PMCNCE |
SPR_970MMCR0_PMC1SEL(0x8) | SPR_970MMCR0_PMC2SEL(0x8));
mtspr(SPR_970MMCR1, 0x4218420);
return 0;
}
static int
ppc970_pcpu_fini(struct pmc_mdep *md, int cpu)
{
register_t mmcr0 = mfspr(SPR_MMCR0);
mmcr0 |= SPR_MMCR0_FC;
mmcr0 &= ~SPR_MMCR0_PMXE;
mtspr(SPR_MMCR0, mmcr0);
free(powerpc_pcpu[cpu]->pc_ppcpmcs, M_PMC);
free(powerpc_pcpu[cpu], M_PMC);
return 0;
}
static int
ppc970_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event pe;
uint32_t caps, config = 0, counter = 0;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < PPC970_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
caps = a->pm_caps;
pe = a->pm_ev;
if (pe < PMC_EV_PPC970_FIRST || pe > PMC_EV_PPC970_LAST)
return (EINVAL);
for (i = 0; i < ppc970_event_codes_size; i++) {
if (ppc970_event_codes[i].pe_event == pe) {
config = ppc970_event_codes[i].pe_code;
counter = ppc970_event_codes[i].pe_flags;
break;
}
}
if (i == ppc970_event_codes_size)
return (EINVAL);
if ((counter & (1 << ri)) == 0)
return (EINVAL);
if (caps & PMC_CAP_SYSTEM)
config |= POWERPC_PMC_KERNEL_ENABLE;
if (caps & PMC_CAP_USER)
config |= POWERPC_PMC_USER_ENABLE;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
config |= POWERPC_PMC_ENABLE;
pm->pm_md.pm_powerpc.pm_powerpc_evsel = config;
- PMCDBG(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
ppc970_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < PPC970_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
KASSERT(phw->phw_pmc == NULL,
("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
return 0;
}
int
pmc_ppc970_initialize(struct pmc_mdep *pmc_mdep)
{
struct pmc_classdep *pcd;
pmc_mdep->pmd_cputype = PMC_CPU_PPC_970;
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC];
pcd->pcd_caps = POWERPC_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_PPC970;
pcd->pcd_num = PPC970_MAX_PMCS;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 32;
pcd->pcd_allocate_pmc = ppc970_allocate_pmc;
pcd->pcd_config_pmc = ppc970_config_pmc;
pcd->pcd_pcpu_fini = ppc970_pcpu_fini;
pcd->pcd_pcpu_init = ppc970_pcpu_init;
pcd->pcd_describe = powerpc_describe;
pcd->pcd_get_config = powerpc_get_config;
pcd->pcd_read_pmc = ppc970_read_pmc;
pcd->pcd_release_pmc = ppc970_release_pmc;
pcd->pcd_start_pmc = ppc970_start_pmc;
pcd->pcd_stop_pmc = ppc970_stop_pmc;
pcd->pcd_write_pmc = ppc970_write_pmc;
pmc_mdep->pmd_npmc += PPC970_MAX_PMCS;
pmc_mdep->pmd_intr = ppc970_intr;
return (0);
}
Index: head/sys/dev/hwpmc/hwpmc_ppro.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_ppro.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_ppro.c (revision 282658)
@@ -1,866 +1,866 @@
/*-
* Copyright (c) 2003-2005,2008 Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/cputypes.h>
#include <machine/md_var.h>
#include <machine/pmc_mdep.h>
#include <machine/specialreg.h>
/*
* PENTIUM PRO SUPPORT
*
* Quirks:
*
* - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance
* counter '0'. This bit needs to be '1' if any of the two
* performance counters are in use. Perf counters can also be
* switched off by writing zeros to their EVSEL register.
*
* - While the width of these counters is 40 bits, we do not appear to
* have a way of writing 40 bits to the counter MSRs. A WRMSR
* instruction will sign extend bit 31 of the value being written to
* the perf counter -- a value of 0x80000000 written to an perf
* counter register will be sign extended to 0xFF80000000.
*
* This quirk primarily affects thread-mode PMCs in counting mode, as
* these PMCs read and write PMC registers at every context switch.
*/
struct p6pmc_descr {
struct pmc_descr pm_descr; /* common information */
uint32_t pm_pmc_msr;
uint32_t pm_evsel_msr;
};
static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = {
#define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
PMC_CAP_INVERT | PMC_CAP_QUALIFIER)
/* PMC 0 */
{
.pm_descr =
{
.pd_name ="P6-0",
.pd_class = PMC_CLASS_P6,
.pd_caps = P6_PMC_CAPS,
.pd_width = 40
},
.pm_pmc_msr = P6_MSR_PERFCTR0,
.pm_evsel_msr = P6_MSR_EVSEL0
},
/* PMC 1 */
{
.pm_descr =
{
.pd_name ="P6-1",
.pd_class = PMC_CLASS_P6,
.pd_caps = P6_PMC_CAPS,
.pd_width = 40
},
.pm_pmc_msr = P6_MSR_PERFCTR1,
.pm_evsel_msr = P6_MSR_EVSEL1
}
};
static enum pmc_cputype p6_cputype;
/*
* P6 Event descriptor
*
* The 'pm_flags' field has the following structure:
* - The upper 4 bits are used to track which counter an event is valid on.
* - The lower bits form a bitmask of flags indicating support for the event
* on a given CPU.
*/
struct p6_event_descr {
const enum pmc_event pm_event;
uint32_t pm_evsel;
uint32_t pm_flags;
uint32_t pm_unitmask;
};
#define P6F_CTR(C) (1 << (28 + (C)))
#define P6F_CTR0 P6F_CTR(0)
#define P6F_CTR1 P6F_CTR(1)
#define P6F(CPU) (1 << ((CPU) - PMC_CPU_INTEL_P6))
#define _P6F(C) P6F(PMC_CPU_INTEL_##C)
#define P6F_P6 _P6F(P6)
#define P6F_CL _P6F(CL)
#define P6F_PII _P6F(PII)
#define P6F_PIII _P6F(PIII)
#define P6F_PM _P6F(PM)
#define P6F_ALL_CPUS (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM)
#define P6F_ALL_CTRS (P6F_CTR0 | P6F_CTR1)
#define P6F_ALL (P6F_ALL_CPUS | P6F_ALL_CTRS)
#define P6_EVENT_VALID_FOR_CPU(P,CPU) ((P)->pm_flags & P6F(CPU))
#define P6_EVENT_VALID_FOR_CTR(P,CTR) ((P)->pm_flags & P6F_CTR(CTR))
static const struct p6_event_descr p6_events[] = {
#define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK) \
{ \
.pm_event = PMC_EV_P6_##NAME, \
.pm_evsel = (EVSEL), \
.pm_flags = (FLAGS), \
.pm_unitmask = (UMASK) \
}
P6_EVDESCR(DATA_MEM_REFS, 0x43, P6F_ALL, 0x00),
P6_EVDESCR(DCU_LINES_IN, 0x45, P6F_ALL, 0x00),
P6_EVDESCR(DCU_M_LINES_IN, 0x46, P6F_ALL, 0x00),
P6_EVDESCR(DCU_M_LINES_OUT, 0x47, P6F_ALL, 0x00),
P6_EVDESCR(DCU_MISS_OUTSTANDING, 0x47, P6F_ALL, 0x00),
P6_EVDESCR(IFU_FETCH, 0x80, P6F_ALL, 0x00),
P6_EVDESCR(IFU_FETCH_MISS, 0x81, P6F_ALL, 0x00),
P6_EVDESCR(ITLB_MISS, 0x85, P6F_ALL, 0x00),
P6_EVDESCR(IFU_MEM_STALL, 0x86, P6F_ALL, 0x00),
P6_EVDESCR(ILD_STALL, 0x87, P6F_ALL, 0x00),
P6_EVDESCR(L2_IFETCH, 0x28, P6F_ALL, 0x0F),
P6_EVDESCR(L2_LD, 0x29, P6F_ALL, 0x0F),
P6_EVDESCR(L2_ST, 0x2A, P6F_ALL, 0x0F),
P6_EVDESCR(L2_LINES_IN, 0x24, P6F_ALL, 0x0F),
P6_EVDESCR(L2_LINES_OUT, 0x26, P6F_ALL, 0x0F),
P6_EVDESCR(L2_M_LINES_INM, 0x25, P6F_ALL, 0x00),
P6_EVDESCR(L2_M_LINES_OUTM, 0x27, P6F_ALL, 0x0F),
P6_EVDESCR(L2_RQSTS, 0x2E, P6F_ALL, 0x0F),
P6_EVDESCR(L2_ADS, 0x21, P6F_ALL, 0x00),
P6_EVDESCR(L2_DBUS_BUSY, 0x22, P6F_ALL, 0x00),
P6_EVDESCR(L2_DBUS_BUSY_RD, 0x23, P6F_ALL, 0x00),
P6_EVDESCR(BUS_DRDY_CLOCKS, 0x62, P6F_ALL, 0x20),
P6_EVDESCR(BUS_LOCK_CLOCKS, 0x63, P6F_ALL, 0x20),
P6_EVDESCR(BUS_REQ_OUTSTANDING, 0x60, P6F_ALL, 0x00),
P6_EVDESCR(BUS_TRAN_BRD, 0x65, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_RFO, 0x66, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRANS_WB, 0x67, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_IFETCH, 0x68, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_INVAL, 0x69, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_PWR, 0x6A, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRANS_P, 0x6B, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRANS_IO, 0x6C, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_DEF, 0x6D, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_BURST, 0x6E, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_ANY, 0x70, P6F_ALL, 0x20),
P6_EVDESCR(BUS_TRAN_MEM, 0x6F, P6F_ALL, 0x20),
P6_EVDESCR(BUS_DATA_RCV, 0x64, P6F_ALL, 0x00),
P6_EVDESCR(BUS_BNR_DRV, 0x61, P6F_ALL, 0x00),
P6_EVDESCR(BUS_HIT_DRV, 0x7A, P6F_ALL, 0x00),
P6_EVDESCR(BUS_HITM_DRV, 0x7B, P6F_ALL, 0x00),
P6_EVDESCR(BUS_SNOOP_STALL, 0x7E, P6F_ALL, 0x00),
P6_EVDESCR(FLOPS, 0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00),
P6_EVDESCR(FP_COMPS_OPS_EXE, 0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00),
P6_EVDESCR(FP_ASSIST, 0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00),
P6_EVDESCR(MUL, 0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00),
P6_EVDESCR(DIV, 0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00),
P6_EVDESCR(CYCLES_DIV_BUSY, 0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00),
P6_EVDESCR(LD_BLOCKS, 0x03, P6F_ALL, 0x00),
P6_EVDESCR(SB_DRAINS, 0x04, P6F_ALL, 0x00),
P6_EVDESCR(MISALIGN_MEM_REF, 0x05, P6F_ALL, 0x00),
P6_EVDESCR(EMON_KNI_PREF_DISPATCHED, 0x07, P6F_PIII | P6F_ALL_CTRS, 0x03),
P6_EVDESCR(EMON_KNI_PREF_MISS, 0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03),
P6_EVDESCR(INST_RETIRED, 0xC0, P6F_ALL, 0x00),
P6_EVDESCR(UOPS_RETIRED, 0xC2, P6F_ALL, 0x00),
P6_EVDESCR(INST_DECODED, 0xD0, P6F_ALL, 0x00),
P6_EVDESCR(EMON_KNI_INST_RETIRED, 0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01),
P6_EVDESCR(EMON_KNI_COMP_INST_RET, 0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01),
P6_EVDESCR(HW_INT_RX, 0xC8, P6F_ALL, 0x00),
P6_EVDESCR(CYCLES_INT_MASKED, 0xC6, P6F_ALL, 0x00),
P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00),
P6_EVDESCR(BR_INST_RETIRED, 0xC4, P6F_ALL, 0x00),
P6_EVDESCR(BR_MISS_PRED_RETIRED, 0xC5, P6F_ALL, 0x00),
P6_EVDESCR(BR_TAKEN_RETIRED, 0xC9, P6F_ALL, 0x00),
P6_EVDESCR(BR_MISS_PRED_TAKEN_RET, 0xCA, P6F_ALL, 0x00),
P6_EVDESCR(BR_INST_DECODED, 0xE0, P6F_ALL, 0x00),
P6_EVDESCR(BTB_MISSES, 0xE2, P6F_ALL, 0x00),
P6_EVDESCR(BR_BOGUS, 0xE4, P6F_ALL, 0x00),
P6_EVDESCR(BACLEARS, 0xE6, P6F_ALL, 0x00),
P6_EVDESCR(RESOURCE_STALLS, 0xA2, P6F_ALL, 0x00),
P6_EVDESCR(PARTIAL_RAT_STALLS, 0xD2, P6F_ALL, 0x00),
P6_EVDESCR(SEGMENT_REG_LOADS, 0x06, P6F_ALL, 0x00),
P6_EVDESCR(CPU_CLK_UNHALTED, 0x79, P6F_ALL, 0x00),
P6_EVDESCR(MMX_INSTR_EXEC, 0xB0,
P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00),
P6_EVDESCR(MMX_SAT_INSTR_EXEC, 0xB1,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
P6_EVDESCR(MMX_UOPS_EXEC, 0xB2,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
P6_EVDESCR(MMX_INSTR_TYPE_EXEC, 0xB3,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F),
P6_EVDESCR(FP_MMX_TRANS, 0xCC,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01),
P6_EVDESCR(MMX_ASSIST, 0xCD,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
P6_EVDESCR(MMX_INSTR_RET, 0xCE, P6F_ALL_CTRS | P6F_PII, 0x00),
P6_EVDESCR(SEG_RENAME_STALLS, 0xD4,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
P6_EVDESCR(SEG_REG_RENAMES, 0xD5,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F),
P6_EVDESCR(RET_SEG_RENAMES, 0xD6,
P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00),
P6_EVDESCR(EMON_EST_TRANS, 0x58, P6F_ALL_CTRS | P6F_PM, 0x02),
P6_EVDESCR(EMON_THERMAL_TRIP, 0x59, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_INST_EXEC, 0x88, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_MISSP_EXEC, 0x89, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_BAC_MISSP_EXEC, 0x8A, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_CND_EXEC, 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_CND_MISSP_EXEC, 0x8C, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_IND_EXEC, 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_IND_MISSP_EXEC, 0x8E, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_RET_EXEC, 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_RET_MISSP_EXEC, 0x90, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_RET_BAC_MISSP_EXEC, 0x91, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_CALL_EXEC, 0x92, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_CALL_MISSP_EXEC, 0x93, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(BR_IND_CALL_EXEC, 0x94, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_SIMD_INSTR_RETIRED, 0xCE, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_SYNCH_UOPS, 0xD3, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_ESP_UOPS, 0xD7, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_FUSED_UOPS_RET, 0xDA, P6F_ALL_CTRS | P6F_PM, 0x03),
P6_EVDESCR(EMON_UNFUSION, 0xDB, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_PREF_RQSTS_UP, 0xF0, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_PREF_RQSTS_DN, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x00),
P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x03),
P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03)
#undef P6_EVDESCR
};
#define P6_NEVENTS (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1)
static const struct p6_event_descr *
p6_find_event(enum pmc_event ev)
{
int n;
for (n = 0; n < P6_NEVENTS; n++)
if (p6_events[n].pm_event == ev)
break;
if (n == P6_NEVENTS)
return NULL;
return &p6_events[n];
}
/*
* Per-CPU data structure for P6 class CPUs
*
* [common stuff]
* [flags for maintaining PMC start/stop state]
* [3 struct pmc_hw pointers]
* [3 struct pmc_hw structures]
*/
struct p6_cpu {
struct pmc_hw pc_p6pmcs[P6_NPMCS];
uint32_t pc_state;
};
static struct p6_cpu **p6_pcpu;
/*
* If CTR1 is active, we need to keep the 'EN' bit if CTR0 set,
* with the rest of CTR0 being zero'ed out.
*/
#define P6_SYNC_CTR_STATE(PC) do { \
uint32_t _config, _enable; \
_enable = 0; \
if ((PC)->pc_state & 0x02) \
_enable |= P6_EVSEL_EN; \
if ((PC)->pc_state & 0x01) \
_config = rdmsr(P6_MSR_EVSEL0) | \
P6_EVSEL_EN; \
else \
_config = 0; \
wrmsr(P6_MSR_EVSEL0, _config | _enable); \
} while (0)
#define P6_MARK_STARTED(PC,RI) do { \
(PC)->pc_state |= (1 << ((RI)-1)); \
} while (0)
#define P6_MARK_STOPPED(PC,RI) do { \
(PC)->pc_state &= ~(1<< ((RI)-1)); \
} while (0)
static int
p6_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, n;
struct p6_cpu *p6c;
struct pmc_cpu *pc;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] bad cpu %d", __LINE__, cpu));
- PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,0,"p6-init cpu=%d", cpu);
p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu));
phw = p6c->pc_p6pmcs;
p6_pcpu[cpu] = p6c;
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
for (n = 0; n < P6_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
return (0);
}
static int
p6_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int first_ri, n;
struct p6_cpu *p6c;
struct pmc_cpu *pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] bad cpu %d", __LINE__, cpu));
- PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
+ PMCDBG1(MDP,INI,0,"p6-cleanup cpu=%d", cpu);
p6c = p6_pcpu[cpu];
p6_pcpu[cpu] = NULL;
KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__));
free(p6c, M_PMC);
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri;
pc = pmc_pcpu[cpu];
for (n = 0; n < P6_NPMCS; n++)
pc->pc_hwpmcs[n + first_ri] = NULL;
return (0);
}
static int
p6_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
struct p6pmc_descr *pd;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
pd = &p6_pmcdesc[ri];
KASSERT(pm,
("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
- PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
+ PMCDBG4(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
pd->pm_pmc_msr, *v);
return (0);
}
static int
p6_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
struct p6pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
pd = &p6_pmcdesc[ri];
KASSERT(pm,
("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
- PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
+ PMCDBG4(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
pd->pm_pmc_msr, v);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK);
return (0);
}
static int
p6_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__,
cpu));
p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm;
return (0);
}
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
p6_get_config(int cpu, int ri, struct pmc **ppm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
*ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc;
return (0);
}
/*
* A pmc may be allocated to a given row index if:
* - the event is valid for this CPU
* - the event is valid for this counter index
*/
static int
p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
uint32_t allowed_unitmask, caps, config, unitmask;
const struct p6pmc_descr *pd;
const struct p6_event_descr *pevent;
enum pmc_event ev;
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index value %d", __LINE__, ri));
pd = &p6_pmcdesc[ri];
- PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
+ PMCDBG4(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x "
"reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
pm->pm_caps);
/* check class */
if (pd->pm_descr.pd_class != a->pm_class)
return (EINVAL);
/* check requested capabilities */
caps = a->pm_caps;
if ((pd->pm_descr.pd_caps & caps) != caps)
return (EPERM);
ev = pm->pm_event;
if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST)
return (EINVAL);
if ((pevent = p6_find_event(ev)) == NULL)
return (ESRCH);
if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) ||
!P6_EVENT_VALID_FOR_CTR(pevent, (ri-1)))
return (EINVAL);
/* For certain events, Pentium M differs from the stock P6 */
allowed_unitmask = 0;
if (p6_cputype == PMC_CPU_INTEL_PM) {
if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN ||
ev == PMC_EV_P6_L2_LINES_OUT)
allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F);
else if (ev == PMC_EV_P6_L2_M_LINES_OUTM)
allowed_unitmask = P6_EVSEL_TO_UMASK(0x30);
} else
allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask);
unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK;
if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
return (EINVAL);
if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */
unitmask = P6_EVSEL_TO_UMASK(0x0F);
config = 0;
config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel);
if (unitmask & (caps & PMC_CAP_QUALIFIER))
config |= unitmask;
if (caps & PMC_CAP_THRESHOLD)
config |= a->pm_md.pm_ppro.pm_ppro_config &
P6_EVSEL_CMASK_MASK;
/* set at least one of the 'usr' or 'os' caps */
if (caps & PMC_CAP_USER)
config |= P6_EVSEL_USR;
if (caps & PMC_CAP_SYSTEM)
config |= P6_EVSEL_OS;
if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
config |= (P6_EVSEL_USR|P6_EVSEL_OS);
if (caps & PMC_CAP_EDGE)
config |= P6_EVSEL_E;
if (caps & PMC_CAP_INVERT)
config |= P6_EVSEL_INV;
if (caps & PMC_CAP_INTERRUPT)
config |= P6_EVSEL_INT;
pm->pm_md.pm_ppro.pm_ppro_evsel = config;
- PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config);
+ PMCDBG1(MDP,ALL,2, "p6-allocate config=0x%x", config);
return (0);
}
static int
p6_release_pmc(int cpu, int ri, struct pmc *pm)
{
(void) pm;
- PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL,
("[p6,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
p6_start_pmc(int cpu, int ri)
{
uint32_t config;
struct pmc *pm;
struct p6_cpu *pc;
const struct p6pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row-index %d", __LINE__, ri));
pc = p6_pcpu[cpu];
pm = pc->pc_p6pmcs[ri].phw_pmc;
pd = &p6_pmcdesc[ri];
KASSERT(pm,
("[p6,%d] starting cpu%d,ri%d with no pmc configured",
__LINE__, cpu, ri));
- PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri);
config = pm->pm_md.pm_ppro.pm_ppro_evsel;
- PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
+ PMCDBG4(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x",
cpu, ri, pd->pm_evsel_msr, config);
P6_MARK_STARTED(pc, ri);
wrmsr(pd->pm_evsel_msr, config);
P6_SYNC_CTR_STATE(pc);
return (0);
}
static int
p6_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct p6_cpu *pc;
struct p6pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] illegal row index %d", __LINE__, ri));
pc = p6_pcpu[cpu];
pm = pc->pc_p6pmcs[ri].phw_pmc;
pd = &p6_pmcdesc[ri];
KASSERT(pm,
("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
- PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri);
wrmsr(pd->pm_evsel_msr, 0); /* stop hw */
P6_MARK_STOPPED(pc, ri); /* update software state */
P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */
- PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri);
return (0);
}
static int
p6_intr(int cpu, struct trapframe *tf)
{
int error, retval, ri;
uint32_t perf0cfg;
struct pmc *pm;
struct p6_cpu *pc;
pmc_value_t v;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] CPU %d out of range", __LINE__, cpu));
retval = 0;
pc = p6_pcpu[cpu];
/* stop both PMCs */
perf0cfg = rdmsr(P6_MSR_EVSEL0);
wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN);
for (ri = 0; ri < P6_NPMCS; ri++) {
if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!P6_PMC_HAS_OVERFLOWED(ri))
continue;
retval = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
P6_MARK_STOPPED(pc,ri);
/* reload sampling count */
v = pm->pm_sc.pm_reloadcount;
wrmsr(P6_MSR_PERFCTR0 + ri,
P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
}
/*
* On P6 processors, the LAPIC needs to have its PMC interrupt
* unmasked after a PMC interrupt.
*/
if (retval)
lapic_reenable_pmc();
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
/* restart counters that can be restarted */
P6_SYNC_CTR_STATE(pc);
return (retval);
}
static int
p6_describe(int cpu, int ri, struct pmc_info *pi,
struct pmc **ppmc)
{
int error;
size_t copied;
struct pmc_hw *phw;
struct p6pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p6,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d] row-index %d out of range", __LINE__, ri));
phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
pd = &p6_pmcdesc[ri];
KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri],
("[p6,%d] phw mismatch", __LINE__));
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
p6_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d ri %d out of range", __LINE__, ri));
*msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
return (0);
}
int
pmc_p6_initialize(struct pmc_mdep *md, int ncpus)
{
struct pmc_classdep *pcd;
KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL,
("[p6,%d] Initializing non-intel processor", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "p6-initialize");
+ PMCDBG0(MDP,INI,1, "p6-initialize");
/* Allocate space for pointers to per-cpu descriptors. */
p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC,
M_ZERO|M_WAITOK);
/* Fill in the class dependent descriptor. */
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6];
switch (md->pmd_cputype) {
/*
* P6 Family Processors
*/
case PMC_CPU_INTEL_P6:
case PMC_CPU_INTEL_CL:
case PMC_CPU_INTEL_PII:
case PMC_CPU_INTEL_PIII:
case PMC_CPU_INTEL_PM:
p6_cputype = md->pmd_cputype;
pcd->pcd_caps = P6_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_P6;
pcd->pcd_num = P6_NPMCS;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = 40;
pcd->pcd_allocate_pmc = p6_allocate_pmc;
pcd->pcd_config_pmc = p6_config_pmc;
pcd->pcd_describe = p6_describe;
pcd->pcd_get_config = p6_get_config;
pcd->pcd_get_msr = p6_get_msr;
pcd->pcd_pcpu_fini = p6_pcpu_fini;
pcd->pcd_pcpu_init = p6_pcpu_init;
pcd->pcd_read_pmc = p6_read_pmc;
pcd->pcd_release_pmc = p6_release_pmc;
pcd->pcd_start_pmc = p6_start_pmc;
pcd->pcd_stop_pmc = p6_stop_pmc;
pcd->pcd_write_pmc = p6_write_pmc;
md->pmd_pcpu_fini = NULL;
md->pmd_pcpu_init = NULL;
md->pmd_intr = p6_intr;
md->pmd_npmc += P6_NPMCS;
break;
default:
KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__));
return ENOSYS;
}
return (0);
}
void
pmc_p6_finalize(struct pmc_mdep *md)
{
#if defined(INVARIANTS)
int i, ncpus;
#endif
KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__));
#if defined(INVARIANTS)
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d",
__LINE__, i));
#endif
free(p6_pcpu, M_PMC);
p6_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_soft.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_soft.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_soft.c (revision 282658)
@@ -1,498 +1,498 @@
/*-
* Copyright (c) 2012 Fabien Thomas
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <sys/mutex.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include "hwpmc_soft.h"
/*
* Software PMC support.
*/
#define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
PMC_CAP_USER | PMC_CAP_SYSTEM)
struct soft_descr {
struct pmc_descr pm_descr; /* "base class" */
};
static struct soft_descr soft_pmcdesc[SOFT_NPMCS] =
{
#define SOFT_PMCDESCR(N) \
{ \
.pm_descr = \
{ \
.pd_name = #N, \
.pd_class = PMC_CLASS_SOFT, \
.pd_caps = SOFT_CAPS, \
.pd_width = 64 \
}, \
}
SOFT_PMCDESCR(SOFT0),
SOFT_PMCDESCR(SOFT1),
SOFT_PMCDESCR(SOFT2),
SOFT_PMCDESCR(SOFT3),
SOFT_PMCDESCR(SOFT4),
SOFT_PMCDESCR(SOFT5),
SOFT_PMCDESCR(SOFT6),
SOFT_PMCDESCR(SOFT7),
SOFT_PMCDESCR(SOFT8),
SOFT_PMCDESCR(SOFT9),
SOFT_PMCDESCR(SOFT10),
SOFT_PMCDESCR(SOFT11),
SOFT_PMCDESCR(SOFT12),
SOFT_PMCDESCR(SOFT13),
SOFT_PMCDESCR(SOFT14),
SOFT_PMCDESCR(SOFT15)
};
/*
* Per-CPU data structure.
*/
struct soft_cpu {
struct pmc_hw soft_hw[SOFT_NPMCS];
pmc_value_t soft_values[SOFT_NPMCS];
};
static struct soft_cpu **soft_pcpu;
static int
soft_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
struct pmc_soft *ps;
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
if (a->pm_class != PMC_CLASS_SOFT)
return (EINVAL);
if ((pm->pm_caps & SOFT_CAPS) == 0)
return (EINVAL);
if ((pm->pm_caps & ~SOFT_CAPS) != 0)
return (EPERM);
ev = pm->pm_event;
if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST)
return (EINVAL);
/* Check if event is registered. */
ps = pmc_soft_ev_acquire(ev);
if (ps == NULL)
return (EINVAL);
pmc_soft_ev_release(ps);
/* Module unload is protected by pmc SX lock. */
if (ps->ps_alloc != NULL)
ps->ps_alloc();
return (0);
}
static int
soft_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
pm, phw->phw_pmc));
phw->phw_pmc = pm;
return (0);
}
static int
soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
size_t copied;
const struct soft_descr *pd;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
pd = &soft_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
soft_get_config(int cpu, int ri, struct pmc **ppm)
{
(void) ri;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
*ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
return (0);
}
static int
soft_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int ri;
struct pmc_cpu *pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__));
free(soft_pcpu[cpu], M_PMC);
soft_pcpu[cpu] = NULL;
ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] ri=%d", __LINE__, ri));
pc = pmc_pcpu[cpu];
pc->pc_hwpmcs[ri] = NULL;
return (0);
}
static int
soft_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, n;
struct pmc_cpu *pc;
struct soft_cpu *soft_pc;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__));
KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu",
__LINE__));
soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu));
soft_pcpu[cpu] = soft_pc;
phw = soft_pc->soft_hw;
first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
for (n = 0; n < SOFT_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
return (0);
}
static int
soft_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
const struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
pm = phw->phw_pmc;
KASSERT(pm != NULL,
("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri));
- PMCDBG(MDP,REA,1,"soft-read id=%d", ri);
+ PMCDBG1(MDP,REA,1,"soft-read id=%d", ri);
*v = soft_pcpu[cpu]->soft_values[ri];
return (0);
}
static int
soft_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
const struct soft_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
pd = &soft_pmcdesc[ri];
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
- PMCDBG(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v);
soft_pcpu[cpu]->soft_values[ri] = v;
return (0);
}
static int
soft_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
enum pmc_event ev;
struct pmc_soft *ps;
(void) pmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
KASSERT(phw->phw_pmc == NULL,
("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
ev = pmc->pm_event;
/* Check if event is registered. */
ps = pmc_soft_ev_acquire(ev);
KASSERT(ps != NULL,
("[soft,%d] unregistered event %d", __LINE__, ev));
pmc_soft_ev_release(ps);
/* Module unload is protected by pmc SX lock. */
if (ps->ps_release != NULL)
ps->ps_release();
return (0);
}
static int
soft_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct soft_cpu *pc;
struct pmc_soft *ps;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pc = soft_pcpu[cpu];
pm = pc->soft_hw[ri].phw_pmc;
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
ps = pmc_soft_ev_acquire(pm->pm_event);
if (ps == NULL)
return (EINVAL);
atomic_add_int(&ps->ps_running, 1);
pmc_soft_ev_release(ps);
return (0);
}
static int
soft_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct soft_cpu *pc;
struct pmc_soft *ps;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pc = soft_pcpu[cpu];
pm = pc->soft_hw[ri].phw_pmc;
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
ps = pmc_soft_ev_acquire(pm->pm_event);
/* event unregistered ? */
if (ps != NULL) {
atomic_subtract_int(&ps->ps_running, 1);
pmc_soft_ev_release(ps);
}
return (0);
}
int
pmc_soft_intr(struct pmckern_soft *ks)
{
struct pmc *pm;
struct soft_cpu *pc;
int ri, processed, error, user_mode;
KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(),
("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu));
processed = 0;
pc = soft_pcpu[ks->pm_cpu];
for (ri = 0; ri < SOFT_NPMCS; ri++) {
pm = pc->soft_hw[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
pm->pm_event != ks->pm_ev) {
continue;
}
processed = 1;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
if ((pc->soft_values[ri]--) <= 0)
pc->soft_values[ri] += pm->pm_sc.pm_reloadcount;
else
continue;
user_mode = TRAPF_USERMODE(ks->pm_tf);
error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm,
ks->pm_tf, user_mode);
if (error) {
soft_stop_pmc(ks->pm_cpu, ri);
continue;
}
if (user_mode) {
/* If in user mode setup AST to process
* callchain out of interrupt context.
*/
curthread->td_flags |= TDF_ASTPENDING;
}
} else
pc->soft_values[ri]++;
}
atomic_add_int(processed ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (processed);
}
void
pmc_soft_initialize(struct pmc_mdep *md)
{
struct pmc_classdep *pcd;
/* Add SOFT PMCs. */
soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC,
M_ZERO|M_WAITOK);
pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT];
pcd->pcd_caps = SOFT_CAPS;
pcd->pcd_class = PMC_CLASS_SOFT;
pcd->pcd_num = SOFT_NPMCS;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = 64;
pcd->pcd_allocate_pmc = soft_allocate_pmc;
pcd->pcd_config_pmc = soft_config_pmc;
pcd->pcd_describe = soft_describe;
pcd->pcd_get_config = soft_get_config;
pcd->pcd_get_msr = NULL;
pcd->pcd_pcpu_init = soft_pcpu_init;
pcd->pcd_pcpu_fini = soft_pcpu_fini;
pcd->pcd_read_pmc = soft_read_pmc;
pcd->pcd_write_pmc = soft_write_pmc;
pcd->pcd_release_pmc = soft_release_pmc;
pcd->pcd_start_pmc = soft_start_pmc;
pcd->pcd_stop_pmc = soft_stop_pmc;
md->pmd_npmc += SOFT_NPMCS;
}
void
pmc_soft_finalize(struct pmc_mdep *md)
{
#ifdef INVARIANTS
int i, ncpus;
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d",
__LINE__, i));
KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class ==
PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__));
#endif
free(soft_pcpu, M_PMC);
soft_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_tsc.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_tsc.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_tsc.c (revision 282658)
@@ -1,385 +1,385 @@
/*-
* Copyright (c) 2008 Joseph Koshy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/specialreg.h>
/*
* TSC support.
*/
#define TSC_CAPS PMC_CAP_READ
struct tsc_descr {
struct pmc_descr pm_descr; /* "base class" */
};
static struct tsc_descr tsc_pmcdesc[TSC_NPMCS] =
{
{
.pm_descr =
{
.pd_name = "TSC",
.pd_class = PMC_CLASS_TSC,
.pd_caps = TSC_CAPS,
.pd_width = 64
}
}
};
/*
* Per-CPU data structure for TSCs.
*/
struct tsc_cpu {
struct pmc_hw tc_hw;
};
static struct tsc_cpu **tsc_pcpu;
static int
tsc_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < TSC_NPMCS,
("[tsc,%d] illegal row index %d", __LINE__, ri));
if (a->pm_class != PMC_CLASS_TSC)
return (EINVAL);
if ((pm->pm_caps & TSC_CAPS) == 0)
return (EINVAL);
if ((pm->pm_caps & ~TSC_CAPS) != 0)
return (EPERM);
if (a->pm_ev != PMC_EV_TSC_TSC ||
a->pm_mode != PMC_MODE_SC)
return (EINVAL);
return (0);
}
static int
tsc_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
phw = &tsc_pcpu[cpu]->tc_hw;
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[tsc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
pm, phw->phw_pmc));
phw->phw_pmc = pm;
return (0);
}
static int
tsc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
size_t copied;
const struct tsc_descr *pd;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
phw = &tsc_pcpu[cpu]->tc_hw;
pd = &tsc_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
tsc_get_config(int cpu, int ri, struct pmc **ppm)
{
(void) ri;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
*ppm = tsc_pcpu[cpu]->tc_hw.phw_pmc;
return (0);
}
static int
tsc_get_msr(int ri, uint32_t *msr)
{
(void) ri;
KASSERT(ri >= 0 && ri < TSC_NPMCS,
("[tsc,%d] ri %d out of range", __LINE__, ri));
*msr = MSR_TSC;
return (0);
}
static int
tsc_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int ri;
struct pmc_cpu *pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(tsc_pcpu[cpu] != NULL, ("[tsc,%d] null pcpu", __LINE__));
free(tsc_pcpu[cpu], M_PMC);
tsc_pcpu[cpu] = NULL;
ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri;
pc = pmc_pcpu[cpu];
pc->pc_hwpmcs[ri] = NULL;
return (0);
}
static int
tsc_pcpu_init(struct pmc_mdep *md, int cpu)
{
int ri;
struct pmc_cpu *pc;
struct tsc_cpu *tsc_pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(tsc_pcpu, ("[tsc,%d] null pcpu", __LINE__));
KASSERT(tsc_pcpu[cpu] == NULL, ("[tsc,%d] non-null per-cpu",
__LINE__));
tsc_pc = malloc(sizeof(struct tsc_cpu), M_PMC, M_WAITOK|M_ZERO);
tsc_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) |
PMC_PHW_FLAG_IS_SHAREABLE;
tsc_pcpu[cpu] = tsc_pc;
ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri;
KASSERT(pmc_pcpu, ("[tsc,%d] null generic pcpu", __LINE__));
pc = pmc_pcpu[cpu];
KASSERT(pc, ("[tsc,%d] null generic per-cpu", __LINE__));
pc->pc_hwpmcs[ri] = &tsc_pc->tc_hw;
return (0);
}
static int
tsc_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
enum pmc_mode mode;
const struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal ri %d", __LINE__, ri));
phw = &tsc_pcpu[cpu]->tc_hw;
pm = phw->phw_pmc;
KASSERT(pm != NULL,
("[tsc,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri));
mode = PMC_TO_MODE(pm);
KASSERT(mode == PMC_MODE_SC,
("[tsc,%d] illegal pmc mode %d", __LINE__, mode));
- PMCDBG(MDP,REA,1,"tsc-read id=%d", ri);
+ PMCDBG1(MDP,REA,1,"tsc-read id=%d", ri);
*v = rdtsc();
return (0);
}
static int
tsc_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
(void) pmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0,
("[tsc,%d] illegal row-index %d", __LINE__, ri));
phw = &tsc_pcpu[cpu]->tc_hw;
KASSERT(phw->phw_pmc == NULL,
("[tsc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
/*
* Nothing to do.
*/
return (0);
}
static int
tsc_start_pmc(int cpu, int ri)
{
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
return (0); /* TSCs are always running. */
}
static int
tsc_stop_pmc(int cpu, int ri)
{
(void) cpu; (void) ri;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
return (0); /* Cannot actually stop a TSC. */
}
static int
tsc_write_pmc(int cpu, int ri, pmc_value_t v)
{
(void) cpu; (void) ri; (void) v;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[tsc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri));
/*
* The TSCs are used as timecounters by the kernel, so even
* though some i386 CPUs support writeable TSCs, we don't
* support writing changing TSC values through the HWPMC API.
*/
return (0);
}
int
pmc_tsc_initialize(struct pmc_mdep *md, int maxcpu)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[tsc,%d] md is NULL", __LINE__));
KASSERT(md->pmd_nclass >= 1, ("[tsc,%d] dubious md->nclass %d",
__LINE__, md->pmd_nclass));
tsc_pcpu = malloc(sizeof(struct tsc_cpu *) * maxcpu, M_PMC,
M_ZERO|M_WAITOK);
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC];
pcd->pcd_caps = PMC_CAP_READ;
pcd->pcd_class = PMC_CLASS_TSC;
pcd->pcd_num = TSC_NPMCS;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = 64;
pcd->pcd_allocate_pmc = tsc_allocate_pmc;
pcd->pcd_config_pmc = tsc_config_pmc;
pcd->pcd_describe = tsc_describe;
pcd->pcd_get_config = tsc_get_config;
pcd->pcd_get_msr = tsc_get_msr;
pcd->pcd_pcpu_init = tsc_pcpu_init;
pcd->pcd_pcpu_fini = tsc_pcpu_fini;
pcd->pcd_read_pmc = tsc_read_pmc;
pcd->pcd_release_pmc = tsc_release_pmc;
pcd->pcd_start_pmc = tsc_start_pmc;
pcd->pcd_stop_pmc = tsc_stop_pmc;
pcd->pcd_write_pmc = tsc_write_pmc;
md->pmd_npmc += TSC_NPMCS;
return (0);
}
void
pmc_tsc_finalize(struct pmc_mdep *md)
{
#ifdef INVARIANTS
int i, ncpus;
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(tsc_pcpu[i] == NULL, ("[tsc,%d] non-null pcpu cpu %d",
__LINE__, i));
KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_class ==
PMC_CLASS_TSC, ("[tsc,%d] class mismatch", __LINE__));
#else
(void) md;
#endif
free(tsc_pcpu, M_PMC);
tsc_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_uncore.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_uncore.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_uncore.c (revision 282658)
@@ -1,1232 +1,1232 @@
/*-
* Copyright (c) 2010 Fabien Thomas
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Intel Uncore PMCs.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <machine/intr_machdep.h>
#if (__FreeBSD_version >= 1100000)
#include <x86/apicvar.h>
#else
#include <machine/apicvar.h>
#endif
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/specialreg.h>
#define UCF_PMC_CAPS \
(PMC_CAP_READ | PMC_CAP_WRITE)
#define UCP_PMC_CAPS \
(PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
#define SELECTSEL(x) \
(((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \
UCP_CB0_EVSEL0 : UCP_EVSEL0)
#define SELECTOFF(x) \
(((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \
UCF_OFFSET_SB : UCF_OFFSET)
static enum pmc_cputype uncore_cputype;
struct uncore_cpu {
volatile uint32_t pc_resync;
volatile uint32_t pc_ucfctrl; /* Fixed function control. */
volatile uint64_t pc_globalctrl; /* Global control register. */
struct pmc_hw pc_uncorepmcs[];
};
static struct uncore_cpu **uncore_pcpu;
static uint64_t uncore_pmcmask;
static int uncore_ucf_ri; /* relative index of fixed counters */
static int uncore_ucf_width;
static int uncore_ucf_npmc;
static int uncore_ucp_width;
static int uncore_ucp_npmc;
static int
uncore_pcpu_noop(struct pmc_mdep *md, int cpu)
{
(void) md;
(void) cpu;
return (0);
}
static int
uncore_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct uncore_cpu *cc;
struct pmc_hw *phw;
int uncore_ri, n, npmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[ucf,%d] insane cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"uncore-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu);
uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;
cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw),
M_PMC, M_WAITOK | M_ZERO);
uncore_pcpu[cpu] = cc;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL && cc != NULL,
("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) |
PMC_PHW_INDEX_TO_STATE(n + uncore_ri);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + uncore_ri] = phw;
}
return (0);
}
static int
uncore_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int uncore_ri, n, npmc;
struct pmc_cpu *pc;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] insane cpu number (%d)", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu);
if ((cc = uncore_pcpu[cpu]) == NULL)
return (0);
uncore_pcpu[cpu] = NULL;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__,
cpu));
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;
uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;
for (n = 0; n < npmc; n++)
wrmsr(SELECTSEL(uncore_cputype) + n, 0);
wrmsr(UCF_CTRL, 0);
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;
for (n = 0; n < npmc; n++)
pc->pc_hwpmcs[n + uncore_ri] = NULL;
free(cc, M_PMC);
return (0);
}
/*
* Fixed function counters.
*/
static pmc_value_t
ucf_perfctr_value_to_reload_count(pmc_value_t v)
{
v &= (1ULL << uncore_ucf_width) - 1;
return (1ULL << uncore_ucf_width) - v;
}
static pmc_value_t
ucf_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << uncore_ucf_width) - rlc;
}
static int
ucf_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
uint32_t caps, flags;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
- PMCDBG(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
+ PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
if (ri < 0 || ri > uncore_ucf_npmc)
return (EINVAL);
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_UCF ||
(caps & UCF_PMC_CAPS) != caps)
return (EINVAL);
ev = pm->pm_event;
if (ev < PMC_EV_UCF_FIRST || ev > PMC_EV_UCF_LAST)
return (EINVAL);
flags = UCF_EN;
pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4));
- PMCDBG(MDP,ALL,2, "ucf-allocate config=0x%jx",
+ PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx",
(uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl);
return (0);
}
static int
ucf_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__,
cpu));
uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm;
return (0);
}
static int
ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char ucf_name[PMC_NAME_MAX];
phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri];
(void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri);
if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return (error);
pi->pm_class = PMC_CLASS_UCF;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
ucf_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
return (0);
}
static int
ucf_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
ri, ri + uncore_ucf_ri));
tmp = rdmsr(UCF_CTR0 + ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = ucf_perfctr_value_to_reload_count(tmp);
else
*v = tmp;
- PMCDBG(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v);
+ PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v);
return (0);
}
static int
ucf_release_pmc(int cpu, int ri, struct pmc *pmc)
{
- PMCDBG(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
+ PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL,
("[uncore,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
ucf_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct uncore_cpu *ucfc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl;
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
- PMCDBG(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
+ PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucf_stop_pmc(int cpu, int ri)
{
uint32_t fc;
struct uncore_cpu *ucfc;
- PMCDBG(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
fc = (UCF_MASK << (ri * 4));
ucfc->pc_ucfctrl &= ~fc;
- PMCDBG(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);
+ PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
- PMCDBG(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
+ PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucf_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct uncore_cpu *cc;
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = ucf_reload_count_to_perfctr_value(v);
wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */
wrmsr(UCF_CTR0 + ri, v);
wrmsr(UCF_CTRL, cc->pc_ucfctrl);
- PMCDBG(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ",
+ PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ",
cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL));
return (0);
}
static void
ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "ucf-initialize");
+ PMCDBG0(MDP,INI,1, "ucf-initialize");
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF];
pcd->pcd_caps = UCF_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_UCF;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = ucf_allocate_pmc;
pcd->pcd_config_pmc = ucf_config_pmc;
pcd->pcd_describe = ucf_describe;
pcd->pcd_get_config = ucf_get_config;
pcd->pcd_get_msr = NULL;
pcd->pcd_pcpu_fini = uncore_pcpu_noop;
pcd->pcd_pcpu_init = uncore_pcpu_noop;
pcd->pcd_read_pmc = ucf_read_pmc;
pcd->pcd_release_pmc = ucf_release_pmc;
pcd->pcd_start_pmc = ucf_start_pmc;
pcd->pcd_stop_pmc = ucf_stop_pmc;
pcd->pcd_write_pmc = ucf_write_pmc;
md->pmd_npmc += npmc;
}
/*
* Intel programmable PMCs.
*/
/*
* Event descriptor tables.
*
* For each event id, we track:
*
* 1. The CPUs that the event is valid for.
*
* 2. If the event uses a fixed UMASK, the value of the umask field.
* If the event doesn't use a fixed UMASK, a mask of legal bits
* to check against.
*/
struct ucp_event_descr {
enum pmc_event ucp_ev;
unsigned char ucp_evcode;
unsigned char ucp_umask;
unsigned char ucp_flags;
};
#define UCP_F_I7 (1 << 0) /* CPU: Core i7 */
#define UCP_F_WM (1 << 1) /* CPU: Westmere */
#define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */
#define UCP_F_HW (1 << 3) /* CPU: Haswell */
#define UCP_F_FM (1 << 4) /* Fixed mask */
#define UCP_F_ALLCPUS \
(UCP_F_I7 | UCP_F_WM)
#define UCP_F_CMASK 0xFF000000
static struct ucp_event_descr ucp_events[] = {
#undef UCPDESCR
#define UCPDESCR(N,EV,UM,FLAGS) { \
.ucp_ev = PMC_EV_UCP_EVENT_##N, \
.ucp_evcode = (EV), \
.ucp_umask = (UM), \
.ucp_flags = (FLAGS) \
}
UCPDESCR(00H_01H, 0x00, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(00H_02H, 0x00, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(00H_04H, 0x00, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(01H_01H, 0x01, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(01H_02H, 0x01, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(01H_04H, 0x01, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(02H_01H, 0x02, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_01H, 0x03, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_02H, 0x03, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_04H, 0x03, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_08H, 0x03, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_10H, 0x03, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_20H, 0x03, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(03H_40H, 0x03, 0x40, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(04H_01H, 0x04, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(04H_02H, 0x04, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(04H_04H, 0x04, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(04H_08H, 0x04, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(04H_10H, 0x04, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(05H_01H, 0x05, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(05H_02H, 0x05, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(05H_04H, 0x05, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_01H, 0x06, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_02H, 0x06, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_04H, 0x06, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_08H, 0x06, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_10H, 0x06, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(06H_20H, 0x06, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_01H, 0x07, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_02H, 0x07, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_04H, 0x07, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_08H, 0x07, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_10H, 0x07, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_20H, 0x07, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(07H_24H, 0x07, 0x24, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(08H_01H, 0x08, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(08H_02H, 0x08, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(08H_04H, 0x08, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(08H_03H, 0x08, 0x03, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(09H_01H, 0x09, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(09H_02H, 0x09, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(09H_04H, 0x09, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(09H_03H, 0x09, 0x03, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0AH_01H, 0x0A, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0AH_02H, 0x0A, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0AH_04H, 0x0A, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0AH_08H, 0x0A, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0AH_0FH, 0x0A, 0x0F, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_01H, 0x0B, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_02H, 0x0B, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_04H, 0x0B, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_08H, 0x0B, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_10H, 0x0B, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0BH_1FH, 0x0B, 0x1F, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(0CH_01H, 0x0C, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_02H, 0x0C, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_04H_E, 0x0C, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_04H_F, 0x0C, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_04H_M, 0x0C, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_04H_S, 0x0C, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_08H_E, 0x0C, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_08H_F, 0x0C, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_08H_M, 0x0C, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(0CH_08H_S, 0x0C, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(20H_01H, 0x20, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(20H_02H, 0x20, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(20H_04H, 0x20, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(20H_08H, 0x20, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(20H_10H, 0x20, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(20H_20H, 0x20, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(21H_01H, 0x21, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(21H_02H, 0x21, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(21H_04H, 0x21, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(22H_01H, 0x22, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM |
UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_02H, 0x22, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM |
UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_04H, 0x22, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM |
UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_08H, 0x22, 0x08, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_10H, 0x22, 0x10, UCP_F_FM | UCP_F_HW),
UCPDESCR(22H_20H, 0x22, 0x20, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_40H, 0x22, 0x40, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(22H_80H, 0x22, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(23H_01H, 0x23, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(23H_02H, 0x23, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(23H_04H, 0x23, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(24H_02H, 0x24, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(24H_04H, 0x24, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(25H_01H, 0x25, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(25H_02H, 0x25, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(25H_04H, 0x25, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(26H_01H, 0x26, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(27H_01H, 0x27, 0x01, UCP_F_FM | UCP_F_I7),
UCPDESCR(27H_02H, 0x27, 0x02, UCP_F_FM | UCP_F_I7),
UCPDESCR(27H_04H, 0x27, 0x04, UCP_F_FM | UCP_F_I7),
UCPDESCR(27H_08H, 0x27, 0x08, UCP_F_FM | UCP_F_I7),
UCPDESCR(27H_10H, 0x27, 0x10, UCP_F_FM | UCP_F_I7),
UCPDESCR(27H_20H, 0x27, 0x20, UCP_F_FM | UCP_F_I7),
UCPDESCR(28H_01H, 0x28, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(28H_02H, 0x28, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(28H_04H, 0x28, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(28H_08H, 0x28, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(28H_10H, 0x28, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(28H_20H, 0x28, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_01H, 0x29, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_02H, 0x29, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_04H, 0x29, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_08H, 0x29, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_10H, 0x29, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(29H_20H, 0x29, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2AH_01H, 0x2A, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2AH_02H, 0x2A, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2AH_04H, 0x2A, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2AH_07H, 0x2A, 0x07, UCP_F_FM | UCP_F_WM),
UCPDESCR(2BH_01H, 0x2B, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2BH_02H, 0x2B, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2BH_04H, 0x2B, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2BH_07H, 0x2B, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2CH_01H, 0x2C, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2CH_02H, 0x2C, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2CH_04H, 0x2C, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2CH_07H, 0x2C, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2DH_01H, 0x2D, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2DH_02H, 0x2D, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2DH_04H, 0x2D, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2DH_07H, 0x2D, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2EH_01H, 0x2E, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2EH_02H, 0x2E, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2EH_04H, 0x2E, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2EH_07H, 0x2E, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_01H, 0x2F, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_02H, 0x2F, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_04H, 0x2F, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_07H, 0x2F, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_08H, 0x2F, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_10H, 0x2F, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_20H, 0x2F, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(2FH_38H, 0x2F, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(30H_01H, 0x30, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(30H_02H, 0x30, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(30H_04H, 0x30, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(30H_07H, 0x30, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(31H_01H, 0x31, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(31H_02H, 0x31, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(31H_04H, 0x31, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(31H_07H, 0x31, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(32H_01H, 0x32, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(32H_02H, 0x32, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(32H_04H, 0x32, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(32H_07H, 0x32, 0x07, UCP_F_FM | UCP_F_WM),
UCPDESCR(33H_01H, 0x33, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(33H_02H, 0x33, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(33H_04H, 0x33, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(33H_07H, 0x33, 0x07, UCP_F_FM | UCP_F_WM),
UCPDESCR(34H_01H, 0x34, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(34H_02H, 0x34, 0x02, UCP_F_FM | UCP_F_WM | UCP_F_SB),
UCPDESCR(34H_04H, 0x34, 0x04, UCP_F_FM | UCP_F_WM | UCP_F_SB),
UCPDESCR(34H_06H, 0x34, 0x06, UCP_F_FM | UCP_F_HW),
UCPDESCR(34H_08H, 0x34, 0x08, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(34H_10H, 0x34, 0x10, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(34H_20H, 0x34, 0x20, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(34H_40H, 0x34, 0x40, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(34H_80H, 0x34, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(35H_01H, 0x35, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(35H_02H, 0x35, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(35H_04H, 0x35, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(40H_01H, 0x40, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_02H, 0x40, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_04H, 0x40, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_08H, 0x40, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_10H, 0x40, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_20H, 0x40, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_07H, 0x40, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(40H_38H, 0x40, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_01H, 0x41, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_02H, 0x41, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_04H, 0x41, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_08H, 0x41, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_10H, 0x41, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_20H, 0x41, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_07H, 0x41, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(41H_38H, 0x41, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(42H_01H, 0x42, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(42H_02H, 0x42, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(42H_04H, 0x42, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(42H_08H, 0x42, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(43H_01H, 0x43, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(43H_02H, 0x43, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(60H_01H, 0x60, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(60H_02H, 0x60, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(60H_04H, 0x60, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(61H_01H, 0x61, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(61H_02H, 0x61, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(61H_04H, 0x61, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(62H_01H, 0x62, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(62H_02H, 0x62, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(62H_04H, 0x62, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_01H, 0x63, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_02H, 0x63, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_04H, 0x63, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_08H, 0x63, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_10H, 0x63, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(63H_20H, 0x63, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_01H, 0x64, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_02H, 0x64, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_04H, 0x64, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_08H, 0x64, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_10H, 0x64, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(64H_20H, 0x64, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(65H_01H, 0x65, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(65H_02H, 0x65, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(65H_04H, 0x65, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(66H_01H, 0x66, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(66H_02H, 0x66, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(66H_04H, 0x66, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM),
UCPDESCR(67H_01H, 0x67, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(80H_01H, 0x80, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(80H_02H, 0x80, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(80H_04H, 0x80, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(80H_08H, 0x80, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(81H_01H, 0x81, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(81H_02H, 0x81, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(81H_04H, 0x81, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(81H_08H, 0x81, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(81H_20H, 0x81, 0x20, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(81H_80H, 0x81, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW),
UCPDESCR(82H_01H, 0x82, 0x01, UCP_F_FM | UCP_F_WM),
UCPDESCR(83H_01H, 0x83, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(83H_02H, 0x83, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(83H_04H, 0x83, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(83H_08H, 0x83, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(84H_01H, 0x84, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB |
UCP_F_HW),
UCPDESCR(84H_02H, 0x84, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(84H_04H, 0x84, 0x04, UCP_F_FM | UCP_F_WM),
UCPDESCR(84H_08H, 0x84, 0x08, UCP_F_FM | UCP_F_WM),
UCPDESCR(85H_02H, 0x85, 0x02, UCP_F_FM | UCP_F_WM),
UCPDESCR(86H_01H, 0x86, 0x01, UCP_F_FM | UCP_F_WM)
};
static const int nucp_events = sizeof(ucp_events) / sizeof(ucp_events[0]);
static pmc_value_t
ucp_perfctr_value_to_reload_count(pmc_value_t v)
{
v &= (1ULL << uncore_ucp_width) - 1;
return (1ULL << uncore_ucp_width) - v;
}
static pmc_value_t
ucp_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << uncore_ucp_width) - rlc;
}
/*
* Counter specific event information for Sandybridge and Haswell
*/
static int
ucp_event_sb_hw_ok_on_counter(enum pmc_event pe, int ri)
{
uint32_t mask;
switch (pe) {
/*
* Events valid only on counter 0.
*/
case PMC_EV_UCP_EVENT_80H_01H:
case PMC_EV_UCP_EVENT_83H_01H:
mask = (1 << 0);
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
ucp_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int n;
enum pmc_event ev;
struct ucp_event_descr *ie;
uint32_t caps, config, cpuflag, evsel;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index value %d", __LINE__, ri));
/* check requested capabilities */
caps = a->pm_caps;
if ((UCP_PMC_CAPS & caps) != caps)
return (EPERM);
ev = pm->pm_event;
switch (uncore_cputype) {
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_SANDYBRIDGE:
if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
default:
break;
}
/*
* Look for an event descriptor with matching CPU and event id
* fields.
*/
switch (uncore_cputype) {
case PMC_CPU_INTEL_COREI7:
cpuflag = UCP_F_I7;
break;
case PMC_CPU_INTEL_HASWELL:
cpuflag = UCP_F_HW;
break;
case PMC_CPU_INTEL_SANDYBRIDGE:
cpuflag = UCP_F_SB;
break;
case PMC_CPU_INTEL_WESTMERE:
cpuflag = UCP_F_WM;
break;
default:
return (EINVAL);
}
for (n = 0, ie = ucp_events; n < nucp_events; n++, ie++)
if (ie->ucp_ev == ev && ie->ucp_flags & cpuflag)
break;
if (n == nucp_events)
return (EINVAL);
/*
* A matching event descriptor has been found, so start
* assembling the contents of the event select register.
*/
evsel = ie->ucp_evcode | UCP_EN;
config = a->pm_md.pm_ucp.pm_ucp_config & ~UCP_F_CMASK;
/*
* If the event uses a fixed umask value, reject any umask
* bits set by the user.
*/
if (ie->ucp_flags & UCP_F_FM) {
if (UCP_UMASK(config) != 0)
return (EINVAL);
evsel |= (ie->ucp_umask << 8);
} else
return (EINVAL);
if (caps & PMC_CAP_THRESHOLD)
evsel |= (a->pm_md.pm_ucp.pm_ucp_config & UCP_F_CMASK);
if (caps & PMC_CAP_EDGE)
evsel |= UCP_EDGE;
if (caps & PMC_CAP_INVERT)
evsel |= UCP_INV;
pm->pm_md.pm_ucp.pm_ucp_evsel = evsel;
return (0);
}
static int
ucp_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
- PMCDBG(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__,
cpu));
uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm;
return (0);
}
static int
ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char ucp_name[PMC_NAME_MAX];
phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri];
(void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri);
if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return (error);
pi->pm_class = PMC_CLASS_UCP;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
ucp_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc;
return (0);
}
static int
ucp_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = rdmsr(UCP_PMC0 + ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = ucp_perfctr_value_to_reload_count(tmp);
else
*v = tmp;
- PMCDBG(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
+ PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
ri, *v);
return (0);
}
static int
ucp_release_pmc(int cpu, int ri, struct pmc *pm)
{
(void) pm;
- PMCDBG(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri,
+ PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri,
pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc
== NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
ucp_start_pmc(int cpu, int ri)
{
struct pmc *pm;
uint32_t evsel;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] starting cpu%d,ri%d with no pmc configured",
__LINE__, cpu, ri));
- PMCDBG(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri);
evsel = pm->pm_md.pm_ucp.pm_ucp_evsel;
- PMCDBG(MDP,STA,2,
+ PMCDBG4(MDP,STA,2,
"ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel);
/* Event specific configuration. */
switch (pm->pm_event) {
case PMC_EV_UCP_EVENT_0CH_04H_E:
case PMC_EV_UCP_EVENT_0CH_08H_E:
wrmsr(MSR_GQ_SNOOP_MESF,0x2);
break;
case PMC_EV_UCP_EVENT_0CH_04H_F:
case PMC_EV_UCP_EVENT_0CH_08H_F:
wrmsr(MSR_GQ_SNOOP_MESF,0x8);
break;
case PMC_EV_UCP_EVENT_0CH_04H_M:
case PMC_EV_UCP_EVENT_0CH_08H_M:
wrmsr(MSR_GQ_SNOOP_MESF,0x1);
break;
case PMC_EV_UCP_EVENT_0CH_04H_S:
case PMC_EV_UCP_EVENT_0CH_08H_S:
wrmsr(MSR_GQ_SNOOP_MESF,0x4);
break;
default:
break;
}
wrmsr(SELECTSEL(uncore_cputype) + ri, evsel);
do {
cc->pc_resync = 0;
cc->pc_globalctrl |= (1ULL << ri);
wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
ucp_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
- PMCDBG(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri);
+ PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri);
/* stop hw. */
wrmsr(SELECTSEL(uncore_cputype) + ri, 0);
do {
cc->pc_resync = 0;
cc->pc_globalctrl &= ~(1ULL << ri);
wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
ucp_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
- PMCDBG(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
+ PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
UCP_PMC0 + ri, v);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = ucp_reload_count_to_perfctr_value(v);
/*
* Write the new value to the counter. The counter will be in
* a stopped state when the pcd_write() entry point is called.
*/
wrmsr(UCP_PMC0 + ri, v);
return (0);
}
static void
ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__));
- PMCDBG(MDP,INI,1, "%s", "ucp-initialize");
+ PMCDBG0(MDP,INI,1, "ucp-initialize");
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP];
pcd->pcd_caps = UCP_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_UCP;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = ucp_allocate_pmc;
pcd->pcd_config_pmc = ucp_config_pmc;
pcd->pcd_describe = ucp_describe;
pcd->pcd_get_config = ucp_get_config;
pcd->pcd_get_msr = NULL;
pcd->pcd_pcpu_fini = uncore_pcpu_fini;
pcd->pcd_pcpu_init = uncore_pcpu_init;
pcd->pcd_read_pmc = ucp_read_pmc;
pcd->pcd_release_pmc = ucp_release_pmc;
pcd->pcd_start_pmc = ucp_start_pmc;
pcd->pcd_stop_pmc = ucp_stop_pmc;
pcd->pcd_write_pmc = ucp_write_pmc;
md->pmd_npmc += npmc;
}
int
pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu)
{
uncore_cputype = md->pmd_cputype;
uncore_pmcmask = 0;
/*
* Initialize programmable counters.
*/
uncore_ucp_npmc = 8;
uncore_ucp_width = 48;
uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1);
ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width);
/*
* Initialize fixed function counters, if present.
*/
uncore_ucf_ri = uncore_ucp_npmc;
uncore_ucf_npmc = 1;
uncore_ucf_width = 48;
ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width);
uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype);
- PMCDBG(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask,
+ PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask,
uncore_ucf_ri);
uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC,
M_ZERO | M_WAITOK);
return (0);
}
void
pmc_uncore_finalize(struct pmc_mdep *md)
{
- PMCDBG(MDP,INI,1, "%s", "uncore-finalize");
+ PMCDBG0(MDP,INI,1, "uncore-finalize");
free(uncore_pcpu, M_PMC);
uncore_pcpu = NULL;
}
Index: head/sys/dev/hwpmc/hwpmc_xscale.c
===================================================================
--- head/sys/dev/hwpmc/hwpmc_xscale.c (revision 282657)
+++ head/sys/dev/hwpmc/hwpmc_xscale.c (revision 282658)
@@ -1,676 +1,676 @@
/*-
* Copyright (c) 2009 Rui Paulo <rpaulo@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <machine/pmc_mdep.h>
/*
* Support for the Intel XScale network processors
*
* XScale processors have up to now three generations.
*
* The first generation has two PMC; the event selection, interrupt config
* and overflow flag setup are done by writing to the PMNC register.
* It also has less monitoring events than the latter generations.
*
* The second and third generatiosn have four PMCs, one register for the event
* selection, one register for the interrupt config and one register for
* the overflow flags.
*/
static int xscale_npmcs;
static int xscale_gen; /* XScale Core generation */
struct xscale_event_code_map {
enum pmc_event pe_ev;
uint8_t pe_code;
};
const struct xscale_event_code_map xscale_event_codes[] = {
/* 1st and 2nd Generation XScale cores */
{ PMC_EV_XSCALE_IC_FETCH, 0x00 },
{ PMC_EV_XSCALE_IC_MISS, 0x01 },
{ PMC_EV_XSCALE_DATA_DEPENDENCY_STALLED,0x02 },
{ PMC_EV_XSCALE_ITLB_MISS, 0x03 },
{ PMC_EV_XSCALE_DTLB_MISS, 0x04 },
{ PMC_EV_XSCALE_BRANCH_RETIRED, 0x05 },
{ PMC_EV_XSCALE_BRANCH_MISPRED, 0x06 },
{ PMC_EV_XSCALE_INSTR_RETIRED, 0x07 },
{ PMC_EV_XSCALE_DC_FULL_CYCLE, 0x08 },
{ PMC_EV_XSCALE_DC_FULL_CONTIG, 0x09 },
{ PMC_EV_XSCALE_DC_ACCESS, 0x0a },
{ PMC_EV_XSCALE_DC_MISS, 0x0b },
{ PMC_EV_XSCALE_DC_WRITEBACK, 0x0c },
{ PMC_EV_XSCALE_PC_CHANGE, 0x0d },
/* 3rd Generation XScale cores */
{ PMC_EV_XSCALE_BRANCH_RETIRED_ALL, 0x0e },
{ PMC_EV_XSCALE_INSTR_CYCLE, 0x0f },
{ PMC_EV_XSCALE_CP_STALL, 0x17 },
{ PMC_EV_XSCALE_PC_CHANGE_ALL, 0x18 },
{ PMC_EV_XSCALE_PIPELINE_FLUSH, 0x19 },
{ PMC_EV_XSCALE_BACKEND_STALL, 0x1a },
{ PMC_EV_XSCALE_MULTIPLIER_USE, 0x1b },
{ PMC_EV_XSCALE_MULTIPLIER_STALLED, 0x1c },
{ PMC_EV_XSCALE_DATA_CACHE_STALLED, 0x1e },
{ PMC_EV_XSCALE_L2_CACHE_REQ, 0x20 },
{ PMC_EV_XSCALE_L2_CACHE_MISS, 0x23 },
{ PMC_EV_XSCALE_ADDRESS_BUS_TRANS, 0x40 },
{ PMC_EV_XSCALE_SELF_ADDRESS_BUS_TRANS, 0x41 },
{ PMC_EV_XSCALE_DATA_BUS_TRANS, 0x48 },
};
const int xscale_event_codes_size =
sizeof(xscale_event_codes) / sizeof(xscale_event_codes[0]);
/*
* Per-processor information.
*/
struct xscale_cpu {
struct pmc_hw *pc_xscalepmcs;
};
static struct xscale_cpu **xscale_pcpu;
/*
* Performance Monitor Control Register
*/
static __inline uint32_t
xscale_pmnc_read(void)
{
uint32_t reg;
__asm __volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (reg));
return (reg);
}
static __inline void
xscale_pmnc_write(uint32_t reg)
{
__asm __volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (reg));
}
/*
* Clock Counter Register
*/
static __inline uint32_t
xscale_ccnt_read(void)
{
uint32_t reg;
__asm __volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (reg));
return (reg);
}
static __inline void
xscale_ccnt_write(uint32_t reg)
{
__asm __volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (reg));
}
/*
* Interrupt Enable Register
*/
static __inline uint32_t
xscale_inten_read(void)
{
uint32_t reg;
__asm __volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (reg));
return (reg);
}
static __inline void
xscale_inten_write(uint32_t reg)
{
__asm __volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (reg));
}
/*
* Overflow Flag Register
*/
static __inline uint32_t
xscale_flag_read(void)
{
uint32_t reg;
__asm __volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (reg));
return (reg);
}
static __inline void
xscale_flag_write(uint32_t reg)
{
__asm __volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (reg));
}
/*
* Event Selection Register
*/
static __inline uint32_t
xscale_evtsel_read(void)
{
uint32_t reg;
__asm __volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (reg));
return (reg);
}
static __inline void
xscale_evtsel_write(uint32_t reg)
{
__asm __volatile("mcr p14, 0, %0, c8, c1, 0" : : "r" (reg));
}
/*
* Performance Count Register N
*/
static uint32_t
xscale_pmcn_read(unsigned int pmc)
{
uint32_t reg = 0;
KASSERT(pmc < 4, ("[xscale,%d] illegal PMC number %d", __LINE__, pmc));
switch (pmc) {
case 0:
__asm __volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (reg));
break;
case 1:
__asm __volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (reg));
break;
case 2:
__asm __volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (reg));
break;
case 3:
__asm __volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (reg));
break;
}
return (reg);
}
static uint32_t
xscale_pmcn_write(unsigned int pmc, uint32_t reg)
{
KASSERT(pmc < 4, ("[xscale,%d] illegal PMC number %d", __LINE__, pmc));
switch (pmc) {
case 0:
__asm __volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (reg));
break;
case 1:
__asm __volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (reg));
break;
case 2:
__asm __volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (reg));
break;
case 3:
__asm __volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (reg));
break;
}
return (reg);
}
static int
xscale_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event pe;
uint32_t caps, config;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] illegal row index %d", __LINE__, ri));
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_XSCALE)
return (EINVAL);
pe = a->pm_ev;
for (i = 0; i < xscale_event_codes_size; i++) {
if (xscale_event_codes[i].pe_ev == pe) {
config = xscale_event_codes[i].pe_code;
break;
}
}
if (i == xscale_event_codes_size)
return EINVAL;
/* Generation 1 has fewer events */
if (xscale_gen == 1 && i > PMC_EV_XSCALE_PC_CHANGE)
return EINVAL;
pm->pm_md.pm_xscale.pm_xscale_evsel = config;
- PMCDBG(MDP,ALL,2,"xscale-allocate ri=%d -> config=0x%x", ri, config);
+ PMCDBG2(MDP,ALL,2,"xscale-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
xscale_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] illegal row index %d", __LINE__, ri));
pm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc;
tmp = xscale_pmcn_read(ri);
- PMCDBG(MDP,REA,2,"xscale-read id=%d -> %jd", ri, tmp);
+ PMCDBG2(MDP,REA,2,"xscale-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = XSCALE_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
return 0;
}
static int
xscale_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] illegal row-index %d", __LINE__, ri));
pm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = XSCALE_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
- PMCDBG(MDP,WRI,1,"xscale-write cpu=%d ri=%d v=%jx", cpu, ri, v);
+ PMCDBG3(MDP,WRI,1,"xscale-write cpu=%d ri=%d v=%jx", cpu, ri, v);
xscale_pmcn_write(ri, v);
return 0;
}
static int
xscale_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
- PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
+ PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] illegal row-index %d", __LINE__, ri));
phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[xscale,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
static int
xscale_start_pmc(int cpu, int ri)
{
uint32_t pmnc, config, evtsel;
struct pmc *pm;
struct pmc_hw *phw;
phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri];
pm = phw->phw_pmc;
config = pm->pm_md.pm_xscale.pm_xscale_evsel;
/*
* Configure the event selection.
*
* On the XScale 2nd Generation there's no EVTSEL register.
*/
if (xscale_npmcs == 2) {
pmnc = xscale_pmnc_read();
switch (ri) {
case 0:
pmnc &= ~XSCALE_PMNC_EVT0_MASK;
pmnc |= (config << 12) & XSCALE_PMNC_EVT0_MASK;
break;
case 1:
pmnc &= ~XSCALE_PMNC_EVT1_MASK;
pmnc |= (config << 20) & XSCALE_PMNC_EVT1_MASK;
break;
default:
/* XXX */
break;
}
xscale_pmnc_write(pmnc);
} else {
evtsel = xscale_evtsel_read();
switch (ri) {
case 0:
evtsel &= ~XSCALE_EVTSEL_EVT0_MASK;
evtsel |= config & XSCALE_EVTSEL_EVT0_MASK;
break;
case 1:
evtsel &= ~XSCALE_EVTSEL_EVT1_MASK;
evtsel |= (config << 8) & XSCALE_EVTSEL_EVT1_MASK;
break;
case 2:
evtsel &= ~XSCALE_EVTSEL_EVT2_MASK;
evtsel |= (config << 16) & XSCALE_EVTSEL_EVT2_MASK;
break;
case 3:
evtsel &= ~XSCALE_EVTSEL_EVT3_MASK;
evtsel |= (config << 24) & XSCALE_EVTSEL_EVT3_MASK;
break;
default:
/* XXX */
break;
}
xscale_evtsel_write(evtsel);
}
/*
* Enable the PMC.
*
* Note that XScale provides only one bit to enable/disable _all_
* performance monitoring units.
*/
pmnc = xscale_pmnc_read();
pmnc |= XSCALE_PMNC_ENABLE;
xscale_pmnc_write(pmnc);
return 0;
}
static int
xscale_stop_pmc(int cpu, int ri)
{
uint32_t pmnc, evtsel;
struct pmc *pm;
struct pmc_hw *phw;
phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri];
pm = phw->phw_pmc;
/*
* Disable the PMCs.
*
* Note that XScale provides only one bit to enable/disable _all_
* performance monitoring units.
*/
pmnc = xscale_pmnc_read();
pmnc &= ~XSCALE_PMNC_ENABLE;
xscale_pmnc_write(pmnc);
/*
* A value of 0xff makes the corresponding PMU go into
* power saving mode.
*/
if (xscale_npmcs == 2) {
pmnc = xscale_pmnc_read();
switch (ri) {
case 0:
pmnc |= XSCALE_PMNC_EVT0_MASK;
break;
case 1:
pmnc |= XSCALE_PMNC_EVT1_MASK;
break;
default:
/* XXX */
break;
}
xscale_pmnc_write(pmnc);
} else {
evtsel = xscale_evtsel_read();
switch (ri) {
case 0:
evtsel |= XSCALE_EVTSEL_EVT0_MASK;
break;
case 1:
evtsel |= XSCALE_EVTSEL_EVT1_MASK;
break;
case 2:
evtsel |= XSCALE_EVTSEL_EVT2_MASK;
break;
case 3:
evtsel |= XSCALE_EVTSEL_EVT3_MASK;
break;
default:
/* XXX */
break;
}
xscale_evtsel_write(evtsel);
}
return 0;
}
static int
xscale_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] illegal row-index %d", __LINE__, ri));
phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri];
KASSERT(phw->phw_pmc == NULL,
("[xscale,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
return 0;
}
static int
xscale_intr(int cpu, struct trapframe *tf)
{
printf("intr\n");
return 0;
}
static int
xscale_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
struct pmc_hw *phw;
char xscale_name[PMC_NAME_MAX];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d], illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < xscale_npmcs,
("[xscale,%d] row-index %d out of range", __LINE__, ri));
phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri];
snprintf(xscale_name, sizeof(xscale_name), "XSCALE-%d", ri);
if ((error = copystr(xscale_name, pi->pm_name, PMC_NAME_MAX,
NULL)) != 0)
return error;
pi->pm_class = PMC_CLASS_XSCALE;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
xscale_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc;
return 0;
}
/*
* XXX don't know what we should do here.
*/
static int
xscale_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
return 0;
}
static int
xscale_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
return 0;
}
static int
xscale_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct pmc_cpu *pc;
struct xscale_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[xscale,%d] wrong cpu number %d", __LINE__, cpu));
- PMCDBG(MDP,INI,1,"xscale-init cpu=%d", cpu);
+ PMCDBG1(MDP,INI,1,"xscale-init cpu=%d", cpu);
xscale_pcpu[cpu] = pac = malloc(sizeof(struct xscale_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_xscalepmcs = malloc(sizeof(struct pmc_hw) * xscale_npmcs,
M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE].pcd_ri;
KASSERT(pc != NULL, ("[xscale,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_xscalepmcs; i < xscale_npmcs; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/*
* Disable and put the PMUs into power save mode.
*/
if (xscale_npmcs == 2) {
xscale_pmnc_write(XSCALE_PMNC_EVT1_MASK |
XSCALE_PMNC_EVT0_MASK);
} else {
xscale_evtsel_write(XSCALE_EVTSEL_EVT3_MASK |
XSCALE_EVTSEL_EVT2_MASK | XSCALE_EVTSEL_EVT1_MASK |
XSCALE_EVTSEL_EVT0_MASK);
}
return 0;
}
static int
xscale_pcpu_fini(struct pmc_mdep *md, int cpu)
{
return 0;
}
struct pmc_mdep *
pmc_xscale_initialize()
{
struct pmc_mdep *pmc_mdep;
struct pmc_classdep *pcd;
uint32_t idreg;
/* Get the Core Generation from CP15 */
__asm __volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (idreg));
xscale_gen = (idreg >> 13) & 0x3;
switch (xscale_gen) {
case 1:
xscale_npmcs = 2;
break;
case 2:
case 3:
xscale_npmcs = 4;
break;
default:
printf("%s: unknown XScale core generation\n", __func__);
return (NULL);
}
- PMCDBG(MDP,INI,1,"xscale-init npmcs=%d", xscale_npmcs);
+ PMCDBG1(MDP,INI,1,"xscale-init npmcs=%d", xscale_npmcs);
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
*/
xscale_pcpu = malloc(sizeof(struct xscale_cpu *) * pmc_cpu_max(), M_PMC,
M_WAITOK|M_ZERO);
/* Just one class */
pmc_mdep = pmc_mdep_alloc(1);
pmc_mdep->pmd_cputype = PMC_CPU_INTEL_XSCALE;
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE];
pcd->pcd_caps = XSCALE_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_XSCALE;
pcd->pcd_num = xscale_npmcs;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 32;
pcd->pcd_allocate_pmc = xscale_allocate_pmc;
pcd->pcd_config_pmc = xscale_config_pmc;
pcd->pcd_pcpu_fini = xscale_pcpu_fini;
pcd->pcd_pcpu_init = xscale_pcpu_init;
pcd->pcd_describe = xscale_describe;
pcd->pcd_get_config = xscale_get_config;
pcd->pcd_read_pmc = xscale_read_pmc;
pcd->pcd_release_pmc = xscale_release_pmc;
pcd->pcd_start_pmc = xscale_start_pmc;
pcd->pcd_stop_pmc = xscale_stop_pmc;
pcd->pcd_write_pmc = xscale_write_pmc;
pmc_mdep->pmd_intr = xscale_intr;
pmc_mdep->pmd_switch_in = xscale_switch_in;
pmc_mdep->pmd_switch_out = xscale_switch_out;
pmc_mdep->pmd_npmc += xscale_npmcs;
return (pmc_mdep);
}
void
pmc_xscale_finalize(struct pmc_mdep *md)
{
}
Index: head/sys/sys/pmc.h
===================================================================
--- head/sys/sys/pmc.h (revision 282657)
+++ head/sys/sys/pmc.h (revision 282658)
@@ -1,1113 +1,1148 @@
/*-
* Copyright (c) 2003-2008, Joseph Koshy
* Copyright (c) 2007 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _SYS_PMC_H_
#define _SYS_PMC_H_
#include <dev/hwpmc/pmc_events.h>
#include <machine/pmc_mdep.h>
#include <machine/profile.h>
#define PMC_MODULE_NAME "hwpmc"
#define PMC_NAME_MAX 64 /* HW counter name size */
#define PMC_CLASS_MAX 8 /* max #classes of PMCs per-system */
/*
* Kernel<->userland API version number [MMmmpppp]
*
* Major numbers are to be incremented when an incompatible change to
* the ABI occurs that older clients will not be able to handle.
*
* Minor numbers are incremented when a backwards compatible change
* occurs that allows older correct programs to run unchanged. For
* example, when support for a new PMC type is added.
*
* The patch version is incremented for every bug fix.
*/
#define PMC_VERSION_MAJOR 0x03
#define PMC_VERSION_MINOR 0x01
#define PMC_VERSION_PATCH 0x0000
#define PMC_VERSION (PMC_VERSION_MAJOR << 24 | \
PMC_VERSION_MINOR << 16 | PMC_VERSION_PATCH)
/*
* Kinds of CPUs known.
*
* We keep track of CPU variants that need to be distinguished in
* some way for PMC operations. CPU names are grouped by manufacturer
* and numbered sparsely in order to minimize changes to the ABI involved
* when new CPUs are added.
*/
#define __PMC_CPUS() \
__PMC_CPU(AMD_K7, 0x00, "AMD K7") \
__PMC_CPU(AMD_K8, 0x01, "AMD K8") \
__PMC_CPU(ARMV7, 0x500, "ARMv7") \
__PMC_CPU(INTEL_P5, 0x80, "Intel Pentium") \
__PMC_CPU(INTEL_P6, 0x81, "Intel Pentium Pro") \
__PMC_CPU(INTEL_CL, 0x82, "Intel Celeron") \
__PMC_CPU(INTEL_PII, 0x83, "Intel Pentium II") \
__PMC_CPU(INTEL_PIII, 0x84, "Intel Pentium III") \
__PMC_CPU(INTEL_PM, 0x85, "Intel Pentium M") \
__PMC_CPU(INTEL_PIV, 0x86, "Intel Pentium IV") \
__PMC_CPU(INTEL_CORE, 0x87, "Intel Core Solo/Duo") \
__PMC_CPU(INTEL_CORE2, 0x88, "Intel Core2") \
__PMC_CPU(INTEL_CORE2EXTREME, 0x89, "Intel Core2 Extreme") \
__PMC_CPU(INTEL_ATOM, 0x8A, "Intel Atom") \
__PMC_CPU(INTEL_COREI7, 0x8B, "Intel Core i7") \
__PMC_CPU(INTEL_WESTMERE, 0x8C, "Intel Westmere") \
__PMC_CPU(INTEL_SANDYBRIDGE, 0x8D, "Intel Sandy Bridge") \
__PMC_CPU(INTEL_IVYBRIDGE, 0x8E, "Intel Ivy Bridge") \
__PMC_CPU(INTEL_SANDYBRIDGE_XEON, 0x8F, "Intel Sandy Bridge Xeon") \
__PMC_CPU(INTEL_IVYBRIDGE_XEON, 0x90, "Intel Ivy Bridge Xeon") \
__PMC_CPU(INTEL_HASWELL, 0x91, "Intel Haswell") \
__PMC_CPU(INTEL_ATOM_SILVERMONT, 0x92, "Intel Atom Silvermont") \
__PMC_CPU(INTEL_NEHALEM_EX, 0x93, "Intel Nehalem Xeon 7500") \
__PMC_CPU(INTEL_WESTMERE_EX, 0x94, "Intel Westmere Xeon E7") \
__PMC_CPU(INTEL_HASWELL_XEON, 0x95, "Intel Haswell Xeon E5 v3") \
__PMC_CPU(INTEL_BROADWELL, 0x96, "Intel Broadwell") \
__PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \
__PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \
__PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \
__PMC_CPU(MIPS_74K, 0x202, "MIPS 74K") \
__PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450") \
__PMC_CPU(PPC_E500, 0x340, "PowerPC e500 Core") \
__PMC_CPU(PPC_MPC85XX, 0x340, "Freescale PowerPC MPC85XX") \
__PMC_CPU(PPC_970, 0x380, "IBM PowerPC 970") \
__PMC_CPU(GENERIC, 0x400, "Generic")
enum pmc_cputype {
#undef __PMC_CPU
#define __PMC_CPU(S,V,D) PMC_CPU_##S = V,
__PMC_CPUS()
};
#define PMC_CPU_FIRST PMC_CPU_AMD_K7
#define PMC_CPU_LAST PMC_CPU_GENERIC
/*
* Classes of PMCs
*/
#define __PMC_CLASSES() \
__PMC_CLASS(TSC) /* CPU Timestamp counter */ \
__PMC_CLASS(K7) /* AMD K7 performance counters */ \
__PMC_CLASS(K8) /* AMD K8 performance counters */ \
__PMC_CLASS(P5) /* Intel Pentium counters */ \
__PMC_CLASS(P6) /* Intel Pentium Pro counters */ \
__PMC_CLASS(P4) /* Intel Pentium-IV counters */ \
__PMC_CLASS(IAF) /* Intel Core2/Atom, fixed function */ \
__PMC_CLASS(IAP) /* Intel Core...Atom, programmable */ \
__PMC_CLASS(UCF) /* Intel Uncore fixed function */ \
__PMC_CLASS(UCP) /* Intel Uncore programmable */ \
__PMC_CLASS(XSCALE) /* Intel XScale counters */ \
__PMC_CLASS(ARMV7) /* ARMv7 */ \
__PMC_CLASS(MIPS24K) /* MIPS 24K */ \
__PMC_CLASS(OCTEON) /* Cavium Octeon */ \
__PMC_CLASS(MIPS74K) /* MIPS 74K */ \
__PMC_CLASS(PPC7450) /* Motorola MPC7450 class */ \
__PMC_CLASS(PPC970) /* IBM PowerPC 970 class */ \
__PMC_CLASS(E500) /* Freescale e500 class */ \
__PMC_CLASS(SOFT) /* Software events */
enum pmc_class {
#undef __PMC_CLASS
#define __PMC_CLASS(N) PMC_CLASS_##N ,
__PMC_CLASSES()
};
#define PMC_CLASS_FIRST PMC_CLASS_TSC
#define PMC_CLASS_LAST PMC_CLASS_SOFT
/*
* A PMC can be in the following states:
*
* Hardware states:
* DISABLED -- administratively prohibited from being used.
* FREE -- HW available for use
* Software states:
* ALLOCATED -- allocated
* STOPPED -- allocated, but not counting events
* RUNNING -- allocated, and in operation; 'pm_runcount'
* holds the number of CPUs using this PMC at
* a given instant
* DELETED -- being destroyed
*/
#define __PMC_HWSTATES() \
__PMC_STATE(DISABLED) \
__PMC_STATE(FREE)
#define __PMC_SWSTATES() \
__PMC_STATE(ALLOCATED) \
__PMC_STATE(STOPPED) \
__PMC_STATE(RUNNING) \
__PMC_STATE(DELETED)
#define __PMC_STATES() \
__PMC_HWSTATES() \
__PMC_SWSTATES()
enum pmc_state {
#undef __PMC_STATE
#define __PMC_STATE(S) PMC_STATE_##S,
__PMC_STATES()
__PMC_STATE(MAX)
};
#define PMC_STATE_FIRST PMC_STATE_DISABLED
#define PMC_STATE_LAST PMC_STATE_DELETED
/*
* An allocated PMC may used as a 'global' counter or as a
* 'thread-private' one. Each such mode of use can be in either
* statistical sampling mode or in counting mode. Thus a PMC in use
*
* SS i.e., SYSTEM STATISTICAL -- system-wide statistical profiling
* SC i.e., SYSTEM COUNTER -- system-wide counting mode
* TS i.e., THREAD STATISTICAL -- thread virtual, statistical profiling
* TC i.e., THREAD COUNTER -- thread virtual, counting mode
*
* Statistical profiling modes rely on the PMC periodically delivering
* a interrupt to the CPU (when the configured number of events have
* been measured), so the PMC must have the ability to generate
* interrupts.
*
* In counting modes, the PMC counts its configured events, with the
* value of the PMC being read whenever needed by its owner process.
*
* The thread specific modes "virtualize" the PMCs -- the PMCs appear
* to be thread private and count events only when the profiled thread
* actually executes on the CPU.
*
* The system-wide "global" modes keep the PMCs running all the time
* and are used to measure the behaviour of the whole system.
*/
#define __PMC_MODES() \
__PMC_MODE(SS, 0) \
__PMC_MODE(SC, 1) \
__PMC_MODE(TS, 2) \
__PMC_MODE(TC, 3)
enum pmc_mode {
#undef __PMC_MODE
#define __PMC_MODE(M,N) PMC_MODE_##M = N,
__PMC_MODES()
};
#define PMC_MODE_FIRST PMC_MODE_SS
#define PMC_MODE_LAST PMC_MODE_TC
#define PMC_IS_COUNTING_MODE(mode) \
((mode) == PMC_MODE_SC || (mode) == PMC_MODE_TC)
#define PMC_IS_SYSTEM_MODE(mode) \
((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC)
#define PMC_IS_SAMPLING_MODE(mode) \
((mode) == PMC_MODE_SS || (mode) == PMC_MODE_TS)
#define PMC_IS_VIRTUAL_MODE(mode) \
((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC)
/*
* PMC row disposition
*/
#define __PMC_DISPOSITIONS(N) \
__PMC_DISP(STANDALONE) /* global/disabled counters */ \
__PMC_DISP(FREE) /* free/available */ \
__PMC_DISP(THREAD) /* thread-virtual PMCs */ \
__PMC_DISP(UNKNOWN) /* sentinel */
enum pmc_disp {
#undef __PMC_DISP
#define __PMC_DISP(D) PMC_DISP_##D ,
__PMC_DISPOSITIONS()
};
#define PMC_DISP_FIRST PMC_DISP_STANDALONE
#define PMC_DISP_LAST PMC_DISP_THREAD
/*
* Counter capabilities
*
* __PMC_CAPS(NAME, VALUE, DESCRIPTION)
*/
#define __PMC_CAPS() \
__PMC_CAP(INTERRUPT, 0, "generate interrupts") \
__PMC_CAP(USER, 1, "count user-mode events") \
__PMC_CAP(SYSTEM, 2, "count system-mode events") \
__PMC_CAP(EDGE, 3, "do edge detection of events") \
__PMC_CAP(THRESHOLD, 4, "ignore events below a threshold") \
__PMC_CAP(READ, 5, "read PMC counter") \
__PMC_CAP(WRITE, 6, "reprogram PMC counter") \
__PMC_CAP(INVERT, 7, "invert comparision sense") \
__PMC_CAP(QUALIFIER, 8, "further qualify monitored events") \
__PMC_CAP(PRECISE, 9, "perform precise sampling") \
__PMC_CAP(TAGGING, 10, "tag upstream events") \
__PMC_CAP(CASCADE, 11, "cascade counters")
enum pmc_caps
{
#undef __PMC_CAP
#define __PMC_CAP(NAME, VALUE, DESCR) PMC_CAP_##NAME = (1 << VALUE) ,
__PMC_CAPS()
};
#define PMC_CAP_FIRST PMC_CAP_INTERRUPT
#define PMC_CAP_LAST PMC_CAP_CASCADE
/*
* PMC Event Numbers
*
* These are generated from the definitions in "dev/hwpmc/pmc_events.h".
*/
enum pmc_event {
#undef __PMC_EV
#undef __PMC_EV_BLOCK
#define __PMC_EV_BLOCK(C,V) PMC_EV_ ## C ## __BLOCK_START = (V) - 1 ,
#define __PMC_EV(C,N) PMC_EV_ ## C ## _ ## N ,
__PMC_EVENTS()
};
/*
* PMC SYSCALL INTERFACE
*/
/*
* "PMC_OPS" -- these are the commands recognized by the kernel
* module, and are used when performing a system call from userland.
*/
#define __PMC_OPS() \
__PMC_OP(CONFIGURELOG, "Set log file") \
__PMC_OP(FLUSHLOG, "Flush log file") \
__PMC_OP(GETCPUINFO, "Get system CPU information") \
__PMC_OP(GETDRIVERSTATS, "Get driver statistics") \
__PMC_OP(GETMODULEVERSION, "Get module version") \
__PMC_OP(GETPMCINFO, "Get per-cpu PMC information") \
__PMC_OP(PMCADMIN, "Set PMC state") \
__PMC_OP(PMCALLOCATE, "Allocate and configure a PMC") \
__PMC_OP(PMCATTACH, "Attach a PMC to a process") \
__PMC_OP(PMCDETACH, "Detach a PMC from a process") \
__PMC_OP(PMCGETMSR, "Get a PMC's hardware address") \
__PMC_OP(PMCRELEASE, "Release a PMC") \
__PMC_OP(PMCRW, "Read/Set a PMC") \
__PMC_OP(PMCSETCOUNT, "Set initial count/sampling rate") \
__PMC_OP(PMCSTART, "Start a PMC") \
__PMC_OP(PMCSTOP, "Stop a PMC") \
__PMC_OP(WRITELOG, "Write a cookie to the log file") \
__PMC_OP(CLOSELOG, "Close log file") \
__PMC_OP(GETDYNEVENTINFO, "Get dynamic events list")
enum pmc_ops {
#undef __PMC_OP
#define __PMC_OP(N, D) PMC_OP_##N,
__PMC_OPS()
};
/*
* Flags used in operations on PMCs.
*/
#define PMC_F_FORCE 0x00000001 /*OP ADMIN force operation */
#define PMC_F_DESCENDANTS 0x00000002 /*OP ALLOCATE track descendants */
#define PMC_F_LOG_PROCCSW 0x00000004 /*OP ALLOCATE track ctx switches */
#define PMC_F_LOG_PROCEXIT 0x00000008 /*OP ALLOCATE log proc exits */
#define PMC_F_NEWVALUE 0x00000010 /*OP RW write new value */
#define PMC_F_OLDVALUE 0x00000020 /*OP RW get old value */
#define PMC_F_KGMON 0x00000040 /*OP ALLOCATE kgmon(8) profiling */
/* V2 API */
#define PMC_F_CALLCHAIN 0x00000080 /*OP ALLOCATE capture callchains */
/* internal flags */
#define PMC_F_ATTACHED_TO_OWNER 0x00010000 /*attached to owner*/
#define PMC_F_NEEDS_LOGFILE 0x00020000 /*needs log file */
#define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */
#define PMC_CALLCHAIN_DEPTH_MAX 128
#define PMC_CC_F_USERSPACE 0x01 /*userspace callchain*/
/*
* Cookies used to denote allocated PMCs, and the values of PMCs.
*/
typedef uint32_t pmc_id_t;
typedef uint64_t pmc_value_t;
#define PMC_ID_INVALID (~ (pmc_id_t) 0)
/*
* PMC IDs have the following format:
*
* +--------+----------+-----------+-----------+
* | CPU | PMC MODE | PMC CLASS | ROW INDEX |
* +--------+----------+-----------+-----------+
*
* where each field is 8 bits wide. Field 'CPU' is set to the
* requested CPU for system-wide PMCs or PMC_CPU_ANY for process-mode
* PMCs. Field 'PMC MODE' is the allocated PMC mode. Field 'PMC
* CLASS' is the class of the PMC. Field 'ROW INDEX' is the row index
* for the PMC.
*
* The 'ROW INDEX' ranges over 0..NWPMCS where NHWPMCS is the total
* number of hardware PMCs on this cpu.
*/
#define PMC_ID_TO_ROWINDEX(ID) ((ID) & 0xFF)
#define PMC_ID_TO_CLASS(ID) (((ID) & 0xFF00) >> 8)
#define PMC_ID_TO_MODE(ID) (((ID) & 0xFF0000) >> 16)
#define PMC_ID_TO_CPU(ID) (((ID) & 0xFF000000) >> 24)
#define PMC_ID_MAKE_ID(CPU,MODE,CLASS,ROWINDEX) \
((((CPU) & 0xFF) << 24) | (((MODE) & 0xFF) << 16) | \
(((CLASS) & 0xFF) << 8) | ((ROWINDEX) & 0xFF))
/*
* Data structures for system calls supported by the pmc driver.
*/
/*
* OP PMCALLOCATE
*
* Allocate a PMC on the named CPU.
*/
#define PMC_CPU_ANY ~0
struct pmc_op_pmcallocate {
uint32_t pm_caps; /* PMC_CAP_* */
uint32_t pm_cpu; /* CPU number or PMC_CPU_ANY */
enum pmc_class pm_class; /* class of PMC desired */
enum pmc_event pm_ev; /* [enum pmc_event] desired */
uint32_t pm_flags; /* additional modifiers PMC_F_* */
enum pmc_mode pm_mode; /* desired mode */
pmc_id_t pm_pmcid; /* [return] process pmc id */
union pmc_md_op_pmcallocate pm_md; /* MD layer extensions */
};
/*
* OP PMCADMIN
*
* Set the administrative state (i.e., whether enabled or disabled) of
* a PMC 'pm_pmc' on CPU 'pm_cpu'. Note that 'pm_pmc' specifies an
* absolute PMC number and need not have been first allocated by the
* calling process.
*/
struct pmc_op_pmcadmin {
int pm_cpu; /* CPU# */
uint32_t pm_flags; /* flags */
int pm_pmc; /* PMC# */
enum pmc_state pm_state; /* desired state */
};
/*
* OP PMCATTACH / OP PMCDETACH
*
* Attach/detach a PMC and a process.
*/
struct pmc_op_pmcattach {
pmc_id_t pm_pmc; /* PMC to attach to */
pid_t pm_pid; /* target process */
};
/*
* OP PMCSETCOUNT
*
* Set the sampling rate (i.e., the reload count) for statistical counters.
* 'pm_pmcid' need to have been previously allocated using PMCALLOCATE.
*/
struct pmc_op_pmcsetcount {
pmc_value_t pm_count; /* initial/sample count */
pmc_id_t pm_pmcid; /* PMC id to set */
};
/*
* OP PMCRW
*
* Read the value of a PMC named by 'pm_pmcid'. 'pm_pmcid' needs
* to have been previously allocated using PMCALLOCATE.
*/
struct pmc_op_pmcrw {
uint32_t pm_flags; /* PMC_F_{OLD,NEW}VALUE*/
pmc_id_t pm_pmcid; /* pmc id */
pmc_value_t pm_value; /* new&returned value */
};
/*
* OP GETPMCINFO
*
* retrieve PMC state for a named CPU. The caller is expected to
* allocate 'npmc' * 'struct pmc_info' bytes of space for the return
* values.
*/
struct pmc_info {
char pm_name[PMC_NAME_MAX]; /* pmc name */
enum pmc_class pm_class; /* enum pmc_class */
int pm_enabled; /* whether enabled */
enum pmc_disp pm_rowdisp; /* FREE, THREAD or STANDLONE */
pid_t pm_ownerpid; /* owner, or -1 */
enum pmc_mode pm_mode; /* current mode [enum pmc_mode] */
enum pmc_event pm_event; /* current event */
uint32_t pm_flags; /* current flags */
pmc_value_t pm_reloadcount; /* sampling counters only */
};
struct pmc_op_getpmcinfo {
int32_t pm_cpu; /* 0 <= cpu < mp_maxid */
struct pmc_info pm_pmcs[]; /* space for 'npmc' structures */
};
/*
* OP GETCPUINFO
*
* Retrieve system CPU information.
*/
struct pmc_classinfo {
enum pmc_class pm_class; /* class id */
uint32_t pm_caps; /* counter capabilities */
uint32_t pm_width; /* width of the PMC */
uint32_t pm_num; /* number of PMCs in class */
};
struct pmc_op_getcpuinfo {
enum pmc_cputype pm_cputype; /* what kind of CPU */
uint32_t pm_ncpu; /* max CPU number */
uint32_t pm_npmc; /* #PMCs per CPU */
uint32_t pm_nclass; /* #classes of PMCs */
struct pmc_classinfo pm_classes[PMC_CLASS_MAX];
};
/*
* OP CONFIGURELOG
*
* Configure a log file for writing system-wide statistics to.
*/
struct pmc_op_configurelog {
int pm_flags;
int pm_logfd; /* logfile fd (or -1) */
};
/*
* OP GETDRIVERSTATS
*
* Retrieve pmc(4) driver-wide statistics.
*/
struct pmc_op_getdriverstats {
int pm_intr_ignored; /* #interrupts ignored */
int pm_intr_processed; /* #interrupts processed */
int pm_intr_bufferfull; /* #interrupts with ENOSPC */
int pm_syscalls; /* #syscalls */
int pm_syscall_errors; /* #syscalls with errors */
int pm_buffer_requests; /* #buffer requests */
int pm_buffer_requests_failed; /* #failed buffer requests */
int pm_log_sweeps; /* #sample buffer processing passes */
};
/*
* OP RELEASE / OP START / OP STOP
*
* Simple operations on a PMC id.
*/
struct pmc_op_simple {
pmc_id_t pm_pmcid;
};
/*
* OP WRITELOG
*
* Flush the current log buffer and write 4 bytes of user data to it.
*/
struct pmc_op_writelog {
uint32_t pm_userdata;
};
/*
* OP GETMSR
*
* Retrieve the machine specific address assoicated with the allocated
* PMC. This number can be used subsequently with a read-performance-counter
* instruction.
*/
struct pmc_op_getmsr {
uint32_t pm_msr; /* machine specific address */
pmc_id_t pm_pmcid; /* allocated pmc id */
};
/*
* OP GETDYNEVENTINFO
*
* Retrieve a PMC dynamic class events list.
*/
struct pmc_dyn_event_descr {
char pm_ev_name[PMC_NAME_MAX];
enum pmc_event pm_ev_code;
};
struct pmc_op_getdyneventinfo {
enum pmc_class pm_class;
unsigned int pm_nevent;
struct pmc_dyn_event_descr pm_events[PMC_EV_DYN_COUNT];
};
#ifdef _KERNEL
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <machine/frame.h>
#define PMC_HASH_SIZE 1024
#define PMC_MTXPOOL_SIZE 2048
#define PMC_LOG_BUFFER_SIZE 4
#define PMC_NLOGBUFFERS 1024
#define PMC_NSAMPLES 1024
#define PMC_CALLCHAIN_DEPTH 32
#define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "."
/*
* Locking keys
*
* (b) - pmc_bufferlist_mtx (spin lock)
* (k) - pmc_kthread_mtx (sleep lock)
* (o) - po->po_mtx (spin lock)
*/
/*
* PMC commands
*/
struct pmc_syscall_args {
register_t pmop_code; /* one of PMC_OP_* */
void *pmop_data; /* syscall parameter */
};
/*
* Interface to processor specific s1tuff
*/
/*
* struct pmc_descr
*
* Machine independent (i.e., the common parts) of a human readable
* PMC description.
*/
struct pmc_descr {
char pd_name[PMC_NAME_MAX]; /* name */
uint32_t pd_caps; /* capabilities */
enum pmc_class pd_class; /* class of the PMC */
uint32_t pd_width; /* width in bits */
};
/*
* struct pmc_target
*
* This structure records all the target processes associated with a
* PMC.
*/
struct pmc_target {
LIST_ENTRY(pmc_target) pt_next;
struct pmc_process *pt_process; /* target descriptor */
};
/*
* struct pmc
*
* Describes each allocated PMC.
*
* Each PMC has precisely one owner, namely the process that allocated
* the PMC.
*
* A PMC may be attached to multiple target processes. The
* 'pm_targets' field links all the target processes being monitored
* by this PMC.
*
* The 'pm_savedvalue' field is protected by a mutex.
*
* On a multi-cpu machine, multiple target threads associated with a
* process-virtual PMC could be concurrently executing on different
* CPUs. The 'pm_runcount' field is atomically incremented every time
* the PMC gets scheduled on a CPU and atomically decremented when it
* get descheduled. Deletion of a PMC is only permitted when this
* field is '0'.
*
*/
struct pmc {
LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */
LIST_ENTRY(pmc) pm_next; /* owner's list */
/*
* System-wide PMCs are allocated on a CPU and are not moved
* around. For system-wide PMCs we record the CPU the PMC was
* allocated on in the 'CPU' field of the pmc ID.
*
* Virtual PMCs run on whichever CPU is currently executing
* their targets' threads. For these PMCs we need to save
* their current PMC counter values when they are taken off
* CPU.
*/
union {
pmc_value_t pm_savedvalue; /* Virtual PMCS */
} pm_gv;
/*
* For sampling mode PMCs, we keep track of the PMC's "reload
* count", which is the counter value to be loaded in when
* arming the PMC for the next counting session. For counting
* modes on PMCs that are read-only (e.g., the x86 TSC), we
* keep track of the initial value at the start of
* counting-mode operation.
*/
union {
pmc_value_t pm_reloadcount; /* sampling PMC modes */
pmc_value_t pm_initial; /* counting PMC modes */
} pm_sc;
uint32_t pm_stalled; /* marks stalled sampling PMCs */
uint32_t pm_caps; /* PMC capabilities */
enum pmc_event pm_event; /* event being measured */
uint32_t pm_flags; /* additional flags PMC_F_... */
struct pmc_owner *pm_owner; /* owner thread state */
int pm_runcount; /* #cpus currently on */
enum pmc_state pm_state; /* current PMC state */
/*
* The PMC ID field encodes the row-index for the PMC, its
* mode, class and the CPU# associated with the PMC.
*/
pmc_id_t pm_id; /* allocated PMC id */
/* md extensions */
union pmc_md_pmc pm_md;
};
/*
* Accessor macros for 'struct pmc'
*/
#define PMC_TO_MODE(P) PMC_ID_TO_MODE((P)->pm_id)
#define PMC_TO_CLASS(P) PMC_ID_TO_CLASS((P)->pm_id)
#define PMC_TO_ROWINDEX(P) PMC_ID_TO_ROWINDEX((P)->pm_id)
#define PMC_TO_CPU(P) PMC_ID_TO_CPU((P)->pm_id)
/*
* struct pmc_process
*
* Record a 'target' process being profiled.
*
* The target process being profiled could be different from the owner
* process which allocated the PMCs. Each target process descriptor
* is associated with NHWPMC 'struct pmc *' pointers. Each PMC at a
* given hardware row-index 'n' will use slot 'n' of the 'pp_pmcs[]'
* array. The size of this structure is thus PMC architecture
* dependent.
*
*/
struct pmc_targetstate {
struct pmc *pp_pmc; /* target PMC */
pmc_value_t pp_pmcval; /* per-process value */
};
struct pmc_process {
LIST_ENTRY(pmc_process) pp_next; /* hash chain */
int pp_refcnt; /* reference count */
uint32_t pp_flags; /* flags PMC_PP_* */
struct proc *pp_proc; /* target thread */
struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */
};
#define PMC_PP_ENABLE_MSR_ACCESS 0x00000001
/*
* struct pmc_owner
*
* We associate a PMC with an 'owner' process.
*
* A process can be associated with 0..NCPUS*NHWPMC PMCs during its
* lifetime, where NCPUS is the numbers of CPUS in the system and
* NHWPMC is the number of hardware PMCs per CPU. These are
* maintained in the list headed by the 'po_pmcs' to save on space.
*
*/
struct pmc_owner {
LIST_ENTRY(pmc_owner) po_next; /* hash chain */
LIST_ENTRY(pmc_owner) po_ssnext; /* list of SS PMC owners */
LIST_HEAD(, pmc) po_pmcs; /* owned PMC list */
TAILQ_HEAD(, pmclog_buffer) po_logbuffers; /* (o) logbuffer list */
struct mtx po_mtx; /* spin lock for (o) */
struct proc *po_owner; /* owner proc */
uint32_t po_flags; /* (k) flags PMC_PO_* */
struct proc *po_kthread; /* (k) helper kthread */
struct pmclog_buffer *po_curbuf; /* current log buffer */
struct file *po_file; /* file reference */
int po_error; /* recorded error */
short po_sscount; /* # SS PMCs owned */
short po_logprocmaps; /* global mappings done */
};
#define PMC_PO_OWNS_LOGFILE 0x00000001 /* has a log file */
#define PMC_PO_SHUTDOWN 0x00000010 /* in the process of shutdown */
#define PMC_PO_INITIAL_MAPPINGS_DONE 0x00000020
/*
* struct pmc_hw -- describe the state of the PMC hardware
*
* When in use, a HW PMC is associated with one allocated 'struct pmc'
* pointed to by field 'phw_pmc'. When inactive, this field is NULL.
*
* On an SMP box, one or more HW PMC's in process virtual mode with
* the same 'phw_pmc' could be executing on different CPUs. In order
* to handle this case correctly, we need to ensure that only
* incremental counts get added to the saved value in the associated
* 'struct pmc'. The 'phw_save' field is used to keep the saved PMC
* value at the time the hardware is started during this context
* switch (i.e., the difference between the new (hardware) count and
* the saved count is atomically added to the count field in 'struct
* pmc' at context switch time).
*
*/
struct pmc_hw {
uint32_t phw_state; /* see PHW_* macros below */
struct pmc *phw_pmc; /* current thread PMC */
};
#define PMC_PHW_RI_MASK 0x000000FF
#define PMC_PHW_CPU_SHIFT 8
#define PMC_PHW_CPU_MASK 0x0000FF00
#define PMC_PHW_FLAGS_SHIFT 16
#define PMC_PHW_FLAGS_MASK 0xFFFF0000
#define PMC_PHW_INDEX_TO_STATE(ri) ((ri) & PMC_PHW_RI_MASK)
#define PMC_PHW_STATE_TO_INDEX(state) ((state) & PMC_PHW_RI_MASK)
#define PMC_PHW_CPU_TO_STATE(cpu) (((cpu) << PMC_PHW_CPU_SHIFT) & \
PMC_PHW_CPU_MASK)
#define PMC_PHW_STATE_TO_CPU(state) (((state) & PMC_PHW_CPU_MASK) >> \
PMC_PHW_CPU_SHIFT)
#define PMC_PHW_FLAGS_TO_STATE(flags) (((flags) << PMC_PHW_FLAGS_SHIFT) & \
PMC_PHW_FLAGS_MASK)
#define PMC_PHW_STATE_TO_FLAGS(state) (((state) & PMC_PHW_FLAGS_MASK) >> \
PMC_PHW_FLAGS_SHIFT)
#define PMC_PHW_FLAG_IS_ENABLED (PMC_PHW_FLAGS_TO_STATE(0x01))
#define PMC_PHW_FLAG_IS_SHAREABLE (PMC_PHW_FLAGS_TO_STATE(0x02))
/*
* struct pmc_sample
*
* Space for N (tunable) PC samples and associated control data.
*/
struct pmc_sample {
uint16_t ps_nsamples; /* callchain depth */
uint8_t ps_cpu; /* cpu number */
uint8_t ps_flags; /* other flags */
pid_t ps_pid; /* process PID or -1 */
struct thread *ps_td; /* which thread */
struct pmc *ps_pmc; /* interrupting PMC */
uintptr_t *ps_pc; /* (const) callchain start */
};
#define PMC_SAMPLE_FREE ((uint16_t) 0)
#define PMC_SAMPLE_INUSE ((uint16_t) 0xFFFF)
struct pmc_samplebuffer {
struct pmc_sample * volatile ps_read; /* read pointer */
struct pmc_sample * volatile ps_write; /* write pointer */
uintptr_t *ps_callchains; /* all saved call chains */
struct pmc_sample *ps_fence; /* one beyond ps_samples[] */
struct pmc_sample ps_samples[]; /* array of sample entries */
};
/*
* struct pmc_cpustate
*
* A CPU is modelled as a collection of HW PMCs with space for additional
* flags.
*/
struct pmc_cpu {
uint32_t pc_state; /* physical cpu number + flags */
struct pmc_samplebuffer *pc_sb[2]; /* space for samples */
struct pmc_hw *pc_hwpmcs[]; /* 'npmc' pointers */
};
#define PMC_PCPU_CPU_MASK 0x000000FF
#define PMC_PCPU_FLAGS_MASK 0xFFFFFF00
#define PMC_PCPU_FLAGS_SHIFT 8
#define PMC_PCPU_STATE_TO_CPU(S) ((S) & PMC_PCPU_CPU_MASK)
#define PMC_PCPU_STATE_TO_FLAGS(S) (((S) & PMC_PCPU_FLAGS_MASK) >> PMC_PCPU_FLAGS_SHIFT)
#define PMC_PCPU_FLAGS_TO_STATE(F) (((F) << PMC_PCPU_FLAGS_SHIFT) & PMC_PCPU_FLAGS_MASK)
#define PMC_PCPU_CPU_TO_STATE(C) ((C) & PMC_PCPU_CPU_MASK)
#define PMC_PCPU_FLAG_HTT (PMC_PCPU_FLAGS_TO_STATE(0x1))
/*
* struct pmc_binding
*
* CPU binding information.
*/
struct pmc_binding {
int pb_bound; /* is bound? */
int pb_cpu; /* if so, to which CPU */
};
struct pmc_mdep;
/*
* struct pmc_classdep
*
* PMC class-dependent operations.
*/
struct pmc_classdep {
uint32_t pcd_caps; /* class capabilities */
enum pmc_class pcd_class; /* class id */
int pcd_num; /* number of PMCs */
int pcd_ri; /* row index of the first PMC in class */
int pcd_width; /* width of the PMC */
/* configuring/reading/writing the hardware PMCs */
int (*pcd_config_pmc)(int _cpu, int _ri, struct pmc *_pm);
int (*pcd_get_config)(int _cpu, int _ri, struct pmc **_ppm);
int (*pcd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value);
int (*pcd_write_pmc)(int _cpu, int _ri, pmc_value_t _value);
/* pmc allocation/release */
int (*pcd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t,
const struct pmc_op_pmcallocate *_a);
int (*pcd_release_pmc)(int _cpu, int _ri, struct pmc *_pm);
/* starting and stopping PMCs */
int (*pcd_start_pmc)(int _cpu, int _ri);
int (*pcd_stop_pmc)(int _cpu, int _ri);
/* description */
int (*pcd_describe)(int _cpu, int _ri, struct pmc_info *_pi,
struct pmc **_ppmc);
/* class-dependent initialization & finalization */
int (*pcd_pcpu_init)(struct pmc_mdep *_md, int _cpu);
int (*pcd_pcpu_fini)(struct pmc_mdep *_md, int _cpu);
/* machine-specific interface */
int (*pcd_get_msr)(int _ri, uint32_t *_msr);
};
/*
* struct pmc_mdep
*
* Machine dependent bits needed per CPU type.
*/
struct pmc_mdep {
uint32_t pmd_cputype; /* from enum pmc_cputype */
uint32_t pmd_npmc; /* number of PMCs per CPU */
uint32_t pmd_nclass; /* number of PMC classes present */
/*
* Machine dependent methods.
*/
/* per-cpu initialization and finalization */
int (*pmd_pcpu_init)(struct pmc_mdep *_md, int _cpu);
int (*pmd_pcpu_fini)(struct pmc_mdep *_md, int _cpu);
/* thread context switch in/out */
int (*pmd_switch_in)(struct pmc_cpu *_p, struct pmc_process *_pp);
int (*pmd_switch_out)(struct pmc_cpu *_p, struct pmc_process *_pp);
/* handle a PMC interrupt */
int (*pmd_intr)(int _cpu, struct trapframe *_tf);
/*
* PMC class dependent information.
*/
struct pmc_classdep pmd_classdep[];
};
/*
* Per-CPU state. This is an array of 'mp_ncpu' pointers
* to struct pmc_cpu descriptors.
*/
extern struct pmc_cpu **pmc_pcpu;
/* driver statistics */
extern struct pmc_op_getdriverstats pmc_stats;
#if defined(HWPMC_DEBUG)
+#include <sys/ktr.h>
/* debug flags, major flag groups */
struct pmc_debugflags {
int pdb_CPU;
int pdb_CSW;
int pdb_LOG;
int pdb_MDP;
int pdb_MOD;
int pdb_OWN;
int pdb_PMC;
int pdb_PRC;
int pdb_SAM;
};
extern struct pmc_debugflags pmc_debugflags;
+#define KTR_PMC KTR_SUBSYS
+
#define PMC_DEBUG_STRSIZE 128
#define PMC_DEBUG_DEFAULT_FLAGS { 0, 0, 0, 0, 0, 0, 0, 0 }
-#define PMCDBG(M,N,L,F,...) do { \
+#define PMCDBG0(M, N, L, F) do { \
if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
- printf(#M ":" #N ":" #L ": " F "\n", __VA_ARGS__); \
+ CTR0(KTR_PMC, #M ":" #N ":" #L ": " F); \
} while (0)
-
+#define PMCDBG1(M, N, L, F, p1) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR1(KTR_PMC, #M ":" #N ":" #L ": " F, p1); \
+} while (0)
+#define PMCDBG2(M, N, L, F, p1, p2) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR2(KTR_PMC, #M ":" #N ":" #L ": " F, p1, p2); \
+} while (0)
+#define PMCDBG3(M, N, L, F, p1, p2, p3) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR3(KTR_PMC, #M ":" #N ":" #L ": " F, p1, p2, p3); \
+} while (0)
+#define PMCDBG4(M, N, L, F, p1, p2, p3, p4) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR4(KTR_PMC, #M ":" #N ":" #L ": " F, p1, p2, p3, p4);\
+} while (0)
+#define PMCDBG5(M, N, L, F, p1, p2, p3, p4, p5) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR5(KTR_PMC, #M ":" #N ":" #L ": " F, p1, p2, p3, p4, \
+ p5); \
+} while (0)
+#define PMCDBG6(M, N, L, F, p1, p2, p3, p4, p5, p6) do { \
+ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \
+ CTR6(KTR_PMC, #M ":" #N ":" #L ": " F, p1, p2, p3, p4, \
+ p5, p6); \
+} while (0)
+
/* Major numbers */
#define PMC_DEBUG_MAJ_CPU 0 /* cpu switches */
#define PMC_DEBUG_MAJ_CSW 1 /* context switches */
#define PMC_DEBUG_MAJ_LOG 2 /* logging */
#define PMC_DEBUG_MAJ_MDP 3 /* machine dependent */
#define PMC_DEBUG_MAJ_MOD 4 /* misc module infrastructure */
#define PMC_DEBUG_MAJ_OWN 5 /* owner */
#define PMC_DEBUG_MAJ_PMC 6 /* pmc management */
#define PMC_DEBUG_MAJ_PRC 7 /* processes */
#define PMC_DEBUG_MAJ_SAM 8 /* sampling */
/* Minor numbers */
/* Common (8 bits) */
#define PMC_DEBUG_MIN_ALL 0 /* allocation */
#define PMC_DEBUG_MIN_REL 1 /* release */
#define PMC_DEBUG_MIN_OPS 2 /* ops: start, stop, ... */
#define PMC_DEBUG_MIN_INI 3 /* init */
#define PMC_DEBUG_MIN_FND 4 /* find */
/* MODULE */
#define PMC_DEBUG_MIN_PMH 14 /* pmc_hook */
#define PMC_DEBUG_MIN_PMS 15 /* pmc_syscall */
/* OWN */
#define PMC_DEBUG_MIN_ORM 8 /* owner remove */
#define PMC_DEBUG_MIN_OMR 9 /* owner maybe remove */
/* PROCESSES */
#define PMC_DEBUG_MIN_TLK 8 /* link target */
#define PMC_DEBUG_MIN_TUL 9 /* unlink target */
#define PMC_DEBUG_MIN_EXT 10 /* process exit */
#define PMC_DEBUG_MIN_EXC 11 /* process exec */
#define PMC_DEBUG_MIN_FRK 12 /* process fork */
#define PMC_DEBUG_MIN_ATT 13 /* attach/detach */
#define PMC_DEBUG_MIN_SIG 14 /* signalling */
/* CONTEXT SWITCHES */
#define PMC_DEBUG_MIN_SWI 8 /* switch in */
#define PMC_DEBUG_MIN_SWO 9 /* switch out */
/* PMC */
#define PMC_DEBUG_MIN_REG 8 /* pmc register */
#define PMC_DEBUG_MIN_ALR 9 /* allocate row */
/* MACHINE DEPENDENT LAYER */
#define PMC_DEBUG_MIN_REA 8 /* read */
#define PMC_DEBUG_MIN_WRI 9 /* write */
#define PMC_DEBUG_MIN_CFG 10 /* config */
#define PMC_DEBUG_MIN_STA 11 /* start */
#define PMC_DEBUG_MIN_STO 12 /* stop */
#define PMC_DEBUG_MIN_INT 13 /* interrupts */
/* CPU */
#define PMC_DEBUG_MIN_BND 8 /* bind */
#define PMC_DEBUG_MIN_SEL 9 /* select */
/* LOG */
#define PMC_DEBUG_MIN_GTB 8 /* get buf */
#define PMC_DEBUG_MIN_SIO 9 /* schedule i/o */
#define PMC_DEBUG_MIN_FLS 10 /* flush */
#define PMC_DEBUG_MIN_SAM 11 /* sample */
#define PMC_DEBUG_MIN_CLO 12 /* close */
#else
-#define PMCDBG(M,N,L,F,...) /* nothing */
+#define PMCDBG0(M, N, L, F) /* nothing */
+#define PMCDBG1(M, N, L, F, p1)
+#define PMCDBG2(M, N, L, F, p1, p2)
+#define PMCDBG3(M, N, L, F, p1, p2, p3)
+#define PMCDBG4(M, N, L, F, p1, p2, p3, p4)
+#define PMCDBG5(M, N, L, F, p1, p2, p3, p4, p5)
+#define PMCDBG6(M, N, L, F, p1, p2, p3, p4, p5, p6)
#endif
/* declare a dedicated memory pool */
MALLOC_DECLARE(M_PMC);
/*
* Functions
*/
struct pmc_mdep *pmc_md_initialize(void); /* MD init function */
void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */
int pmc_getrowdisp(int _ri);
int pmc_process_interrupt(int _cpu, int _soft, struct pmc *_pm,
struct trapframe *_tf, int _inuserspace);
int pmc_save_kernel_callchain(uintptr_t *_cc, int _maxsamples,
struct trapframe *_tf);
int pmc_save_user_callchain(uintptr_t *_cc, int _maxsamples,
struct trapframe *_tf);
struct pmc_mdep *pmc_mdep_alloc(int nclasses);
void pmc_mdep_free(struct pmc_mdep *md);
#endif /* _KERNEL */
#endif /* _SYS_PMC_H_ */