Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/hwpmc/hwpmc_core.c
Show First 20 Lines • Show All 117 Lines • ▼ Show 20 Lines | core_pcpu_init(struct pmc_mdep *md, int cpu) | ||||
struct pmc_cpu *pc; | struct pmc_cpu *pc; | ||||
struct core_cpu *cc; | struct core_cpu *cc; | ||||
struct pmc_hw *phw; | struct pmc_hw *phw; | ||||
int core_ri, n, npmc; | int core_ri, n, npmc; | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[iaf,%d] insane cpu number %d", __LINE__, cpu)); | ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); | ||||
PMCDBG(MDP,INI,1,"core-init cpu=%d", cpu); | PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); | ||||
core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; | core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; | ||||
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; | npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; | ||||
if (core_cputype != PMC_CPU_INTEL_CORE) | if (core_cputype != PMC_CPU_INTEL_CORE) | ||||
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; | npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; | ||||
cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), | cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), | ||||
Show All 22 Lines | core_pcpu_fini(struct pmc_mdep *md, int cpu) | ||||
int core_ri, n, npmc; | int core_ri, n, npmc; | ||||
struct pmc_cpu *pc; | struct pmc_cpu *pc; | ||||
struct core_cpu *cc; | struct core_cpu *cc; | ||||
uint64_t msr = 0; | uint64_t msr = 0; | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] insane cpu number (%d)", __LINE__, cpu)); | ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); | ||||
PMCDBG(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); | PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); | ||||
if ((cc = core_pcpu[cpu]) == NULL) | if ((cc = core_pcpu[cpu]) == NULL) | ||||
return (0); | return (0); | ||||
core_pcpu[cpu] = NULL; | core_pcpu[cpu] = NULL; | ||||
pc = pmc_pcpu[cpu]; | pc = pmc_pcpu[cpu]; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, | ||||
const struct pmc_op_pmcallocate *a) | const struct pmc_op_pmcallocate *a) | ||||
{ | { | ||||
enum pmc_event ev; | enum pmc_event ev; | ||||
uint32_t caps, flags, validflags; | uint32_t caps, flags, validflags; | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU %d", __LINE__, cpu)); | ("[core,%d] illegal CPU %d", __LINE__, cpu)); | ||||
PMCDBG(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); | PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); | ||||
if (ri < 0 || ri > core_iaf_npmc) | if (ri < 0 || ri > core_iaf_npmc) | ||||
return (EINVAL); | return (EINVAL); | ||||
caps = a->pm_caps; | caps = a->pm_caps; | ||||
if (a->pm_class != PMC_CLASS_IAF || | if (a->pm_class != PMC_CLASS_IAF || | ||||
(caps & IAF_PMC_CAPS) != caps) | (caps & IAF_PMC_CAPS) != caps) | ||||
Show All 27 Lines | if (caps & PMC_CAP_SYSTEM) | ||||
flags |= IAF_OS; | flags |= IAF_OS; | ||||
if (caps & PMC_CAP_USER) | if (caps & PMC_CAP_USER) | ||||
flags |= IAF_USR; | flags |= IAF_USR; | ||||
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) | if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) | ||||
flags |= (IAF_OS | IAF_USR); | flags |= (IAF_OS | IAF_USR); | ||||
pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); | pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); | ||||
PMCDBG(MDP,ALL,2, "iaf-allocate config=0x%jx", | PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", | ||||
(uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); | (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iaf_config_pmc(int cpu, int ri, struct pmc *pm) | iaf_config_pmc(int cpu, int ri, struct pmc *pm) | ||||
{ | { | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU %d", __LINE__, cpu)); | ("[core,%d] illegal CPU %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iaf_npmc, | KASSERT(ri >= 0 && ri < core_iaf_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
PMCDBG(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); | PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); | ||||
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, | KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, | ||||
cpu)); | cpu)); | ||||
core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; | core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | iaf_read_pmc(int cpu, int ri, pmc_value_t *v) | ||||
tmp = rdpmc(IAF_RI_TO_MSR(ri)); | tmp = rdpmc(IAF_RI_TO_MSR(ri)); | ||||
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | ||||
*v = iaf_perfctr_value_to_reload_count(tmp); | *v = iaf_perfctr_value_to_reload_count(tmp); | ||||
else | else | ||||
*v = tmp; | *v = tmp; | ||||
PMCDBG(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, | PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, | ||||
IAF_RI_TO_MSR(ri), *v); | IAF_RI_TO_MSR(ri), *v); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iaf_release_pmc(int cpu, int ri, struct pmc *pmc) | iaf_release_pmc(int cpu, int ri, struct pmc *pmc) | ||||
{ | { | ||||
PMCDBG(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); | PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iaf_npmc, | KASSERT(ri >= 0 && ri < core_iaf_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, | KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, | ||||
("[core,%d] PHW pmc non-NULL", __LINE__)); | ("[core,%d] PHW pmc non-NULL", __LINE__)); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iaf_start_pmc(int cpu, int ri) | iaf_start_pmc(int cpu, int ri) | ||||
{ | { | ||||
struct pmc *pm; | struct pmc *pm; | ||||
struct core_cpu *iafc; | struct core_cpu *iafc; | ||||
uint64_t msr = 0; | uint64_t msr = 0; | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iaf_npmc, | KASSERT(ri >= 0 && ri < core_iaf_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
PMCDBG(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); | PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); | ||||
iafc = core_pcpu[cpu]; | iafc = core_pcpu[cpu]; | ||||
pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; | pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; | ||||
iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; | iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; | ||||
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | ||||
wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); | wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); | ||||
do { | do { | ||||
iafc->pc_resync = 0; | iafc->pc_resync = 0; | ||||
iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); | iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); | ||||
msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; | msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; | ||||
wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & | wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & | ||||
IAF_GLOBAL_CTRL_MASK)); | IAF_GLOBAL_CTRL_MASK)); | ||||
} while (iafc->pc_resync != 0); | } while (iafc->pc_resync != 0); | ||||
PMCDBG(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", | PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", | ||||
iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), | iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), | ||||
iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); | iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iaf_stop_pmc(int cpu, int ri) | iaf_stop_pmc(int cpu, int ri) | ||||
{ | { | ||||
uint32_t fc; | uint32_t fc; | ||||
struct core_cpu *iafc; | struct core_cpu *iafc; | ||||
uint64_t msr = 0; | uint64_t msr = 0; | ||||
PMCDBG(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri); | PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri); | ||||
iafc = core_pcpu[cpu]; | iafc = core_pcpu[cpu]; | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iaf_npmc, | KASSERT(ri >= 0 && ri < core_iaf_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
fc = (IAF_MASK << (ri * 4)); | fc = (IAF_MASK << (ri * 4)); | ||||
if (core_cputype != PMC_CPU_INTEL_ATOM && | if (core_cputype != PMC_CPU_INTEL_ATOM && | ||||
core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT) | core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT) | ||||
fc &= ~IAF_ANY; | fc &= ~IAF_ANY; | ||||
iafc->pc_iafctrl &= ~fc; | iafc->pc_iafctrl &= ~fc; | ||||
PMCDBG(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl); | PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl); | ||||
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | ||||
wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); | wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); | ||||
do { | do { | ||||
iafc->pc_resync = 0; | iafc->pc_resync = 0; | ||||
iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET)); | iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET)); | ||||
msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; | msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; | ||||
wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & | wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & | ||||
IAF_GLOBAL_CTRL_MASK)); | IAF_GLOBAL_CTRL_MASK)); | ||||
} while (iafc->pc_resync != 0); | } while (iafc->pc_resync != 0); | ||||
PMCDBG(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", | PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", | ||||
iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), | iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), | ||||
iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); | iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iaf_write_pmc(int cpu, int ri, pmc_value_t v) | iaf_write_pmc(int cpu, int ri, pmc_value_t v) | ||||
Show All 21 Lines | iaf_write_pmc(int cpu, int ri, pmc_value_t v) | ||||
wrmsr(IAF_CTRL, msr); | wrmsr(IAF_CTRL, msr); | ||||
wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); | wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); | ||||
/* Turn on fixed counters */ | /* Turn on fixed counters */ | ||||
msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; | ||||
wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK)); | wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK)); | ||||
PMCDBG(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " | PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " | ||||
"pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, | "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, | ||||
(uintmax_t) rdmsr(IAF_CTRL), | (uintmax_t) rdmsr(IAF_CTRL), | ||||
(uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); | (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) | iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) | ||||
{ | { | ||||
struct pmc_classdep *pcd; | struct pmc_classdep *pcd; | ||||
KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); | KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); | ||||
PMCDBG(MDP,INI,1, "%s", "iaf-initialize"); | PMCDBG0(MDP,INI,1, "iaf-initialize"); | ||||
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; | pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; | ||||
pcd->pcd_caps = IAF_PMC_CAPS; | pcd->pcd_caps = IAF_PMC_CAPS; | ||||
pcd->pcd_class = PMC_CLASS_IAF; | pcd->pcd_class = PMC_CLASS_IAF; | ||||
pcd->pcd_num = npmc; | pcd->pcd_num = npmc; | ||||
pcd->pcd_ri = md->pmd_npmc; | pcd->pcd_ri = md->pmd_npmc; | ||||
pcd->pcd_width = pmcwidth; | pcd->pcd_width = pmcwidth; | ||||
▲ Show 20 Lines • Show All 1,761 Lines • ▼ Show 20 Lines | |||||
iap_config_pmc(int cpu, int ri, struct pmc *pm) | iap_config_pmc(int cpu, int ri, struct pmc *pm) | ||||
{ | { | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU %d", __LINE__, cpu)); | ("[core,%d] illegal CPU %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iap_npmc, | KASSERT(ri >= 0 && ri < core_iap_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
PMCDBG(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); | PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); | ||||
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, | KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, | ||||
cpu)); | cpu)); | ||||
core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; | core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | KASSERT(pm, | ||||
ri)); | ri)); | ||||
tmp = rdpmc(ri); | tmp = rdpmc(ri); | ||||
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | ||||
*v = iap_perfctr_value_to_reload_count(tmp); | *v = iap_perfctr_value_to_reload_count(tmp); | ||||
else | else | ||||
*v = tmp & ((1ULL << core_iap_width) - 1); | *v = tmp & ((1ULL << core_iap_width) - 1); | ||||
PMCDBG(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, | PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, | ||||
ri, *v); | ri, *v); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iap_release_pmc(int cpu, int ri, struct pmc *pm) | iap_release_pmc(int cpu, int ri, struct pmc *pm) | ||||
{ | { | ||||
(void) pm; | (void) pm; | ||||
PMCDBG(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, | PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, | ||||
pm); | pm); | ||||
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), | ||||
("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ("[core,%d] illegal CPU value %d", __LINE__, cpu)); | ||||
KASSERT(ri >= 0 && ri < core_iap_npmc, | KASSERT(ri >= 0 && ri < core_iap_npmc, | ||||
("[core,%d] illegal row-index %d", __LINE__, ri)); | ("[core,%d] illegal row-index %d", __LINE__, ri)); | ||||
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc | KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc | ||||
Show All 16 Lines | iap_start_pmc(int cpu, int ri) | ||||
cc = core_pcpu[cpu]; | cc = core_pcpu[cpu]; | ||||
pm = cc->pc_corepmcs[ri].phw_pmc; | pm = cc->pc_corepmcs[ri].phw_pmc; | ||||
KASSERT(pm, | KASSERT(pm, | ||||
("[core,%d] starting cpu%d,ri%d with no pmc configured", | ("[core,%d] starting cpu%d,ri%d with no pmc configured", | ||||
__LINE__, cpu, ri)); | __LINE__, cpu, ri)); | ||||
PMCDBG(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); | PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); | ||||
evsel = pm->pm_md.pm_iap.pm_iap_evsel; | evsel = pm->pm_md.pm_iap.pm_iap_evsel; | ||||
PMCDBG(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", | PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", | ||||
cpu, ri, IAP_EVSEL0 + ri, evsel); | cpu, ri, IAP_EVSEL0 + ri, evsel); | ||||
/* Event specific configuration. */ | /* Event specific configuration. */ | ||||
switch (pm->pm_event) { | switch (pm->pm_event) { | ||||
case PMC_EV_IAP_EVENT_B7H_01H: | case PMC_EV_IAP_EVENT_B7H_01H: | ||||
wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); | wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); | ||||
break; | break; | ||||
case PMC_EV_IAP_EVENT_BBH_01H: | case PMC_EV_IAP_EVENT_BBH_01H: | ||||
Show All 31 Lines | iap_stop_pmc(int cpu, int ri) | ||||
cc = core_pcpu[cpu]; | cc = core_pcpu[cpu]; | ||||
pm = cc->pc_corepmcs[ri].phw_pmc; | pm = cc->pc_corepmcs[ri].phw_pmc; | ||||
KASSERT(pm, | KASSERT(pm, | ||||
("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, | ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, | ||||
cpu, ri)); | cpu, ri)); | ||||
PMCDBG(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); | PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); | ||||
msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; | msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; | ||||
wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */ | wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */ | ||||
if (core_cputype == PMC_CPU_INTEL_CORE) | if (core_cputype == PMC_CPU_INTEL_CORE) | ||||
return (0); | return (0); | ||||
msr = 0; | msr = 0; | ||||
Show All 20 Lines | iap_write_pmc(int cpu, int ri, pmc_value_t v) | ||||
cc = core_pcpu[cpu]; | cc = core_pcpu[cpu]; | ||||
pm = cc->pc_corepmcs[ri].phw_pmc; | pm = cc->pc_corepmcs[ri].phw_pmc; | ||||
KASSERT(pm, | KASSERT(pm, | ||||
("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, | ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, | ||||
cpu, ri)); | cpu, ri)); | ||||
PMCDBG(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, | PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, | ||||
IAP_PMC0 + ri, v); | IAP_PMC0 + ri, v); | ||||
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | ||||
v = iap_reload_count_to_perfctr_value(v); | v = iap_reload_count_to_perfctr_value(v); | ||||
/* | /* | ||||
* Write the new value to the counter. The counter will be in | * Write the new value to the counter. The counter will be in | ||||
* a stopped state when the pcd_write() entry point is called. | * a stopped state when the pcd_write() entry point is called. | ||||
*/ | */ | ||||
wrmsr(IAP_PMC0 + ri, v & ((1ULL << core_iap_width) - 1)); | wrmsr(IAP_PMC0 + ri, v & ((1ULL << core_iap_width) - 1)); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, | iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, | ||||
int flags) | int flags) | ||||
{ | { | ||||
struct pmc_classdep *pcd; | struct pmc_classdep *pcd; | ||||
KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); | KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); | ||||
PMCDBG(MDP,INI,1, "%s", "iap-initialize"); | PMCDBG0(MDP,INI,1, "iap-initialize"); | ||||
/* Remember the set of architectural events supported. */ | /* Remember the set of architectural events supported. */ | ||||
core_architectural_events = ~flags; | core_architectural_events = ~flags; | ||||
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; | pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; | ||||
pcd->pcd_caps = IAP_PMC_CAPS; | pcd->pcd_caps = IAP_PMC_CAPS; | ||||
pcd->pcd_class = PMC_CLASS_IAP; | pcd->pcd_class = PMC_CLASS_IAP; | ||||
Show All 21 Lines | |||||
core_intr(int cpu, struct trapframe *tf) | core_intr(int cpu, struct trapframe *tf) | ||||
{ | { | ||||
pmc_value_t v; | pmc_value_t v; | ||||
struct pmc *pm; | struct pmc *pm; | ||||
struct core_cpu *cc; | struct core_cpu *cc; | ||||
int error, found_interrupt, ri; | int error, found_interrupt, ri; | ||||
uint64_t msr; | uint64_t msr; | ||||
PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, | PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, | ||||
TRAPF_USERMODE(tf)); | TRAPF_USERMODE(tf)); | ||||
found_interrupt = 0; | found_interrupt = 0; | ||||
cc = core_pcpu[cpu]; | cc = core_pcpu[cpu]; | ||||
for (ri = 0; ri < core_iap_npmc; ri++) { | for (ri = 0; ri < core_iap_npmc; ri++) { | ||||
if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || | if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
core2_intr(int cpu, struct trapframe *tf) | core2_intr(int cpu, struct trapframe *tf) | ||||
{ | { | ||||
int error, found_interrupt, n; | int error, found_interrupt, n; | ||||
uint64_t flag, intrstatus, intrenable, msr; | uint64_t flag, intrstatus, intrenable, msr; | ||||
struct pmc *pm; | struct pmc *pm; | ||||
struct core_cpu *cc; | struct core_cpu *cc; | ||||
pmc_value_t v; | pmc_value_t v; | ||||
PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, | PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, | ||||
TRAPF_USERMODE(tf)); | TRAPF_USERMODE(tf)); | ||||
/* | /* | ||||
* The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which | * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which | ||||
* PMCs have a pending PMI interrupt. We take a 'snapshot' of | * PMCs have a pending PMI interrupt. We take a 'snapshot' of | ||||
* the current set of interrupting PMCs and process these | * the current set of interrupting PMCs and process these | ||||
* after stopping them. | * after stopping them. | ||||
*/ | */ | ||||
intrstatus = rdmsr(IA_GLOBAL_STATUS); | intrstatus = rdmsr(IA_GLOBAL_STATUS); | ||||
intrenable = intrstatus & core_pmcmask; | intrenable = intrstatus & core_pmcmask; | ||||
PMCDBG(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, | PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, | ||||
(uintmax_t) intrstatus); | (uintmax_t) intrstatus); | ||||
found_interrupt = 0; | found_interrupt = 0; | ||||
cc = core_pcpu[cpu]; | cc = core_pcpu[cpu]; | ||||
KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); | KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); | ||||
cc->pc_globalctrl &= ~intrenable; | cc->pc_globalctrl &= ~intrenable; | ||||
Show All 29 Lines | for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; | ||||
if (error) | if (error) | ||||
intrenable &= ~flag; | intrenable &= ~flag; | ||||
v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); | v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); | ||||
/* Reload sampling count. */ | /* Reload sampling count. */ | ||||
wrmsr(IAF_CTR0 + n, v); | wrmsr(IAF_CTR0 + n, v); | ||||
PMCDBG(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu, | PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu, | ||||
error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); | error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); | ||||
} | } | ||||
/* | /* | ||||
* Process interrupts from the programmable counters. | * Process interrupts from the programmable counters. | ||||
*/ | */ | ||||
for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { | for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { | ||||
if ((intrstatus & flag) == 0) | if ((intrstatus & flag) == 0) | ||||
continue; | continue; | ||||
found_interrupt = 1; | found_interrupt = 1; | ||||
pm = cc->pc_corepmcs[n].phw_pmc; | pm = cc->pc_corepmcs[n].phw_pmc; | ||||
if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || | if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || | ||||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) | ||||
continue; | continue; | ||||
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, | error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, | ||||
TRAPF_USERMODE(tf)); | TRAPF_USERMODE(tf)); | ||||
if (error) | if (error) | ||||
intrenable &= ~flag; | intrenable &= ~flag; | ||||
v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); | v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); | ||||
PMCDBG(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, | PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, | ||||
(uintmax_t) v); | (uintmax_t) v); | ||||
/* Reload sampling count. */ | /* Reload sampling count. */ | ||||
wrmsr(IAP_PMC0 + n, v); | wrmsr(IAP_PMC0 + n, v); | ||||
} | } | ||||
/* | /* | ||||
* Reenable all non-stalled PMCs. | * Reenable all non-stalled PMCs. | ||||
*/ | */ | ||||
PMCDBG(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu, | PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu, | ||||
(uintmax_t) intrenable); | (uintmax_t) intrenable); | ||||
cc->pc_globalctrl |= intrenable; | cc->pc_globalctrl |= intrenable; | ||||
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK); | wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK); | ||||
PMCDBG(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx " | PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx " | ||||
"ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), | "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), | ||||
(uintmax_t) rdmsr(IA_GLOBAL_CTRL), | (uintmax_t) rdmsr(IA_GLOBAL_CTRL), | ||||
(uintmax_t) rdmsr(IA_GLOBAL_STATUS), | (uintmax_t) rdmsr(IA_GLOBAL_STATUS), | ||||
(uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL)); | (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL)); | ||||
if (found_interrupt) | if (found_interrupt) | ||||
lapic_reenable_pmc(); | lapic_reenable_pmc(); | ||||
Show All 10 Lines | pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) | ||||
int ipa_version, flags, nflags; | int ipa_version, flags, nflags; | ||||
do_cpuid(CORE_CPUID_REQUEST, cpuid); | do_cpuid(CORE_CPUID_REQUEST, cpuid); | ||||
ipa_version = (version_override > 0) ? version_override : | ipa_version = (version_override > 0) ? version_override : | ||||
cpuid[CORE_CPUID_EAX] & 0xFF; | cpuid[CORE_CPUID_EAX] & 0xFF; | ||||
core_cputype = md->pmd_cputype; | core_cputype = md->pmd_cputype; | ||||
PMCDBG(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d", | PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d", | ||||
core_cputype, maxcpu, ipa_version); | core_cputype, maxcpu, ipa_version); | ||||
if (ipa_version < 1 || ipa_version > 3 || | if (ipa_version < 1 || ipa_version > 3 || | ||||
(core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) { | (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) { | ||||
/* Unknown PMC architecture. */ | /* Unknown PMC architecture. */ | ||||
printf("hwpc_core: unknown PMC architecture: %d\n", | printf("hwpc_core: unknown PMC architecture: %d\n", | ||||
ipa_version); | ipa_version); | ||||
return (EPROGMISMATCH); | return (EPROGMISMATCH); | ||||
Show All 21 Lines | if (core_cputype != PMC_CPU_INTEL_CORE) { | ||||
core_iaf_ri = core_iap_npmc; | core_iaf_ri = core_iap_npmc; | ||||
core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; | core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; | ||||
core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; | core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; | ||||
iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); | iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); | ||||
core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; | core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; | ||||
} | } | ||||
PMCDBG(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, | PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, | ||||
core_iaf_ri); | core_iaf_ri); | ||||
core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, | core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, | ||||
M_ZERO | M_WAITOK); | M_ZERO | M_WAITOK); | ||||
/* | /* | ||||
* Choose the appropriate interrupt handler. | * Choose the appropriate interrupt handler. | ||||
*/ | */ | ||||
if (ipa_version == 1) | if (ipa_version == 1) | ||||
md->pmd_intr = core_intr; | md->pmd_intr = core_intr; | ||||
else | else | ||||
md->pmd_intr = core2_intr; | md->pmd_intr = core2_intr; | ||||
md->pmd_pcpu_fini = NULL; | md->pmd_pcpu_fini = NULL; | ||||
md->pmd_pcpu_init = NULL; | md->pmd_pcpu_init = NULL; | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
pmc_core_finalize(struct pmc_mdep *md) | pmc_core_finalize(struct pmc_mdep *md) | ||||
{ | { | ||||
PMCDBG(MDP,INI,1, "%s", "core-finalize"); | PMCDBG0(MDP,INI,1, "core-finalize"); | ||||
free(core_pcpu, M_PMC); | free(core_pcpu, M_PMC); | ||||
core_pcpu = NULL; | core_pcpu = NULL; | ||||
} | } |